diff --git a/app/api/v1/routes/voice_playground.py b/app/api/v1/routes/voice_playground.py
index 2a2ed876..2f1963db 100644
--- a/app/api/v1/routes/voice_playground.py
+++ b/app/api/v1/routes/voice_playground.py
@@ -175,6 +175,8 @@ class TTSComparisonCreate(BaseModel):
voices_b: Optional[List[Dict[str, Any]]] = None
sample_texts: List[str]
num_runs: int = 1
+ eval_stt_provider: Optional[str] = None
+ eval_stt_model: Optional[str] = None
class BlindTestSubmit(BaseModel):
@@ -663,6 +665,8 @@ async def create_comparison(
voices_b=[v if isinstance(v, dict) else {"id": v} for v in voices_b] if voices_b else [],
sample_texts=data.sample_texts,
num_runs=num_runs,
+ eval_stt_provider=data.eval_stt_provider,
+ eval_stt_model=data.eval_stt_model,
)
db.add(comparison)
db.flush()
@@ -1248,6 +1252,8 @@ def _serialize_comparison(c: TTSComparison, db: Session) -> Dict[str, Any]:
"num_runs": c.num_runs or 1,
"blind_test_results": c.blind_test_results,
"evaluation_summary": c.evaluation_summary,
+ "eval_stt_provider": getattr(c, "eval_stt_provider", None),
+ "eval_stt_model": getattr(c, "eval_stt_model", None),
"error_message": c.error_message,
"samples": serialized_samples,
"created_at": c.created_at.isoformat() if c.created_at else None,
diff --git a/app/migrations/015_add_eval_stt_columns_to_tts_comparisons.py b/app/migrations/015_add_eval_stt_columns_to_tts_comparisons.py
new file mode 100644
index 00000000..1eac37a3
--- /dev/null
+++ b/app/migrations/015_add_eval_stt_columns_to_tts_comparisons.py
@@ -0,0 +1,47 @@
+"""
+Migration: Add eval_stt_provider and eval_stt_model to tts_comparisons.
+
+Allows per-comparison STT provider override for WER/CER evaluation.
+When NULL the worker falls back to the org's first Voice Bundle STT config.
+"""
+
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+description = "Add eval_stt_provider and eval_stt_model columns to tts_comparisons"
+
+
+def upgrade(db: Session):
+ existing = db.execute(
+ text(
+ """
+ SELECT column_name
+ FROM information_schema.columns
+ WHERE table_name = 'tts_comparisons'
+ AND column_name = 'eval_stt_provider'
+ """
+ )
+ )
+
+ if existing.fetchone() is not None:
+ print("Column eval_stt_provider already exists on tts_comparisons, skipping...")
+ return
+
+ db.execute(
+ text(
+ """
+ ALTER TABLE tts_comparisons
+ ADD COLUMN eval_stt_provider VARCHAR(100),
+ ADD COLUMN eval_stt_model VARCHAR(100)
+ """
+ )
+ )
+
+ db.commit()
+ print("Added eval_stt_provider and eval_stt_model columns to tts_comparisons")
+
+
+def downgrade(db: Session):
+ db.execute(text("ALTER TABLE tts_comparisons DROP COLUMN IF EXISTS eval_stt_provider"))
+ db.execute(text("ALTER TABLE tts_comparisons DROP COLUMN IF EXISTS eval_stt_model"))
+ db.commit()
diff --git a/app/models/database.py b/app/models/database.py
index e03c2793..fdc30c74 100644
--- a/app/models/database.py
+++ b/app/models/database.py
@@ -750,6 +750,9 @@ class TTSComparison(Base):
blind_test_results = Column(JSON, nullable=True)
evaluation_summary = Column(JSON, nullable=True)
+ eval_stt_provider = Column(String(100), nullable=True)
+ eval_stt_model = Column(String(100), nullable=True)
+
celery_task_id = Column(String, nullable=True, index=True)
error_message = Column(String, nullable=True)
diff --git a/app/services/ai/stt_clients.py b/app/services/ai/stt_clients.py
new file mode 100644
index 00000000..8257082a
--- /dev/null
+++ b/app/services/ai/stt_clients.py
@@ -0,0 +1,185 @@
+"""
+Centralized STT provider API clients for batch / file-based transcription.
+
+Every provider-specific HTTP or SDK call for transcribing an audio *file*
+lives here. ``TranscriptionService`` (and any future consumer) delegates
+to these functions so that endpoint URLs, header conventions, and SDK
+usage patterns are defined in exactly one place.
+
+The streaming / real-time STT services under ``src/efficientai/services/``
+serve a different purpose (WebSocket streams, frame pipelines) and are
+intentionally separate.
+"""
+
+import logging
+from typing import Any, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+
+def transcribe_openai(
+ audio_file_path: str,
+ model: str,
+ api_key: str,
+ language: Optional[str] = None,
+) -> Dict[str, Any]:
+ """Transcribe an audio file via the OpenAI Whisper / GPT-4o transcription API.
+
+ Returns the standardised dict ``{"text", "language", "segments", "words"}``.
+ """
+ try:
+ from openai import OpenAI
+ except ImportError:
+ raise RuntimeError("OpenAI library not installed. Install with: pip install openai")
+
+ client = OpenAI(api_key=api_key)
+
+ with open(audio_file_path, "rb") as audio_file:
+ transcript = client.audio.transcriptions.create(
+ model=model,
+ file=audio_file,
+ language=language,
+ response_format="verbose_json",
+ timestamp_granularities=["word", "segment"],
+ )
+
+ result: Dict[str, Any] = {
+ "text": transcript.text if hasattr(transcript, "text") else str(transcript),
+ "language": (
+ getattr(transcript, "language", language)
+ if language
+ else getattr(transcript, "language", "en")
+ ),
+ "segments": [],
+ "words": [],
+ }
+
+ # --- segments -----------------------------------------------------------
+ raw_segments = (
+ getattr(transcript, "segments", None)
+ if hasattr(transcript, "segments")
+ else (transcript.get("segments") if isinstance(transcript, dict) else None)
+ )
+ if raw_segments:
+ for seg in raw_segments:
+ if isinstance(seg, dict):
+ result["segments"].append(
+ {"start": seg.get("start", 0), "end": seg.get("end", 0), "text": seg.get("text", "")}
+ )
+ else:
+ result["segments"].append(
+ {"start": getattr(seg, "start", 0), "end": getattr(seg, "end", 0), "text": getattr(seg, "text", "")}
+ )
+
+ # --- words --------------------------------------------------------------
+ raw_words = (
+ getattr(transcript, "words", None)
+ if hasattr(transcript, "words")
+ else (transcript.get("words") if isinstance(transcript, dict) else None)
+ )
+ if raw_words:
+ for w in raw_words:
+ if isinstance(w, dict):
+ result["words"].append(
+ {"word": w.get("word", ""), "start": w.get("start", 0) or 0, "end": w.get("end", 0) or 0}
+ )
+ else:
+ word_start = getattr(w, "start", None)
+ if word_start is None:
+ word_start = getattr(w, "start_time", 0) or 0
+ word_end = getattr(w, "end", None)
+ if word_end is None:
+ word_end = getattr(w, "end_time", 0) or 0
+ result["words"].append(
+ {
+ "word": getattr(w, "word", "") or "",
+ "start": float(word_start) if word_start else 0.0,
+ "end": float(word_end) if word_end else 0.0,
+ }
+ )
+
+ # Synthesise a single segment when the API returned none
+ if not result["segments"] and result["text"]:
+ import re
+
+ sentences = [s.strip() for s in re.split(r"[.!?]+\s+", result["text"].strip()) if s.strip()]
+ if sentences:
+ current_time = 0.0
+ for sentence in sentences:
+ dur = max(0.5, (len(sentence.split()) / 150.0) * 60.0)
+ result["segments"].append({"start": current_time, "end": current_time + dur, "text": sentence})
+ current_time += dur
+ else:
+ word_count = len(result["text"].split())
+ est = max(1.0, (word_count / 150.0) * 60.0)
+ result["segments"] = [{"start": 0.0, "end": est, "text": result["text"]}]
+
+ return result
+
+
+def transcribe_deepgram(
+ audio_file_path: str,
+ model: str,
+ api_key: str,
+ language: Optional[str] = None,
+) -> Dict[str, Any]:
+ """Transcribe an audio file via the Deepgram REST (prerecorded) API.
+
+ Uses the ``deepgram-sdk`` which is already a project dependency.
+ """
+ from deepgram import DeepgramClient
+
+ client = DeepgramClient(api_key=api_key)
+
+ with open(audio_file_path, "rb") as f:
+ audio_bytes = f.read()
+
+ options: Dict[str, Any] = {"model": model or "nova-2", "smart_format": True}
+ if language:
+ options["language"] = language
+
+ response = client.listen.rest.v("1").transcribe_file({"buffer": audio_bytes}, options)
+
+ transcript = ""
+ try:
+ transcript = response.results.channels[0].alternatives[0].transcript
+ except (AttributeError, IndexError):
+ pass
+
+ return {"text": transcript, "language": language or "en", "segments": []}
+
+
+def transcribe_elevenlabs(
+ audio_file_path: str,
+ model: str,
+ api_key: str,
+ language: Optional[str] = None,
+) -> Dict[str, Any]:
+ """Transcribe an audio file via the ElevenLabs Speech-to-Text REST API.
+
+ Uses ``httpx`` for a synchronous POST (no ElevenLabs SDK in the project).
+ The endpoint, headers, and form-field names are identical to those used by
+ ``src/efficientai/services/elevenlabs/stt.py`` (async variant).
+ """
+ import httpx
+
+ url = "https://api.elevenlabs.io/v1/speech-to-text"
+ headers = {"xi-api-key": api_key}
+
+ with open(audio_file_path, "rb") as f:
+ audio_bytes = f.read()
+
+ files = {"file": ("audio.wav", audio_bytes, "audio/wav")}
+ data: Dict[str, str] = {"model_id": model or "scribe_v2"}
+ if language:
+ data["language_code"] = language
+
+ resp = httpx.post(url, headers=headers, files=files, data=data, timeout=60.0)
+ resp.raise_for_status()
+ result = resp.json()
+
+ return {
+ "text": result.get("text", ""),
+ "language": result.get("language_code", language or "en"),
+ "segments": [],
+ }
diff --git a/app/services/ai/transcription_service.py b/app/services/ai/transcription_service.py
index 3b962038..c8aea450 100644
--- a/app/services/ai/transcription_service.py
+++ b/app/services/ai/transcription_service.py
@@ -46,7 +46,7 @@
from uuid import UUID
from pathlib import Path
-from app.models.database import ModelProvider, AIProvider
+from app.models.database import ModelProvider, AIProvider, Integration
from app.services.storage.s3_service import s3_service
from app.core.exceptions import StorageError
from sqlalchemy.orm import Session
@@ -85,6 +85,32 @@ def _get_ai_provider(self, provider: ModelProvider, db: Session, organization_id
return ai_provider
+ def _get_api_key_for_provider(
+ self, provider: ModelProvider, db: Session, organization_id: UUID
+ ) -> Optional[str]:
+ """Resolve and decrypt API key from AIProvider or Integration tables.
+
+ Checks AIProvider first (LLM-style providers like OpenAI), then falls
+ back to the Integration table (voice platforms like Deepgram, ElevenLabs).
+ """
+ from app.core.encryption import decrypt_api_key
+ from sqlalchemy import func
+
+ ai_provider = self._get_ai_provider(provider, db, organization_id)
+ if ai_provider:
+ return decrypt_api_key(ai_provider.api_key)
+
+ provider_value = provider.value if hasattr(provider, "value") else provider
+ integration = db.query(Integration).filter(
+ func.lower(Integration.platform) == provider_value.lower(),
+ Integration.organization_id == organization_id,
+ Integration.is_active == True,
+ ).first()
+ if integration:
+ return decrypt_api_key(integration.api_key)
+
+ return None
+
def _download_audio_to_temp(self, audio_file_key: str, db: Optional[Session] = None) -> str:
"""
Download audio from S3 to temporary file, or use local file if S3 is not available.
@@ -131,119 +157,11 @@ def _download_audio_to_temp(self, audio_file_key: str, db: Optional[Session] = N
# If all else fails, raise error
raise StorageError(f"Failed to download audio file: S3 is not enabled and local file not found for key: {audio_file_key}")
- def _transcribe_with_openai(self, audio_file_path: str, model: str, api_key: str, language: Optional[str] = None) -> Dict[str, Any]:
- """Transcribe audio using OpenAI Whisper API with word-level timestamps."""
- try:
- from openai import OpenAI
-
- client = OpenAI(api_key=api_key)
-
- with open(audio_file_path, "rb") as audio_file:
- transcript = client.audio.transcriptions.create(
- model=model,
- file=audio_file,
- language=language,
- response_format="verbose_json",
- timestamp_granularities=["word", "segment"],
- )
-
- result = {
- "text": transcript.text if hasattr(transcript, "text") else str(transcript),
- "language": getattr(transcript, "language", language) if language else getattr(transcript, "language", "en"),
- "segments": [],
- "words": [],
- }
+ # Provider-specific transcription is delegated to app.services.ai.stt_clients
- if hasattr(transcript, "segments") and transcript.segments:
- for seg in transcript.segments:
- result["segments"].append(
- {
- "start": getattr(seg, "start", 0),
- "end": getattr(seg, "end", 0),
- "text": getattr(seg, "text", ""),
- }
- )
- elif isinstance(transcript, dict) and "segments" in transcript:
- for seg in transcript["segments"]:
- result["segments"].append(
- {
- "start": seg.get("start", 0),
- "end": seg.get("end", 0),
- "text": seg.get("text", ""),
- }
- )
-
- if hasattr(transcript, "words") and transcript.words:
- for w in transcript.words:
- # Handle both object attributes and dict-like access
- if isinstance(w, dict):
- word_text = w.get("word", "")
- word_start = w.get("start", 0) or 0
- word_end = w.get("end", 0) or 0
- else:
- word_text = getattr(w, "word", "") or ""
- word_start = getattr(w, "start", None)
- word_end = getattr(w, "end", None)
- # Some SDK versions may use 'start_time'/'end_time'
- if word_start is None:
- word_start = getattr(w, "start_time", 0) or 0
- if word_end is None:
- word_end = getattr(w, "end_time", 0) or 0
- word_start = float(word_start) if word_start else 0.0
- word_end = float(word_end) if word_end else 0.0
- result["words"].append(
- {
- "word": word_text,
- "start": word_start,
- "end": word_end,
- }
- )
- elif isinstance(transcript, dict) and "words" in transcript:
- for w in transcript["words"]:
- result["words"].append(
- {
- "word": w.get("word", ""),
- "start": w.get("start", 0) or 0,
- "end": w.get("end", 0) or 0,
- }
- )
-
- if not result["segments"] and result["text"]:
- import re
- sentences = re.split(r"[.!?]+\s+", result["text"].strip())
- sentences = [s.strip() for s in sentences if s.strip()]
-
- if sentences:
- total_words = len(result["text"].split())
- estimated_duration = max(1.0, (total_words / 150.0) * 60.0)
-
- current_time = 0.0
- for sentence in sentences:
- sentence_words = len(sentence.split())
- sentence_duration = max(0.5, (sentence_words / 150.0) * 60.0)
- result["segments"].append(
- {
- "start": current_time,
- "end": current_time + sentence_duration,
- "text": sentence,
- }
- )
- current_time += sentence_duration
- else:
- word_count = len(result["text"].split())
- estimated_duration = max(1.0, (word_count / 150.0) * 60.0)
- result["segments"] = [{"start": 0.0, "end": estimated_duration, "text": result["text"]}]
-
- return result
- except ImportError:
- raise RuntimeError("OpenAI library not installed. Install with: pip install openai")
- except Exception as e:
- import traceback
- error_details = traceback.format_exc()
- raise RuntimeError(f"OpenAI transcription failed: {str(e)}\nDetails: {error_details}")
-
- def _transcribe_with_whisper_local(self, audio_file_path: str, model_name: str = "base") -> Dict[str, Any]:
- """Transcribe audio using local Whisper model."""
+ @staticmethod
+ def _transcribe_with_whisper_local(audio_file_path: str, model_name: str = "base") -> Dict[str, Any]:
+ """Transcribe audio using local Whisper model (not a remote API)."""
try:
import whisper
@@ -576,6 +494,47 @@ def _detect_speakers_heuristic(self, segments: List[Dict[str, Any]]) -> List[Dic
return speaker_segments
+ def transcribe_text_only(
+ self,
+ audio_file_path: str,
+ stt_provider: ModelProvider,
+ stt_model: str,
+ organization_id: UUID,
+ db: Session,
+ ) -> Optional[str]:
+ """Transcribe a local audio file and return just the text.
+
+ Lightweight alternative to `transcribe()` -- skips S3 download,
+ diarization, and segment extraction. Designed for WER/CER
+ evaluation where only the transcript string is needed.
+ """
+ api_key = self._get_api_key_for_provider(stt_provider, db, organization_id)
+ if not api_key:
+ logger.warning(
+ f"[TranscriptionService] No API key found for {stt_provider} "
+ f"(checked AIProvider and Integration tables) for org {organization_id}"
+ )
+ return None
+
+ from app.services.ai.stt_clients import transcribe_openai, transcribe_deepgram, transcribe_elevenlabs
+
+ try:
+ if stt_provider == ModelProvider.OPENAI:
+ result = transcribe_openai(audio_file_path, stt_model, api_key)
+ elif stt_provider == ModelProvider.DEEPGRAM:
+ result = transcribe_deepgram(audio_file_path, stt_model, api_key)
+ elif stt_provider == ModelProvider.ELEVENLABS:
+ result = transcribe_elevenlabs(audio_file_path, stt_model, api_key)
+ else:
+ logger.warning(f"[TranscriptionService] Unsupported STT provider for text-only: {stt_provider}")
+ return None
+
+ text = (result.get("text") or "").strip()
+ return text or None
+ except Exception as e:
+ logger.error(f"[TranscriptionService] text-only transcription failed ({stt_provider}/{stt_model}): {e}")
+ return None
+
def transcribe(
self,
audio_file_key: str,
@@ -596,38 +555,32 @@ def transcribe(
# Download audio to temporary file
temp_file_path = self._download_audio_to_temp(audio_file_key, db=db)
- # Get provider API key
- ai_provider = self._get_ai_provider(stt_provider, db, organization_id)
- if not ai_provider:
- raise RuntimeError(f"AI provider {stt_provider} not configured for this organization. Please configure an AI provider in the settings.")
+ api_key = self._get_api_key_for_provider(stt_provider, db, organization_id)
+ if not api_key:
+ raise RuntimeError(
+ f"No API key found for {stt_provider} (checked AIProvider and Integration tables). "
+ f"Please configure the provider in Settings."
+ )
- # Decrypt API key
- from app.core.encryption import decrypt_api_key
- try:
- api_key = decrypt_api_key(ai_provider.api_key)
- except Exception as e:
- raise RuntimeError(f"Failed to decrypt API key for provider {stt_provider}: {str(e)}")
+ from app.services.ai.stt_clients import transcribe_openai, transcribe_deepgram, transcribe_elevenlabs
- # Transcribe based on provider
if stt_provider == ModelProvider.OPENAI:
if stt_model.startswith("whisper-"):
- # Use OpenAI API
- result = self._transcribe_with_openai(temp_file_path, stt_model, api_key, language)
+ result = transcribe_openai(temp_file_path, stt_model, api_key, language)
else:
- # Fallback to local Whisper
model_name = stt_model.replace("whisper-", "") if stt_model.startswith("whisper-") else "base"
result = self._transcribe_with_whisper_local(temp_file_path, model_name)
+ elif stt_provider == ModelProvider.DEEPGRAM:
+ result = transcribe_deepgram(temp_file_path, stt_model, api_key, language)
+ elif stt_provider == ModelProvider.ELEVENLABS:
+ result = transcribe_elevenlabs(temp_file_path, stt_model, api_key, language)
elif stt_provider == ModelProvider.GOOGLE:
- # TODO: Implement Google Speech-to-Text
raise NotImplementedError("Google Speech-to-Text not yet implemented")
elif stt_provider == ModelProvider.AZURE:
- # TODO: Implement Azure Speech Services
raise NotImplementedError("Azure Speech Services not yet implemented")
elif stt_provider == ModelProvider.AWS:
- # TODO: Implement AWS Transcribe
raise NotImplementedError("AWS Transcribe not yet implemented")
else:
- # Default to local Whisper
result = self._transcribe_with_whisper_local(temp_file_path, "base")
# Apply speaker diarization if enabled
diff --git a/app/workers/tasks/tts_comparison.py b/app/workers/tasks/tts_comparison.py
index 0baa1690..64a94d14 100644
--- a/app/workers/tasks/tts_comparison.py
+++ b/app/workers/tasks/tts_comparison.py
@@ -12,9 +12,6 @@
from app.workers.config import celery_app
-# Singleton for the NeMo ASR model (loaded once per worker process)
-_nemo_asr_model = None
-
def _compute_wer_cer(ground_truth: str, predicted: str):
"""Compute raw and normalized WER/CER between reference and ASR text.
@@ -120,87 +117,33 @@ def _normalize_entities(text: str) -> str:
}
-_nemo_install_attempted = False
-
-
-def _lazy_install_nemo():
- """One-shot attempt to pip-install nemo_toolkit[asr] at runtime."""
- global _nemo_install_attempted
- if _nemo_install_attempted:
- return False
- _nemo_install_attempted = True
-
- logger.info("[TTS Eval] NeMo not found – attempting auto-install (this may take a few minutes)...")
- try:
- import subprocess, sys
- subprocess.check_call(
- [sys.executable, "-m", "pip", "install", "--quiet", "nemo_toolkit[asr]>=1.20.0"],
- timeout=600,
- )
- logger.info("[TTS Eval] nemo_toolkit[asr] installed successfully")
- return True
- except Exception as install_err:
- logger.warning(f"[TTS Eval] Auto-install of nemo_toolkit[asr] failed: {install_err}")
- return False
-
-
-def _get_nemo_asr_model():
- """Lazy-load NVIDIA NeMo Conformer CTC model for hallucination detection.
+def _resolve_stt_config(comp, db) -> tuple:
+ """Resolve STT provider/model for WER/CER evaluation.
- On first call, if NeMo is not installed, attempts a one-time pip install.
- Returns the model instance, or None if unavailable.
+ Priority:
+ 1. Per-comparison override (comp.eval_stt_provider / eval_stt_model)
+ 2. First Voice Bundle in the org that has STT configured
+ 3. (None, None) – WER/CER will be skipped
"""
- global _nemo_asr_model
+ if getattr(comp, "eval_stt_provider", None) and getattr(comp, "eval_stt_model", None):
+ return comp.eval_stt_provider, comp.eval_stt_model
- if _nemo_asr_model is not None:
- return _nemo_asr_model
-
- try:
- import nemo.collections.asr as nemo_asr
- except ImportError:
- if _lazy_install_nemo():
- try:
- import nemo.collections.asr as nemo_asr
- except ImportError:
- logger.warning("[TTS Eval] NeMo still not importable after install – WER/CER will be skipped")
- return None
- else:
- logger.warning(
- "[TTS Eval] NeMo is not installed – WER/CER hallucination metrics will be skipped. "
- "To install manually: pip install 'nemo_toolkit[asr]'"
- )
- return None
+ from app.models.database import VoiceBundle
- try:
- logger.info("[TTS Eval] Loading NeMo ASR model (stt_en_conformer_ctc_large)...")
- _nemo_asr_model = nemo_asr.models.ASRModel.from_pretrained("stt_en_conformer_ctc_large")
- logger.info("[TTS Eval] NeMo ASR model loaded successfully")
- return _nemo_asr_model
- except Exception as e:
- logger.error(f"[TTS Eval] NeMo ASR model failed to load: {e}", exc_info=True)
- return None
-
-
-def _transcribe_audio_for_eval(audio_path: str) -> str | None:
- """Transcribe an audio file using NVIDIA NeMo Conformer CTC.
-
- Runs entirely on the worker – no API key needed.
- """
- model = _get_nemo_asr_model()
- if model is None:
- return None
+ bundle = (
+ db.query(VoiceBundle)
+ .filter(
+ VoiceBundle.organization_id == comp.organization_id,
+ VoiceBundle.stt_provider.isnot(None),
+ VoiceBundle.stt_model.isnot(None),
+ )
+ .order_by(VoiceBundle.created_at)
+ .first()
+ )
+ if bundle:
+ return bundle.stt_provider, bundle.stt_model
- try:
- transcriptions = model.transcribe([audio_path])
- if transcriptions and len(transcriptions) > 0:
- text = transcriptions[0]
- if hasattr(text, "text"):
- text = text.text
- return str(text).strip() or None
- return None
- except Exception as e:
- logger.warning(f"[TTS Eval] ASR transcription failed: {e}")
- return None
+ return None, None
@celery_app.task(name="generate_tts_comparison", bind=True, max_retries=1)
@@ -363,9 +306,11 @@ def evaluate_tts_comparison_task(self, comparison_id: str):
TTSSample,
TTSComparisonStatus,
TTSSampleStatus,
+ ModelProvider,
)
from app.services.storage.s3_service import s3_service
from app.services.audio.qualitative_voice_service import qualitative_voice_service
+ from app.services.ai.transcription_service import transcription_service
db = SessionLocal()
try:
@@ -388,7 +333,15 @@ def evaluate_tts_comparison_task(self, comparison_id: str):
db.commit()
return {"evaluated": 0}
- nemo_model = _get_nemo_asr_model()
+ stt_provider_str, stt_model = _resolve_stt_config(comp, db)
+ stt_available = bool(stt_provider_str and stt_model)
+ if stt_available:
+ logger.info(f"[TTS Eval] Using STT provider {stt_provider_str}/{stt_model} for WER/CER")
+ else:
+ logger.warning(
+ "[TTS Eval] No STT provider configured (check comparison settings or Voice Bundles) "
+ "– WER/CER metrics will be skipped"
+ )
evaluated = 0
for sample in samples:
@@ -410,8 +363,14 @@ def evaluate_tts_comparison_task(self, comparison_id: str):
metrics = qualitative_voice_service.calculate_all_metrics(tmp_path)
- if nemo_model is not None and sample.text:
- asr_transcript = _transcribe_audio_for_eval(tmp_path)
+ if stt_available and sample.text:
+ asr_transcript = transcription_service.transcribe_text_only(
+ audio_file_path=tmp_path,
+ stt_provider=ModelProvider(stt_provider_str),
+ stt_model=stt_model,
+ organization_id=comp.organization_id,
+ db=db,
+ )
if asr_transcript:
score_bundle = _compute_wer_cer(sample.text, asr_transcript)
metrics["WER Raw"] = score_bundle.get("raw_wer")
diff --git a/frontend/src/pages/playground/voice/components/PlaygroundTab.tsx b/frontend/src/pages/playground/voice/components/PlaygroundTab.tsx
index 2043de92..0ed03a4f 100644
--- a/frontend/src/pages/playground/voice/components/PlaygroundTab.tsx
+++ b/frontend/src/pages/playground/voice/components/PlaygroundTab.tsx
@@ -1,4 +1,7 @@
-import { Loader2, Hash, Play, RotateCcw, ArrowRight, Volume2, Plus, CheckCircle2, Pause } from 'lucide-react'
+import { useState, useMemo } from 'react'
+import { useQuery } from '@tanstack/react-query'
+import { Loader2, Hash, Play, RotateCcw, ArrowRight, Volume2, Plus, CheckCircle2, Pause, ChevronDown, ChevronRight, Mic } from 'lucide-react'
+import { apiClient } from '../../../../lib/api'
import Button from '../../../../components/Button'
import ProviderLogo, { getProviderInfo } from '../../../../components/shared/ProviderLogo'
import { useVoicePlayground } from '../context'
@@ -32,6 +35,11 @@ export default function PlaygroundTab() {
setSampleRateB,
numRuns,
setNumRuns,
+ evalSttProvider,
+ setEvalSttProvider,
+ evalSttModel,
+ setEvalSttModel,
+ voiceBundles,
canRun,
createComparison,
isCreating,
@@ -107,6 +115,15 @@ export default function PlaygroundTab() {
+ {/* Evaluation STT Settings */}
+
+ No STT-capable providers (OpenAI, Deepgram) are configured in Integrations. Add one or configure a Voice Bundle with STT to enable WER/CER evaluation. +
+ )} + {availableSttOptions.length === 0 && bundleWithStt && ( ++ No additional STT providers available. The Voice Bundle default will be used. +
+ )} +