From 5cda82c14b6b7059287905e5ae000378815521dc Mon Sep 17 00:00:00 2001 From: Tejas Narayan Date: Sun, 29 Mar 2026 20:09:59 +0000 Subject: [PATCH 1/2] fix: wer and cer --- app/api/v1/routes/voice_playground.py | 6 + ...add_eval_stt_columns_to_tts_comparisons.py | 47 ++++++ app/models/database.py | 3 + app/services/ai/transcription_service.py | 69 ++++++++ app/workers/tasks/tts_comparison.py | 123 +++++--------- .../voice/components/PlaygroundTab.tsx | 152 +++++++++++++++++- .../voice/context/VoicePlaygroundContext.tsx | 18 +++ frontend/src/pages/playground/voice/types.ts | 2 + 8 files changed, 337 insertions(+), 83 deletions(-) create mode 100644 app/migrations/015_add_eval_stt_columns_to_tts_comparisons.py diff --git a/app/api/v1/routes/voice_playground.py b/app/api/v1/routes/voice_playground.py index 2a2ed876..2f1963db 100644 --- a/app/api/v1/routes/voice_playground.py +++ b/app/api/v1/routes/voice_playground.py @@ -175,6 +175,8 @@ class TTSComparisonCreate(BaseModel): voices_b: Optional[List[Dict[str, Any]]] = None sample_texts: List[str] num_runs: int = 1 + eval_stt_provider: Optional[str] = None + eval_stt_model: Optional[str] = None class BlindTestSubmit(BaseModel): @@ -663,6 +665,8 @@ async def create_comparison( voices_b=[v if isinstance(v, dict) else {"id": v} for v in voices_b] if voices_b else [], sample_texts=data.sample_texts, num_runs=num_runs, + eval_stt_provider=data.eval_stt_provider, + eval_stt_model=data.eval_stt_model, ) db.add(comparison) db.flush() @@ -1248,6 +1252,8 @@ def _serialize_comparison(c: TTSComparison, db: Session) -> Dict[str, Any]: "num_runs": c.num_runs or 1, "blind_test_results": c.blind_test_results, "evaluation_summary": c.evaluation_summary, + "eval_stt_provider": getattr(c, "eval_stt_provider", None), + "eval_stt_model": getattr(c, "eval_stt_model", None), "error_message": c.error_message, "samples": serialized_samples, "created_at": c.created_at.isoformat() if c.created_at else None, diff --git a/app/migrations/015_add_eval_stt_columns_to_tts_comparisons.py b/app/migrations/015_add_eval_stt_columns_to_tts_comparisons.py new file mode 100644 index 00000000..1eac37a3 --- /dev/null +++ b/app/migrations/015_add_eval_stt_columns_to_tts_comparisons.py @@ -0,0 +1,47 @@ +""" +Migration: Add eval_stt_provider and eval_stt_model to tts_comparisons. + +Allows per-comparison STT provider override for WER/CER evaluation. +When NULL the worker falls back to the org's first Voice Bundle STT config. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add eval_stt_provider and eval_stt_model columns to tts_comparisons" + + +def upgrade(db: Session): + existing = db.execute( + text( + """ + SELECT column_name + FROM information_schema.columns + WHERE table_name = 'tts_comparisons' + AND column_name = 'eval_stt_provider' + """ + ) + ) + + if existing.fetchone() is not None: + print("Column eval_stt_provider already exists on tts_comparisons, skipping...") + return + + db.execute( + text( + """ + ALTER TABLE tts_comparisons + ADD COLUMN eval_stt_provider VARCHAR(100), + ADD COLUMN eval_stt_model VARCHAR(100) + """ + ) + ) + + db.commit() + print("Added eval_stt_provider and eval_stt_model columns to tts_comparisons") + + +def downgrade(db: Session): + db.execute(text("ALTER TABLE tts_comparisons DROP COLUMN IF EXISTS eval_stt_provider")) + db.execute(text("ALTER TABLE tts_comparisons DROP COLUMN IF EXISTS eval_stt_model")) + db.commit() diff --git a/app/models/database.py b/app/models/database.py index e03c2793..fdc30c74 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -750,6 +750,9 @@ class TTSComparison(Base): blind_test_results = Column(JSON, nullable=True) evaluation_summary = Column(JSON, nullable=True) + eval_stt_provider = Column(String(100), nullable=True) + eval_stt_model = Column(String(100), nullable=True) + celery_task_id = Column(String, nullable=True, index=True) error_message = Column(String, nullable=True) diff --git a/app/services/ai/transcription_service.py b/app/services/ai/transcription_service.py index 3b962038..de01cd08 100644 --- a/app/services/ai/transcription_service.py +++ b/app/services/ai/transcription_service.py @@ -576,6 +576,75 @@ def _detect_speakers_heuristic(self, segments: List[Dict[str, Any]]) -> List[Dic return speaker_segments + def _transcribe_with_deepgram(self, audio_file_path: str, model: str, api_key: str, language: Optional[str] = None) -> Dict[str, Any]: + """Transcribe audio using the Deepgram SDK (prerecorded / REST).""" + from deepgram import DeepgramClient + + client = DeepgramClient(api_key=api_key) + + with open(audio_file_path, "rb") as f: + audio_bytes = f.read() + + options: Dict[str, Any] = {"model": model or "nova-2", "smart_format": True} + if language: + options["language"] = language + + response = client.listen.rest.v("1").transcribe_file( + {"buffer": audio_bytes}, options + ) + + transcript = "" + try: + transcript = response.results.channels[0].alternatives[0].transcript + except (AttributeError, IndexError): + pass + + return {"text": transcript, "language": language or "en", "segments": []} + + def transcribe_text_only( + self, + audio_file_path: str, + stt_provider: ModelProvider, + stt_model: str, + organization_id: UUID, + db: Session, + ) -> Optional[str]: + """Transcribe a local audio file and return just the text. + + Lightweight alternative to `transcribe()` -- skips S3 download, + diarization, and segment extraction. Designed for WER/CER + evaluation where only the transcript string is needed. + """ + ai_provider = self._get_ai_provider(stt_provider, db, organization_id) + if not ai_provider: + logger.warning( + f"[TranscriptionService] AI provider {stt_provider} not configured " + f"for org {organization_id} – cannot transcribe" + ) + return None + + from app.core.encryption import decrypt_api_key + try: + api_key = decrypt_api_key(ai_provider.api_key) + except Exception as e: + logger.error(f"[TranscriptionService] Failed to decrypt API key for {stt_provider}: {e}") + return None + + try: + if stt_provider == ModelProvider.OPENAI: + result = self._transcribe_with_openai(audio_file_path, stt_model, api_key) + elif stt_provider == ModelProvider.DEEPGRAM: + result = self._transcribe_with_deepgram(audio_file_path, stt_model, api_key) + else: + logger.warning(f"[TranscriptionService] Unsupported STT provider for text-only: {stt_provider}") + return None + + text = (result.get("text") or "").strip() + return text or None + except Exception as e: + logger.error(f"[TranscriptionService] text-only transcription failed ({stt_provider}/{stt_model}): {e}") + return None + def transcribe( self, audio_file_key: str, diff --git a/app/workers/tasks/tts_comparison.py b/app/workers/tasks/tts_comparison.py index 0baa1690..64a94d14 100644 --- a/app/workers/tasks/tts_comparison.py +++ b/app/workers/tasks/tts_comparison.py @@ -12,9 +12,6 @@ from app.workers.config import celery_app -# Singleton for the NeMo ASR model (loaded once per worker process) -_nemo_asr_model = None - def _compute_wer_cer(ground_truth: str, predicted: str): """Compute raw and normalized WER/CER between reference and ASR text. @@ -120,87 +117,33 @@ def _normalize_entities(text: str) -> str: } -_nemo_install_attempted = False - - -def _lazy_install_nemo(): - """One-shot attempt to pip-install nemo_toolkit[asr] at runtime.""" - global _nemo_install_attempted - if _nemo_install_attempted: - return False - _nemo_install_attempted = True - - logger.info("[TTS Eval] NeMo not found – attempting auto-install (this may take a few minutes)...") - try: - import subprocess, sys - subprocess.check_call( - [sys.executable, "-m", "pip", "install", "--quiet", "nemo_toolkit[asr]>=1.20.0"], - timeout=600, - ) - logger.info("[TTS Eval] nemo_toolkit[asr] installed successfully") - return True - except Exception as install_err: - logger.warning(f"[TTS Eval] Auto-install of nemo_toolkit[asr] failed: {install_err}") - return False - - -def _get_nemo_asr_model(): - """Lazy-load NVIDIA NeMo Conformer CTC model for hallucination detection. +def _resolve_stt_config(comp, db) -> tuple: + """Resolve STT provider/model for WER/CER evaluation. - On first call, if NeMo is not installed, attempts a one-time pip install. - Returns the model instance, or None if unavailable. + Priority: + 1. Per-comparison override (comp.eval_stt_provider / eval_stt_model) + 2. First Voice Bundle in the org that has STT configured + 3. (None, None) – WER/CER will be skipped """ - global _nemo_asr_model + if getattr(comp, "eval_stt_provider", None) and getattr(comp, "eval_stt_model", None): + return comp.eval_stt_provider, comp.eval_stt_model - if _nemo_asr_model is not None: - return _nemo_asr_model - - try: - import nemo.collections.asr as nemo_asr - except ImportError: - if _lazy_install_nemo(): - try: - import nemo.collections.asr as nemo_asr - except ImportError: - logger.warning("[TTS Eval] NeMo still not importable after install – WER/CER will be skipped") - return None - else: - logger.warning( - "[TTS Eval] NeMo is not installed – WER/CER hallucination metrics will be skipped. " - "To install manually: pip install 'nemo_toolkit[asr]'" - ) - return None + from app.models.database import VoiceBundle - try: - logger.info("[TTS Eval] Loading NeMo ASR model (stt_en_conformer_ctc_large)...") - _nemo_asr_model = nemo_asr.models.ASRModel.from_pretrained("stt_en_conformer_ctc_large") - logger.info("[TTS Eval] NeMo ASR model loaded successfully") - return _nemo_asr_model - except Exception as e: - logger.error(f"[TTS Eval] NeMo ASR model failed to load: {e}", exc_info=True) - return None - - -def _transcribe_audio_for_eval(audio_path: str) -> str | None: - """Transcribe an audio file using NVIDIA NeMo Conformer CTC. - - Runs entirely on the worker – no API key needed. - """ - model = _get_nemo_asr_model() - if model is None: - return None + bundle = ( + db.query(VoiceBundle) + .filter( + VoiceBundle.organization_id == comp.organization_id, + VoiceBundle.stt_provider.isnot(None), + VoiceBundle.stt_model.isnot(None), + ) + .order_by(VoiceBundle.created_at) + .first() + ) + if bundle: + return bundle.stt_provider, bundle.stt_model - try: - transcriptions = model.transcribe([audio_path]) - if transcriptions and len(transcriptions) > 0: - text = transcriptions[0] - if hasattr(text, "text"): - text = text.text - return str(text).strip() or None - return None - except Exception as e: - logger.warning(f"[TTS Eval] ASR transcription failed: {e}") - return None + return None, None @celery_app.task(name="generate_tts_comparison", bind=True, max_retries=1) @@ -363,9 +306,11 @@ def evaluate_tts_comparison_task(self, comparison_id: str): TTSSample, TTSComparisonStatus, TTSSampleStatus, + ModelProvider, ) from app.services.storage.s3_service import s3_service from app.services.audio.qualitative_voice_service import qualitative_voice_service + from app.services.ai.transcription_service import transcription_service db = SessionLocal() try: @@ -388,7 +333,15 @@ def evaluate_tts_comparison_task(self, comparison_id: str): db.commit() return {"evaluated": 0} - nemo_model = _get_nemo_asr_model() + stt_provider_str, stt_model = _resolve_stt_config(comp, db) + stt_available = bool(stt_provider_str and stt_model) + if stt_available: + logger.info(f"[TTS Eval] Using STT provider {stt_provider_str}/{stt_model} for WER/CER") + else: + logger.warning( + "[TTS Eval] No STT provider configured (check comparison settings or Voice Bundles) " + "– WER/CER metrics will be skipped" + ) evaluated = 0 for sample in samples: @@ -410,8 +363,14 @@ def evaluate_tts_comparison_task(self, comparison_id: str): metrics = qualitative_voice_service.calculate_all_metrics(tmp_path) - if nemo_model is not None and sample.text: - asr_transcript = _transcribe_audio_for_eval(tmp_path) + if stt_available and sample.text: + asr_transcript = transcription_service.transcribe_text_only( + audio_file_path=tmp_path, + stt_provider=ModelProvider(stt_provider_str), + stt_model=stt_model, + organization_id=comp.organization_id, + db=db, + ) if asr_transcript: score_bundle = _compute_wer_cer(sample.text, asr_transcript) metrics["WER Raw"] = score_bundle.get("raw_wer") diff --git a/frontend/src/pages/playground/voice/components/PlaygroundTab.tsx b/frontend/src/pages/playground/voice/components/PlaygroundTab.tsx index 2043de92..75bd3f99 100644 --- a/frontend/src/pages/playground/voice/components/PlaygroundTab.tsx +++ b/frontend/src/pages/playground/voice/components/PlaygroundTab.tsx @@ -1,4 +1,7 @@ -import { Loader2, Hash, Play, RotateCcw, ArrowRight, Volume2, Plus, CheckCircle2, Pause } from 'lucide-react' +import { useState, useMemo } from 'react' +import { useQuery } from '@tanstack/react-query' +import { Loader2, Hash, Play, RotateCcw, ArrowRight, Volume2, Plus, CheckCircle2, Pause, ChevronDown, ChevronRight, Mic } from 'lucide-react' +import { apiClient } from '../../../../lib/api' import Button from '../../../../components/Button' import ProviderLogo, { getProviderInfo } from '../../../../components/shared/ProviderLogo' import { useVoicePlayground } from '../context' @@ -32,6 +35,11 @@ export default function PlaygroundTab() { setSampleRateB, numRuns, setNumRuns, + evalSttProvider, + setEvalSttProvider, + evalSttModel, + setEvalSttModel, + voiceBundles, canRun, createComparison, isCreating, @@ -107,6 +115,15 @@ export default function PlaygroundTab() { + {/* Evaluation STT Settings */} + + {/* Provider Config Panel */}

Configure Comparison

@@ -407,3 +424,136 @@ export default function PlaygroundTab() { return null } + +const ALL_STT_OPTIONS: Array<{ provider: string; model: string; label: string }> = [ + { provider: 'openai', model: 'whisper-1', label: 'OpenAI / whisper-1' }, + { provider: 'openai', model: 'gpt-4o-transcribe', label: 'OpenAI / gpt-4o-transcribe' }, + { provider: 'openai', model: 'gpt-4o-mini-transcribe', label: 'OpenAI / gpt-4o-mini-transcribe' }, + { provider: 'deepgram', model: 'nova-2', label: 'Deepgram / nova-2' }, + { provider: 'deepgram', model: 'nova-3', label: 'Deepgram / nova-3' }, +] + +const INTEGRATION_TO_PROVIDER: Record = { + deepgram: 'deepgram', + elevenlabs: 'elevenlabs', + cartesia: 'cartesia', + sarvam: 'sarvam', +} + +function EvalSttPanel({ + evalSttProvider, + setEvalSttProvider, + evalSttModel, + setEvalSttModel, + voiceBundles, +}: { + evalSttProvider: string + setEvalSttProvider: (p: string) => void + evalSttModel: string + setEvalSttModel: (m: string) => void + voiceBundles: Array<{ id: string; name: string; stt_provider?: string | null; stt_model?: string | null }> +}) { + const [expanded, setExpanded] = useState(false) + + const { data: aiProviders = [] } = useQuery>({ + queryKey: ['ai-providers'], + queryFn: () => apiClient.listAIProviders(), + }) + + const { data: integrations = [] } = useQuery>({ + queryKey: ['integrations'], + queryFn: () => apiClient.listIntegrations(), + }) + + const activeProviderNames = useMemo(() => { + const names = new Set() + for (const p of aiProviders) { + if (p.is_active) names.add(p.provider.toLowerCase()) + } + for (const i of integrations) { + if (i.is_active) { + const mapped = INTEGRATION_TO_PROVIDER[i.platform.toLowerCase()] + if (mapped) names.add(mapped) + } + } + return names + }, [aiProviders, integrations]) + + const availableSttOptions = useMemo( + () => ALL_STT_OPTIONS.filter(opt => activeProviderNames.has(opt.provider)), + [activeProviderNames], + ) + + const bundleWithStt = voiceBundles.find(b => b.stt_provider && b.stt_model) + const autoLabel = bundleWithStt + ? `Auto — ${bundleWithStt.stt_provider} / ${bundleWithStt.stt_model} (from "${bundleWithStt.name}")` + : 'Auto — uses first Voice Bundle with STT configured' + + const selectedValue = evalSttProvider && evalSttModel + ? `${evalSttProvider}::${evalSttModel}` + : '' + + const selectedLabel = selectedValue + ? availableSttOptions.find(o => `${o.provider}::${o.model}` === selectedValue)?.label ?? `${evalSttProvider} / ${evalSttModel}` + : 'Auto' + + return ( +
+ + + {expanded && ( +
+ + {availableSttOptions.length === 0 && !bundleWithStt && ( +

+ No STT-capable providers (OpenAI, Deepgram) are configured in Integrations. Add one or configure a Voice Bundle with STT to enable WER/CER evaluation. +

+ )} + {availableSttOptions.length === 0 && bundleWithStt && ( +

+ No additional STT providers available. The Voice Bundle default will be used. +

+ )} +
+ )} +
+ ) +} diff --git a/frontend/src/pages/playground/voice/context/VoicePlaygroundContext.tsx b/frontend/src/pages/playground/voice/context/VoicePlaygroundContext.tsx index 9a78fb28..8b230751 100644 --- a/frontend/src/pages/playground/voice/context/VoicePlaygroundContext.tsx +++ b/frontend/src/pages/playground/voice/context/VoicePlaygroundContext.tsx @@ -95,6 +95,11 @@ interface VoicePlaygroundContextType { numRuns: number setNumRuns: (n: number) => void + evalSttProvider: string + setEvalSttProvider: (p: string) => void + evalSttModel: string + setEvalSttModel: (m: string) => void + // AI sample generation showAiGenerate: boolean setShowAiGenerate: (show: boolean) => void @@ -311,6 +316,9 @@ export function VoicePlaygroundProvider({ children }: { children: ReactNode }) { const [customText, setCustomText] = useState('') const [numRuns, setNumRuns] = useState(1) + const [evalSttProvider, setEvalSttProvider] = useState('') + const [evalSttModel, setEvalSttModel] = useState('') + // AI sample generation const [showAiGenerate, setShowAiGenerate] = useState(false) const [aiScenario, setAiScenario] = useState('') @@ -444,6 +452,10 @@ export function VoicePlaygroundProvider({ children }: { children: ReactNode }) { ...(sampleRateB ? { sample_rate_hz: sampleRateB } : {}), })) } + if (evalSttProvider && evalSttModel) { + payload.eval_stt_provider = evalSttProvider + payload.eval_stt_model = evalSttModel + } const comp = await apiClient.createTTSComparison(payload) await apiClient.generateTTSComparison(comp.id) return comp @@ -617,6 +629,8 @@ export function VoicePlaygroundProvider({ children }: { children: ReactNode }) { setSelectedVoicesB([]) setSampleTexts(DEFAULT_SAMPLE_TEXTS.slice(0, 1)) setNumRuns(1) + setEvalSttProvider('') + setEvalSttModel('') setActiveComparisonId(null) setStep('configure') setBlindChoices({}) @@ -685,6 +699,10 @@ export function VoicePlaygroundProvider({ children }: { children: ReactNode }) { setCustomText, numRuns, setNumRuns, + evalSttProvider, + setEvalSttProvider, + evalSttModel, + setEvalSttModel, showAiGenerate, setShowAiGenerate, aiScenario, diff --git a/frontend/src/pages/playground/voice/types.ts b/frontend/src/pages/playground/voice/types.ts index 097d103c..f476b2e2 100644 --- a/frontend/src/pages/playground/voice/types.ts +++ b/frontend/src/pages/playground/voice/types.ts @@ -51,6 +51,8 @@ export interface TTSComparison { num_runs: number blind_test_results: Array<{ sample_index: number; preferred: string }> | null evaluation_summary: Record | null + eval_stt_provider?: string | null + eval_stt_model?: string | null error_message: string | null samples: TTSSample[] created_at: string From fc5e8a6c57e4a49ab9d787fd99ab60f88f42dff8 Mon Sep 17 00:00:00 2001 From: Tejas Narayan Date: Sun, 29 Mar 2026 20:36:19 +0000 Subject: [PATCH 2/2] fix: updating stt providers stuff --- app/services/ai/stt_clients.py | 185 +++++++++++++++ app/services/ai/transcription_service.py | 220 +++++------------- .../voice/components/PlaygroundTab.tsx | 1 + 3 files changed, 238 insertions(+), 168 deletions(-) create mode 100644 app/services/ai/stt_clients.py diff --git a/app/services/ai/stt_clients.py b/app/services/ai/stt_clients.py new file mode 100644 index 00000000..8257082a --- /dev/null +++ b/app/services/ai/stt_clients.py @@ -0,0 +1,185 @@ +""" +Centralized STT provider API clients for batch / file-based transcription. + +Every provider-specific HTTP or SDK call for transcribing an audio *file* +lives here. ``TranscriptionService`` (and any future consumer) delegates +to these functions so that endpoint URLs, header conventions, and SDK +usage patterns are defined in exactly one place. + +The streaming / real-time STT services under ``src/efficientai/services/`` +serve a different purpose (WebSocket streams, frame pipelines) and are +intentionally separate. +""" + +import logging +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + + +def transcribe_openai( + audio_file_path: str, + model: str, + api_key: str, + language: Optional[str] = None, +) -> Dict[str, Any]: + """Transcribe an audio file via the OpenAI Whisper / GPT-4o transcription API. + + Returns the standardised dict ``{"text", "language", "segments", "words"}``. + """ + try: + from openai import OpenAI + except ImportError: + raise RuntimeError("OpenAI library not installed. Install with: pip install openai") + + client = OpenAI(api_key=api_key) + + with open(audio_file_path, "rb") as audio_file: + transcript = client.audio.transcriptions.create( + model=model, + file=audio_file, + language=language, + response_format="verbose_json", + timestamp_granularities=["word", "segment"], + ) + + result: Dict[str, Any] = { + "text": transcript.text if hasattr(transcript, "text") else str(transcript), + "language": ( + getattr(transcript, "language", language) + if language + else getattr(transcript, "language", "en") + ), + "segments": [], + "words": [], + } + + # --- segments ----------------------------------------------------------- + raw_segments = ( + getattr(transcript, "segments", None) + if hasattr(transcript, "segments") + else (transcript.get("segments") if isinstance(transcript, dict) else None) + ) + if raw_segments: + for seg in raw_segments: + if isinstance(seg, dict): + result["segments"].append( + {"start": seg.get("start", 0), "end": seg.get("end", 0), "text": seg.get("text", "")} + ) + else: + result["segments"].append( + {"start": getattr(seg, "start", 0), "end": getattr(seg, "end", 0), "text": getattr(seg, "text", "")} + ) + + # --- words -------------------------------------------------------------- + raw_words = ( + getattr(transcript, "words", None) + if hasattr(transcript, "words") + else (transcript.get("words") if isinstance(transcript, dict) else None) + ) + if raw_words: + for w in raw_words: + if isinstance(w, dict): + result["words"].append( + {"word": w.get("word", ""), "start": w.get("start", 0) or 0, "end": w.get("end", 0) or 0} + ) + else: + word_start = getattr(w, "start", None) + if word_start is None: + word_start = getattr(w, "start_time", 0) or 0 + word_end = getattr(w, "end", None) + if word_end is None: + word_end = getattr(w, "end_time", 0) or 0 + result["words"].append( + { + "word": getattr(w, "word", "") or "", + "start": float(word_start) if word_start else 0.0, + "end": float(word_end) if word_end else 0.0, + } + ) + + # Synthesise a single segment when the API returned none + if not result["segments"] and result["text"]: + import re + + sentences = [s.strip() for s in re.split(r"[.!?]+\s+", result["text"].strip()) if s.strip()] + if sentences: + current_time = 0.0 + for sentence in sentences: + dur = max(0.5, (len(sentence.split()) / 150.0) * 60.0) + result["segments"].append({"start": current_time, "end": current_time + dur, "text": sentence}) + current_time += dur + else: + word_count = len(result["text"].split()) + est = max(1.0, (word_count / 150.0) * 60.0) + result["segments"] = [{"start": 0.0, "end": est, "text": result["text"]}] + + return result + + +def transcribe_deepgram( + audio_file_path: str, + model: str, + api_key: str, + language: Optional[str] = None, +) -> Dict[str, Any]: + """Transcribe an audio file via the Deepgram REST (prerecorded) API. + + Uses the ``deepgram-sdk`` which is already a project dependency. + """ + from deepgram import DeepgramClient + + client = DeepgramClient(api_key=api_key) + + with open(audio_file_path, "rb") as f: + audio_bytes = f.read() + + options: Dict[str, Any] = {"model": model or "nova-2", "smart_format": True} + if language: + options["language"] = language + + response = client.listen.rest.v("1").transcribe_file({"buffer": audio_bytes}, options) + + transcript = "" + try: + transcript = response.results.channels[0].alternatives[0].transcript + except (AttributeError, IndexError): + pass + + return {"text": transcript, "language": language or "en", "segments": []} + + +def transcribe_elevenlabs( + audio_file_path: str, + model: str, + api_key: str, + language: Optional[str] = None, +) -> Dict[str, Any]: + """Transcribe an audio file via the ElevenLabs Speech-to-Text REST API. + + Uses ``httpx`` for a synchronous POST (no ElevenLabs SDK in the project). + The endpoint, headers, and form-field names are identical to those used by + ``src/efficientai/services/elevenlabs/stt.py`` (async variant). + """ + import httpx + + url = "https://api.elevenlabs.io/v1/speech-to-text" + headers = {"xi-api-key": api_key} + + with open(audio_file_path, "rb") as f: + audio_bytes = f.read() + + files = {"file": ("audio.wav", audio_bytes, "audio/wav")} + data: Dict[str, str] = {"model_id": model or "scribe_v2"} + if language: + data["language_code"] = language + + resp = httpx.post(url, headers=headers, files=files, data=data, timeout=60.0) + resp.raise_for_status() + result = resp.json() + + return { + "text": result.get("text", ""), + "language": result.get("language_code", language or "en"), + "segments": [], + } diff --git a/app/services/ai/transcription_service.py b/app/services/ai/transcription_service.py index de01cd08..c8aea450 100644 --- a/app/services/ai/transcription_service.py +++ b/app/services/ai/transcription_service.py @@ -46,7 +46,7 @@ from uuid import UUID from pathlib import Path -from app.models.database import ModelProvider, AIProvider +from app.models.database import ModelProvider, AIProvider, Integration from app.services.storage.s3_service import s3_service from app.core.exceptions import StorageError from sqlalchemy.orm import Session @@ -85,6 +85,32 @@ def _get_ai_provider(self, provider: ModelProvider, db: Session, organization_id return ai_provider + def _get_api_key_for_provider( + self, provider: ModelProvider, db: Session, organization_id: UUID + ) -> Optional[str]: + """Resolve and decrypt API key from AIProvider or Integration tables. + + Checks AIProvider first (LLM-style providers like OpenAI), then falls + back to the Integration table (voice platforms like Deepgram, ElevenLabs). + """ + from app.core.encryption import decrypt_api_key + from sqlalchemy import func + + ai_provider = self._get_ai_provider(provider, db, organization_id) + if ai_provider: + return decrypt_api_key(ai_provider.api_key) + + provider_value = provider.value if hasattr(provider, "value") else provider + integration = db.query(Integration).filter( + func.lower(Integration.platform) == provider_value.lower(), + Integration.organization_id == organization_id, + Integration.is_active == True, + ).first() + if integration: + return decrypt_api_key(integration.api_key) + + return None + def _download_audio_to_temp(self, audio_file_key: str, db: Optional[Session] = None) -> str: """ Download audio from S3 to temporary file, or use local file if S3 is not available. @@ -131,119 +157,11 @@ def _download_audio_to_temp(self, audio_file_key: str, db: Optional[Session] = N # If all else fails, raise error raise StorageError(f"Failed to download audio file: S3 is not enabled and local file not found for key: {audio_file_key}") - def _transcribe_with_openai(self, audio_file_path: str, model: str, api_key: str, language: Optional[str] = None) -> Dict[str, Any]: - """Transcribe audio using OpenAI Whisper API with word-level timestamps.""" - try: - from openai import OpenAI - - client = OpenAI(api_key=api_key) + # Provider-specific transcription is delegated to app.services.ai.stt_clients - with open(audio_file_path, "rb") as audio_file: - transcript = client.audio.transcriptions.create( - model=model, - file=audio_file, - language=language, - response_format="verbose_json", - timestamp_granularities=["word", "segment"], - ) - - result = { - "text": transcript.text if hasattr(transcript, "text") else str(transcript), - "language": getattr(transcript, "language", language) if language else getattr(transcript, "language", "en"), - "segments": [], - "words": [], - } - - if hasattr(transcript, "segments") and transcript.segments: - for seg in transcript.segments: - result["segments"].append( - { - "start": getattr(seg, "start", 0), - "end": getattr(seg, "end", 0), - "text": getattr(seg, "text", ""), - } - ) - elif isinstance(transcript, dict) and "segments" in transcript: - for seg in transcript["segments"]: - result["segments"].append( - { - "start": seg.get("start", 0), - "end": seg.get("end", 0), - "text": seg.get("text", ""), - } - ) - - if hasattr(transcript, "words") and transcript.words: - for w in transcript.words: - # Handle both object attributes and dict-like access - if isinstance(w, dict): - word_text = w.get("word", "") - word_start = w.get("start", 0) or 0 - word_end = w.get("end", 0) or 0 - else: - word_text = getattr(w, "word", "") or "" - word_start = getattr(w, "start", None) - word_end = getattr(w, "end", None) - # Some SDK versions may use 'start_time'/'end_time' - if word_start is None: - word_start = getattr(w, "start_time", 0) or 0 - if word_end is None: - word_end = getattr(w, "end_time", 0) or 0 - word_start = float(word_start) if word_start else 0.0 - word_end = float(word_end) if word_end else 0.0 - result["words"].append( - { - "word": word_text, - "start": word_start, - "end": word_end, - } - ) - elif isinstance(transcript, dict) and "words" in transcript: - for w in transcript["words"]: - result["words"].append( - { - "word": w.get("word", ""), - "start": w.get("start", 0) or 0, - "end": w.get("end", 0) or 0, - } - ) - - if not result["segments"] and result["text"]: - import re - sentences = re.split(r"[.!?]+\s+", result["text"].strip()) - sentences = [s.strip() for s in sentences if s.strip()] - - if sentences: - total_words = len(result["text"].split()) - estimated_duration = max(1.0, (total_words / 150.0) * 60.0) - - current_time = 0.0 - for sentence in sentences: - sentence_words = len(sentence.split()) - sentence_duration = max(0.5, (sentence_words / 150.0) * 60.0) - result["segments"].append( - { - "start": current_time, - "end": current_time + sentence_duration, - "text": sentence, - } - ) - current_time += sentence_duration - else: - word_count = len(result["text"].split()) - estimated_duration = max(1.0, (word_count / 150.0) * 60.0) - result["segments"] = [{"start": 0.0, "end": estimated_duration, "text": result["text"]}] - - return result - except ImportError: - raise RuntimeError("OpenAI library not installed. Install with: pip install openai") - except Exception as e: - import traceback - error_details = traceback.format_exc() - raise RuntimeError(f"OpenAI transcription failed: {str(e)}\nDetails: {error_details}") - - def _transcribe_with_whisper_local(self, audio_file_path: str, model_name: str = "base") -> Dict[str, Any]: - """Transcribe audio using local Whisper model.""" + @staticmethod + def _transcribe_with_whisper_local(audio_file_path: str, model_name: str = "base") -> Dict[str, Any]: + """Transcribe audio using local Whisper model (not a remote API).""" try: import whisper @@ -576,31 +494,6 @@ def _detect_speakers_heuristic(self, segments: List[Dict[str, Any]]) -> List[Dic return speaker_segments - def _transcribe_with_deepgram(self, audio_file_path: str, model: str, api_key: str, language: Optional[str] = None) -> Dict[str, Any]: - """Transcribe audio using the Deepgram SDK (prerecorded / REST).""" - from deepgram import DeepgramClient - - client = DeepgramClient(api_key=api_key) - - with open(audio_file_path, "rb") as f: - audio_bytes = f.read() - - options: Dict[str, Any] = {"model": model or "nova-2", "smart_format": True} - if language: - options["language"] = language - - response = client.listen.rest.v("1").transcribe_file( - {"buffer": audio_bytes}, options - ) - - transcript = "" - try: - transcript = response.results.channels[0].alternatives[0].transcript - except (AttributeError, IndexError): - pass - - return {"text": transcript, "language": language or "en", "segments": []} - def transcribe_text_only( self, audio_file_path: str, @@ -615,26 +508,23 @@ def transcribe_text_only( diarization, and segment extraction. Designed for WER/CER evaluation where only the transcript string is needed. """ - ai_provider = self._get_ai_provider(stt_provider, db, organization_id) - if not ai_provider: + api_key = self._get_api_key_for_provider(stt_provider, db, organization_id) + if not api_key: logger.warning( - f"[TranscriptionService] AI provider {stt_provider} not configured " - f"for org {organization_id} – cannot transcribe" + f"[TranscriptionService] No API key found for {stt_provider} " + f"(checked AIProvider and Integration tables) for org {organization_id}" ) return None - from app.core.encryption import decrypt_api_key - try: - api_key = decrypt_api_key(ai_provider.api_key) - except Exception as e: - logger.error(f"[TranscriptionService] Failed to decrypt API key for {stt_provider}: {e}") - return None + from app.services.ai.stt_clients import transcribe_openai, transcribe_deepgram, transcribe_elevenlabs try: if stt_provider == ModelProvider.OPENAI: - result = self._transcribe_with_openai(audio_file_path, stt_model, api_key) + result = transcribe_openai(audio_file_path, stt_model, api_key) elif stt_provider == ModelProvider.DEEPGRAM: - result = self._transcribe_with_deepgram(audio_file_path, stt_model, api_key) + result = transcribe_deepgram(audio_file_path, stt_model, api_key) + elif stt_provider == ModelProvider.ELEVENLABS: + result = transcribe_elevenlabs(audio_file_path, stt_model, api_key) else: logger.warning(f"[TranscriptionService] Unsupported STT provider for text-only: {stt_provider}") return None @@ -665,38 +555,32 @@ def transcribe( # Download audio to temporary file temp_file_path = self._download_audio_to_temp(audio_file_key, db=db) - # Get provider API key - ai_provider = self._get_ai_provider(stt_provider, db, organization_id) - if not ai_provider: - raise RuntimeError(f"AI provider {stt_provider} not configured for this organization. Please configure an AI provider in the settings.") + api_key = self._get_api_key_for_provider(stt_provider, db, organization_id) + if not api_key: + raise RuntimeError( + f"No API key found for {stt_provider} (checked AIProvider and Integration tables). " + f"Please configure the provider in Settings." + ) - # Decrypt API key - from app.core.encryption import decrypt_api_key - try: - api_key = decrypt_api_key(ai_provider.api_key) - except Exception as e: - raise RuntimeError(f"Failed to decrypt API key for provider {stt_provider}: {str(e)}") + from app.services.ai.stt_clients import transcribe_openai, transcribe_deepgram, transcribe_elevenlabs - # Transcribe based on provider if stt_provider == ModelProvider.OPENAI: if stt_model.startswith("whisper-"): - # Use OpenAI API - result = self._transcribe_with_openai(temp_file_path, stt_model, api_key, language) + result = transcribe_openai(temp_file_path, stt_model, api_key, language) else: - # Fallback to local Whisper model_name = stt_model.replace("whisper-", "") if stt_model.startswith("whisper-") else "base" result = self._transcribe_with_whisper_local(temp_file_path, model_name) + elif stt_provider == ModelProvider.DEEPGRAM: + result = transcribe_deepgram(temp_file_path, stt_model, api_key, language) + elif stt_provider == ModelProvider.ELEVENLABS: + result = transcribe_elevenlabs(temp_file_path, stt_model, api_key, language) elif stt_provider == ModelProvider.GOOGLE: - # TODO: Implement Google Speech-to-Text raise NotImplementedError("Google Speech-to-Text not yet implemented") elif stt_provider == ModelProvider.AZURE: - # TODO: Implement Azure Speech Services raise NotImplementedError("Azure Speech Services not yet implemented") elif stt_provider == ModelProvider.AWS: - # TODO: Implement AWS Transcribe raise NotImplementedError("AWS Transcribe not yet implemented") else: - # Default to local Whisper result = self._transcribe_with_whisper_local(temp_file_path, "base") # Apply speaker diarization if enabled diff --git a/frontend/src/pages/playground/voice/components/PlaygroundTab.tsx b/frontend/src/pages/playground/voice/components/PlaygroundTab.tsx index 75bd3f99..0ed03a4f 100644 --- a/frontend/src/pages/playground/voice/components/PlaygroundTab.tsx +++ b/frontend/src/pages/playground/voice/components/PlaygroundTab.tsx @@ -431,6 +431,7 @@ const ALL_STT_OPTIONS: Array<{ provider: string; model: string; label: string }> { provider: 'openai', model: 'gpt-4o-mini-transcribe', label: 'OpenAI / gpt-4o-mini-transcribe' }, { provider: 'deepgram', model: 'nova-2', label: 'Deepgram / nova-2' }, { provider: 'deepgram', model: 'nova-3', label: 'Deepgram / nova-3' }, + { provider: 'elevenlabs', model: 'scribe_v2', label: 'ElevenLabs / Scribe v2' }, ] const INTEGRATION_TO_PROVIDER: Record = {