From a7949b528559fff581ffaf8213992e855d6a979c Mon Sep 17 00:00:00 2001 From: Tejas Narayan Date: Tue, 17 Mar 2026 19:48:22 +0000 Subject: [PATCH 1/3] feat: updating lib lazy loading --- .../voice_playground_report_service.py | 43 +++++++++++++ app/workers/tasks/tts_comparison.py | 62 +++++++++++++------ docker/Dockerfile.worker | 16 ++++- 3 files changed, 99 insertions(+), 22 deletions(-) diff --git a/app/services/reporting/voice_playground_report_service.py b/app/services/reporting/voice_playground_report_service.py index cc4ac0ef..f815e6df 100644 --- a/app/services/reporting/voice_playground_report_service.py +++ b/app/services/reporting/voice_playground_report_service.py @@ -1155,14 +1155,57 @@ def _threshold_to_pct(raw_threshold: Any) -> float | None: "report_options": normalized_options, } + _weasyprint_deps_checked = False + + @staticmethod + def _ensure_weasyprint_system_deps(): + """One-shot check/install of system libraries required by WeasyPrint.""" + if VoicePlaygroundReportService._weasyprint_deps_checked: + return + VoicePlaygroundReportService._weasyprint_deps_checked = True + + import ctypes.util + if ctypes.util.find_library("gobject-2.0"): + return + + import shutil, subprocess, os + if shutil.which("apt-get") and os.geteuid() == 0: + from loguru import logger + logger.info("[PDF] WeasyPrint system libs missing – installing via apt-get...") + try: + subprocess.check_call( + ["apt-get", "update", "-qq"], + timeout=120, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + subprocess.check_call( + [ + "apt-get", "install", "-y", "-qq", + "libgobject-2.0-0", "libpango-1.0-0", "libpangocairo-1.0-0", + "libcairo2", "libgdk-pixbuf-2.0-0", "libffi-dev", "shared-mime-info", + ], + timeout=120, + ) + logger.info("[PDF] WeasyPrint system libs installed successfully") + except Exception as e: + logger.warning(f"[PDF] Auto-install of system libs failed: {e}") + def render_pdf(self, payload: dict[str, Any]) -> bytes: """Render report payload to PDF bytes.""" + self._ensure_weasyprint_system_deps() + try: from weasyprint import HTML except ImportError as exc: raise RuntimeError( "PDF generation requires weasyprint. Install dependencies and retry." ) from exc + except OSError as exc: + raise RuntimeError( + f"WeasyPrint system libraries missing: {exc}. " + "Install them with: apt-get install -y libgobject-2.0-0 libpango-1.0-0 " + "libpangocairo-1.0-0 libcairo2 libgdk-pixbuf-2.0-0 libffi-dev shared-mime-info" + ) from exc template = self._jinja_env.get_template("reports/voice_playground_report.html") html_content = template.render(**payload, service=self) diff --git a/app/workers/tasks/tts_comparison.py b/app/workers/tasks/tts_comparison.py index b028db26..0baa1690 100644 --- a/app/workers/tasks/tts_comparison.py +++ b/app/workers/tasks/tts_comparison.py @@ -120,11 +120,35 @@ def _normalize_entities(text: str) -> str: } +_nemo_install_attempted = False + + +def _lazy_install_nemo(): + """One-shot attempt to pip-install nemo_toolkit[asr] at runtime.""" + global _nemo_install_attempted + if _nemo_install_attempted: + return False + _nemo_install_attempted = True + + logger.info("[TTS Eval] NeMo not found – attempting auto-install (this may take a few minutes)...") + try: + import subprocess, sys + subprocess.check_call( + [sys.executable, "-m", "pip", "install", "--quiet", "nemo_toolkit[asr]>=1.20.0"], + timeout=600, + ) + logger.info("[TTS Eval] nemo_toolkit[asr] installed successfully") + return True + except Exception as install_err: + logger.warning(f"[TTS Eval] Auto-install of nemo_toolkit[asr] failed: {install_err}") + return False + + def _get_nemo_asr_model(): """Lazy-load NVIDIA NeMo Conformer CTC model for hallucination detection. - Requires: pip install efficientai[nemo-asr] - Returns the model instance, or None if NeMo is not installed. + On first call, if NeMo is not installed, attempts a one-time pip install. + Returns the model instance, or None if unavailable. """ global _nemo_asr_model @@ -133,30 +157,28 @@ def _get_nemo_asr_model(): try: import nemo.collections.asr as nemo_asr + except ImportError: + if _lazy_install_nemo(): + try: + import nemo.collections.asr as nemo_asr + except ImportError: + logger.warning("[TTS Eval] NeMo still not importable after install – WER/CER will be skipped") + return None + else: + logger.warning( + "[TTS Eval] NeMo is not installed – WER/CER hallucination metrics will be skipped. " + "To install manually: pip install 'nemo_toolkit[asr]'" + ) + return None + try: logger.info("[TTS Eval] Loading NeMo ASR model (stt_en_conformer_ctc_large)...") _nemo_asr_model = nemo_asr.models.ASRModel.from_pretrained("stt_en_conformer_ctc_large") logger.info("[TTS Eval] NeMo ASR model loaded successfully") return _nemo_asr_model - except ImportError as e: - logger.warning( - f"[TTS Eval] NeMo import failed: {e} – " - "WER/CER hallucination metrics will be skipped. " - "To enable, run:\n" - " pip install 'nemo_toolkit[asr]'\n" - " python -c \"import nemo.collections.asr as nemo_asr; " - "nemo_asr.models.ASRModel.from_pretrained('stt_en_conformer_ctc_large')\"" - ) except Exception as e: - logger.error( - f"[TTS Eval] NeMo ASR model failed to load: {e} – " - "The model may not be cached yet. To download it manually, run:\n" - " python -c \"import nemo.collections.asr as nemo_asr; " - "nemo_asr.models.ASRModel.from_pretrained('stt_en_conformer_ctc_large')\"", - exc_info=True, - ) - - return None + logger.error(f"[TTS Eval] NeMo ASR model failed to load: {e}", exc_info=True) + return None def _transcribe_audio_for_eval(audio_path: str) -> str | None: diff --git a/docker/Dockerfile.worker b/docker/Dockerfile.worker index 99292b81..1d582edf 100644 --- a/docker/Dockerfile.worker +++ b/docker/Dockerfile.worker @@ -4,12 +4,21 @@ FROM python:3.11-slim ARG INSTALL_EXTRAS="qualitative-voice,reports" ARG TORCH_CPU_ONLY="true" -# Install system dependencies (git needed for torch.hub clone, build-essential + -# cmake needed for packages like praat-parselmouth on arm64/Apple Silicon) +# Install system dependencies: +# - git: needed for torch.hub clone +# - build-essential + cmake: needed for packages like praat-parselmouth on arm64 +# - libgobject/libpango/libcairo/libgdk-pixbuf/libffi/shared-mime-info: WeasyPrint PDF rendering RUN apt-get update && apt-get install -y \ git \ build-essential \ cmake \ + libgobject-2.0-0 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libcairo2 \ + libgdk-pixbuf-2.0-0 \ + libffi-dev \ + shared-mime-info \ && rm -rf /var/lib/apt/lists/* # Install uv @@ -67,6 +76,9 @@ RUN mkdir -p /app/uploads # Notes: # - CPU-only PyTorch is installed by default (TORCH_CPU_ONLY=true), saving ~1.5GB # Set TORCH_CPU_ONLY=false if you need GPU support +# - NeMo ASR (WER/CER hallucination metrics) is NOT included by default (~2GB). +# It will be auto-installed on first use, or you can bake it in at build time: +# docker compose build --build-arg INSTALL_EXTRAS="qualitative-voice,reports,nemo-asr" worker # - ML models (emotion classifier, valence/arousal, UTMOS, NeMo ASR) are # downloaded lazily on first use, keeping the image smaller # - For persistent model caching across container restarts, mount volumes: From 485cc803770e1e185e98aaa92b46fe732e8f81d5 Mon Sep 17 00:00:00 2001 From: Tejas Narayan Date: Tue, 17 Mar 2026 20:09:41 +0000 Subject: [PATCH 2/3] feat: fixing voice maker and voice lib --- app/services/voice_agent/voice_bundle.py | 64 ++++++-- .../services/voicemaker/__init__.py | 3 + src/efficientai/services/voicemaker/tts.py | 148 ++++++++++++++++++ 3 files changed, 203 insertions(+), 12 deletions(-) create mode 100644 src/efficientai/services/voicemaker/tts.py diff --git a/app/services/voice_agent/voice_bundle.py b/app/services/voice_agent/voice_bundle.py index 094ec72f..378a0295 100644 --- a/app/services/voice_agent/voice_bundle.py +++ b/app/services/voice_agent/voice_bundle.py @@ -132,6 +132,9 @@ def _get_service(service_name: str): elif service_name == "SarvamTTSService": from efficientai.services.sarvam.tts import SarvamTTSService service_class = SarvamTTSService + elif service_name == "VoiceMakerTTSService": + from efficientai.services.voicemaker.tts import VoiceMakerTTSService + service_class = VoiceMakerTTSService # LLM Services elif service_name == "OpenAILLMService": @@ -270,6 +273,16 @@ def _get_tts_providers(): model=model if model else "bulbul:v3", ), }, + "voicemaker": { + "env_key": "VOICEMAKER_API_KEY", + "default_voice": "ai3-Jony", + "default_model": "neural", + "factory": lambda api_key, voice_id, model: _get_service("VoiceMakerTTSService")( + api_key=api_key, + voice_id=voice_id, + model=model if model else "neural", + ), + }, } @@ -448,6 +461,8 @@ async def run_voice_bundle_fastapi( call_start_time = time.time() s3_key_result = None duration_result = None + transcript_text = None + conversation_turns = [] # Storage for audio data from the buffer processor recorded_audio_data = {"audio": None, "sample_rate": None, "num_channels": None} @@ -643,6 +658,34 @@ async def on_client_disconnected(transport, client): # Calculate duration duration_result = time.time() - call_start_time + # Extract conversation transcript from the LLM context + try: + raw_messages = context.messages if hasattr(context, 'messages') else [] + conversation_turns = [] + transcript_parts = [] + elapsed = 0.0 + for msg in raw_messages: + role = msg.get("role", "") + content = msg.get("content", "") + if not content or role == "system": + continue + speaker = "user" if role == "user" else "assistant" + turn_duration = max(1.0, len(content.split()) * 0.4) + conversation_turns.append({ + "speaker": speaker, + "text": content, + "start": round(elapsed, 2), + "end": round(elapsed + turn_duration, 2), + }) + transcript_parts.append(f"{speaker}: {content}") + elapsed += turn_duration + transcript_text = "\n".join(transcript_parts) if transcript_parts else None + logger.info(f"Captured {len(conversation_turns)} conversation turns from live pipeline") + except Exception as ctx_err: + logger.warning(f"Failed to extract conversation context: {ctx_err}") + conversation_turns = [] + transcript_text = None + # Merge captured audio chunks total_input_audio = b''.join(input_audio_chunks) if input_audio_chunks else b'' total_output_audio = b''.join(output_audio_chunks) if output_audio_chunks else b'' @@ -700,26 +743,23 @@ async def on_client_disconnected(transport, client): "agent_id": agent_id, "persona_id": persona_id, "scenario_id": scenario_id, + "transcription": transcript_text, + "speaker_segments": conversation_turns if conversation_turns else None, "error": str(e), } - if s3_key_result: - return { - "s3_key": s3_key_result, - "duration": duration_result, - "agent_id": agent_id, - "persona_id": persona_id, - "scenario_id": scenario_id, - } - - return { - "s3_key": None, + metadata = { + "s3_key": s3_key_result, "duration": duration_result, "agent_id": agent_id, "persona_id": persona_id, "scenario_id": scenario_id, - "error": "No audio file was uploaded", + "transcription": transcript_text, + "speaker_segments": conversation_turns if conversation_turns else None, } + if not s3_key_result and not transcript_text: + metadata["error"] = "No audio file was uploaded and no transcript captured" + return metadata if __name__ == "__main__": diff --git a/src/efficientai/services/voicemaker/__init__.py b/src/efficientai/services/voicemaker/__init__.py index a7427aa9..6d823726 100644 --- a/src/efficientai/services/voicemaker/__init__.py +++ b/src/efficientai/services/voicemaker/__init__.py @@ -6,4 +6,7 @@ def __getattr__(name): if name == "synthesize_voicemaker_bytes": from .http_tts import synthesize_voicemaker_bytes return synthesize_voicemaker_bytes + if name == "VoiceMakerTTSService": + from .tts import VoiceMakerTTSService + return VoiceMakerTTSService raise AttributeError(f"module 'efficientai.services.voicemaker' has no attribute '{name}'") diff --git a/src/efficientai/services/voicemaker/tts.py b/src/efficientai/services/voicemaker/tts.py new file mode 100644 index 00000000..349c9311 --- /dev/null +++ b/src/efficientai/services/voicemaker/tts.py @@ -0,0 +1,148 @@ +# +# VoiceMaker TTS service for Pipecat pipeline integration. +# + +from typing import AsyncGenerator, Optional + +import aiohttp +from loguru import logger +from pydantic import BaseModel + +from efficientai.frames.frames import ( + ErrorFrame, + Frame, + TTSAudioRawFrame, + TTSStartedFrame, + TTSStoppedFrame, +) +from efficientai.services.tts_service import TTSService +from efficientai.utils.tracing.service_decorators import traced_tts + +from .http_tts import _infer_language_code + + +class VoiceMakerTTSService(TTSService): + """VoiceMaker TTS service for real-time voice pipeline use. + + Downloads audio from VoiceMaker's REST API and yields PCM frames + compatible with the Pipecat pipeline. + """ + + class InputParams(BaseModel): + output_format: str = "wav" + sample_rate_hz: int = 24000 + + def __init__( + self, + *, + api_key: str, + voice_id: str = "ai3-Jony", + model: str = "neural", + sample_rate: int = 24000, + params: Optional[InputParams] = None, + **kwargs, + ): + super().__init__(sample_rate=sample_rate, **kwargs) + self._api_key = api_key + self._voice_id = voice_id + self._model = model + self._params = params or VoiceMakerTTSService.InputParams(sample_rate_hz=sample_rate) + self._session: Optional[aiohttp.ClientSession] = None + + def can_generate_metrics(self) -> bool: + return True + + async def _get_session(self) -> aiohttp.ClientSession: + if self._session is None or self._session.closed: + self._session = aiohttp.ClientSession() + return self._session + + async def stop(self, frame: Frame): + await super().stop(frame) + if self._session: + await self._session.close() + self._session = None + + async def cancel(self, frame: Frame): + await super().cancel(frame) + if self._session: + await self._session.close() + self._session = None + + @traced_tts + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + logger.debug(f"Generating VoiceMaker TTS: [{text}]") + + language_code = _infer_language_code(self._voice_id) + + payload = { + "VoiceId": self._voice_id, + "Text": text, + "LanguageCode": language_code, + "OutputFormat": "wav", + "SampleRate": str(self.sample_rate), + "ResponseType": "file", + } + if self._model: + engine = self._model + if engine.lower().startswith("voicemaker-"): + engine = engine[len("voicemaker-"):] + payload["Engine"] = engine + + headers = { + "Authorization": f"Bearer {self._api_key}", + "Content-Type": "application/json", + } + + try: + session = await self._get_session() + await self.start_ttfb_metrics() + + async with session.post( + "https://developer.voicemaker.in/api/v1/voice/convert", + json=payload, + headers=headers, + ) as response: + if response.status != 200: + error_text = await response.text() + logger.error(f"VoiceMaker TTS error: {response.status} - {error_text[:300]}") + yield ErrorFrame(error=f"VoiceMaker TTS error: {error_text[:300]}") + return + + data = await response.json() + audio_url = data.get("path") + if not audio_url: + yield ErrorFrame(error="VoiceMaker returned no audio path") + return + + await self.start_tts_usage_metrics(text) + yield TTSStartedFrame() + await self.stop_ttfb_metrics() + + async with session.get(audio_url) as audio_resp: + if audio_resp.status != 200: + yield ErrorFrame(error=f"VoiceMaker audio download failed: {audio_resp.status}") + return + + audio_bytes = await audio_resp.read() + + # Strip WAV header (44 bytes) to get raw PCM + pcm_data = audio_bytes[44:] if len(audio_bytes) > 44 else audio_bytes + + # Yield in chunks for smooth pipeline flow + chunk_size = 4096 + for i in range(0, len(pcm_data), chunk_size): + chunk = pcm_data[i : i + chunk_size] + if chunk: + yield TTSAudioRawFrame( + audio=chunk, + sample_rate=self.sample_rate, + num_channels=1, + ) + + yield TTSStoppedFrame() + + except Exception as e: + logger.error(f"VoiceMaker TTS exception: {e}") + yield ErrorFrame(error=f"VoiceMaker TTS exception: {str(e)}") + yield TTSStoppedFrame() From 61201b527e264020d97f185fbdc04b76eb3b9ebf Mon Sep 17 00:00:00 2001 From: Tejas Narayan Date: Tue, 17 Mar 2026 21:21:00 +0000 Subject: [PATCH 3/3] feat: updating migrations start --- app/cli.py | 16 ++++++++++------ app/database.py | 5 +++++ .../001_add_default_agent_to_org_member.py | 9 +++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/app/cli.py b/app/cli.py index 1f71f3a8..538a264c 100644 --- a/app/cli.py +++ b/app/cli.py @@ -265,14 +265,16 @@ def start(config: str, host: Optional[str], port: Optional[int], build_frontend: click.echo("❌ npm not found. Please install Node.js and npm.", err=True) sys.exit(1) - # Run migrations before starting (unless explicitly skipped) + # Initialize DB tables and run migrations before starting (unless explicitly skipped) if not skip_migrations: - click.echo("🔄 Running database migrations before startup...") + click.echo("🔄 Initializing database and running migrations...") + from app.database import init_db from app.core.migrations import run_migrations, ensure_migrations_directory try: + init_db() ensure_migrations_directory() run_migrations() - click.echo("✅ Migrations completed successfully") + click.echo("✅ Database initialized and migrations completed") except Exception as e: click.echo(f"❌ Migration failed: {e}", err=True) click.echo("💡 You can skip migrations with --skip-migrations (not recommended)", err=True) @@ -472,14 +474,16 @@ def signal_handler(sig, frame): signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) - # Run migrations before starting (unless explicitly skipped) + # Initialize DB tables and run migrations before starting (unless explicitly skipped) if not skip_migrations: - click.echo("🔄 Running database migrations before startup...") + click.echo("🔄 Initializing database and running migrations...") + from app.database import init_db from app.core.migrations import run_migrations, ensure_migrations_directory try: + init_db() ensure_migrations_directory() run_migrations() - click.echo("✅ Migrations completed successfully") + click.echo("✅ Database initialized and migrations completed") except Exception as e: click.echo(f"❌ Migration failed: {e}", err=True) click.echo("💡 You can skip migrations with --skip-migrations (not recommended)", err=True) diff --git a/app/database.py b/app/database.py index af8fdae0..e990d212 100644 --- a/app/database.py +++ b/app/database.py @@ -37,6 +37,11 @@ def get_db(): def init_db(): """Initialize database by creating all tables and run column migrations.""" + # Import all models so SQLAlchemy registers them with Base.metadata + # before create_all is called. Without this, tables won't be created + # on a fresh database. + import app.models.database # noqa: F401 + Base.metadata.create_all(bind=engine) _run_column_migrations() diff --git a/app/migrations/001_add_default_agent_to_org_member.py b/app/migrations/001_add_default_agent_to_org_member.py index 49a562dd..face5050 100644 --- a/app/migrations/001_add_default_agent_to_org_member.py +++ b/app/migrations/001_add_default_agent_to_org_member.py @@ -14,6 +14,15 @@ def upgrade(db: Session): """Add default_agent_id column with foreign key to agents table.""" + # Check if the table exists at all (on fresh DBs, create_all handles this) + table_check = db.execute(text(""" + SELECT 1 FROM information_schema.tables + WHERE table_name = 'organization_members' + """)) + if table_check.fetchone() is None: + print("Table organization_members does not exist yet (fresh DB, handled by create_all), skipping...") + return + # Check if column already exists result = db.execute(text(""" SELECT column_name