diff --git a/.gitignore b/.gitignore index df987a07..a8f8d4f2 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ newenv/ env/ ENV/ .venv +.pptx_venv config.yml # uv .python-version diff --git a/app/api/v1/api.py b/app/api/v1/api.py index d6dc6d9d..bbf422e0 100644 --- a/app/api/v1/api.py +++ b/app/api/v1/api.py @@ -41,10 +41,12 @@ call_import_tags, call_import_evaluations, judge_alignment, + metric_studio, workspaces, workspace_iam, dashboard, llm_gateway, + platform_admin, ) api_router = APIRouter() @@ -89,7 +91,9 @@ api_router.include_router(call_import_tags.router) api_router.include_router(call_import_evaluations.router) api_router.include_router(judge_alignment.router) +api_router.include_router(metric_studio.router) api_router.include_router(workspaces.router) api_router.include_router(workspace_iam.router) api_router.include_router(dashboard.router) api_router.include_router(llm_gateway.router) +api_router.include_router(platform_admin.router) diff --git a/app/api/v1/routes/agents.py b/app/api/v1/routes/agents.py index 9d1d77b0..0be8cd99 100644 --- a/app/api/v1/routes/agents.py +++ b/app/api/v1/routes/agents.py @@ -1,1025 +1,1025 @@ -""" -Agents API Routes -Complete CRUD operations for test agents -""" -from fastapi import APIRouter, Depends, HTTPException, status, Query -from fastapi.responses import JSONResponse, Response -from sqlalchemy.orm import Session -from typing import List, Optional -from uuid import UUID -import random -from pydantic import BaseModel -from loguru import logger - -from app.dependencies import get_db, get_organization_id, get_workspace_id, get_api_key -from app.models.database import ( - Agent, ConversationEvaluation, TestAgentConversation, VoiceBundle, - AIProvider, Integration, IntegrationPlatform, CallMediumEnum, - Evaluator, EvaluatorResult, CallRecording, Scenario, -) -from sqlalchemy import and_ -from app.models.schemas import ( - AgentCreate, - AgentUpdate, - AgentResponse, - AgentPhoneAssignmentCheckResponse, - AgentPhoneAssignmentConflict, - CallMediumEnum as CallMediumEnumSchema, - GenerateTestPromptRequest, - GenerateTestPromptResponse, - GenerateScenariosFromPromptRequest, - GenerateScenariosFromPromptResponse, - GenerateTestSetupRequest, - GenerateTestSetupResponse, - GeneratedScenarioDraftResponse, - TestPromptSectionResponse, -) - -router = APIRouter(prefix="/agents", tags=["agents"]) - - -def _validate_agent_phone_assignment( - db: Session, - *, - organization_id: UUID, - call_medium, - phone_number: Optional[str], - telephony_phone_number_id: Optional[UUID], - exclude_agent_id: Optional[UUID] = None, -) -> None: - """Raise HTTPException if phone assignment conflicts with another agent.""" - if call_medium != CallMediumEnum.PHONE_CALL: - return - if not phone_number and not telephony_phone_number_id: - return - - from app.services.telephony.phone_routing import find_agent_phone_assignment_conflict - - conflict = find_agent_phone_assignment_conflict( - db, - organization_id=organization_id, - phone_number=phone_number, - telephony_phone_number_id=telephony_phone_number_id, - exclude_agent_id=exclude_agent_id, - ) - if not conflict: - return - if conflict.get("error") == "telephony_not_found": - raise HTTPException(status_code=404, detail="Telephony phone number not found") - - agent_name = conflict["agent_name"] - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail={ - "message": f'This number is already assigned to agent "{agent_name}".', - "agent_id": str(conflict["agent_id"]), - "agent_name": agent_name, - "phone_number": conflict["phone_number"], - }, - ) - - -def resolve_agent_by_path_id( - db: Session, - *, - organization_id: UUID, - workspace_id: UUID, - agent_id: str, -) -> Agent: - """Resolve agent by UUID primary key or 6-digit agent_id (same as GET /agents/{id}).""" - try: - agent_uuid = UUID(agent_id) - agent = db.query(Agent).filter( - and_( - Agent.id == agent_uuid, - Agent.organization_id == organization_id, - Agent.workspace_id == workspace_id, - ) - ).first() - except ValueError: - agent = db.query(Agent).filter( - and_( - Agent.agent_id == agent_id, - Agent.organization_id == organization_id, - Agent.workspace_id == workspace_id, - ) - ).first() - if not agent: - raise HTTPException(status_code=404, detail=f"Agent {agent_id} not found") - return agent - - -# ====================================================================== -# AI Generation for agent descriptions -# ====================================================================== - -class GenerateAgentDescriptionRequest(BaseModel): - description: str - tone: Optional[str] = "professional" - format_style: Optional[str] = "structured" - provider: Optional[str] = None - model: Optional[str] = None - agent_id: Optional[UUID] = None - include_linked_scenarios: bool = True - append_scenarios_to_output: bool = False - - -GENERATE_AGENT_DESCRIPTION_SYSTEM = ( - "You are an expert at writing clear, well-structured descriptions for voice AI test agents. " - "The user will describe what they need the agent to do, and you will generate a comprehensive, " - "well-formatted agent description in markdown.\n\n" - "Guidelines:\n" - "- Use clear markdown structure: headings, bullet points, numbered lists\n" - "- Include sections for: Purpose, Behavior, Expected Interactions, Personality Traits, and Constraints\n" - "- Be specific about the agent's role, tone of voice, and how it should handle conversations\n" - "- Include example scenarios or edge cases where helpful\n" - "- Return ONLY the description in markdown, no preamble or explanation about what you did" -) - - -from app.services.ai.llm_resolver import get_llm_provider_and_model as _get_llm_provider_and_model - - -@router.post("/generate-description") -async def generate_agent_description( - data: GenerateAgentDescriptionRequest, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - """Generate an agent description using AI from a brief description.""" - from app.services.ai.llm_service import llm_service - from app.services.testing.test_agent_simulation_prompt import ( - format_scenarios_for_generation_context, - format_scenarios_reference_appendix, - load_linked_scenarios_for_agent, - merge_generated_description_with_scenario_appendix, - ) - - if not data.description.strip(): - raise HTTPException(400, "Description is required") - - provider_enum, model_str = _get_llm_provider_and_model( - organization_id, db, data.provider, data.model - ) - - linked_scenarios = [] - if data.agent_id and data.include_linked_scenarios: - agent = db.query(Agent).filter( - Agent.id == data.agent_id, - Agent.organization_id == organization_id, - Agent.workspace_id == workspace_id, - ).first() - if not agent: - raise HTTPException(status_code=404, detail=f"Agent {data.agent_id} not found") - linked_scenarios = load_linked_scenarios_for_agent( - db, - organization_id=organization_id, - workspace_id=workspace_id, - agent_id=agent.id, - ) - - user_prompt_parts = [ - "Create a detailed agent description for the following:", - "", - f"Description: {data.description}", - f"Tone: {data.tone or 'professional'}", - f"Format: {data.format_style or 'structured'}", - ] - scenario_context = format_scenarios_for_generation_context(linked_scenarios) - if scenario_context: - user_prompt_parts.extend(["", scenario_context]) - user_prompt_parts.extend([ - "", - "Generate a comprehensive, well-formatted agent description in markdown.", - ]) - user_prompt = "\n".join(user_prompt_parts) - - messages = [ - {"role": "system", "content": GENERATE_AGENT_DESCRIPTION_SYSTEM}, - {"role": "user", "content": user_prompt}, - ] - - try: - result = llm_service.generate_response( - messages=messages, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - temperature=0.7, - max_tokens=4000, - ) - content = result["text"] - if data.append_scenarios_to_output and linked_scenarios: - appendix = format_scenarios_reference_appendix(linked_scenarios) - content = merge_generated_description_with_scenario_appendix(content, appendix) - return {"content": content, "provider": provider_enum.value, "model": model_str} - except Exception as e: - logger.error(f"[Agents] AI description generation failed: {repr(e)}") - raise HTTPException(500, f"AI generation failed: {str(e)}") - - -def _test_prompt_section_responses(sections) -> list[TestPromptSectionResponse]: - return [ - TestPromptSectionResponse(key=s.key, title=s.title, content=s.content) - for s in sections - ] - - -def _scenario_draft_responses(scenarios) -> list[GeneratedScenarioDraftResponse]: - return [ - GeneratedScenarioDraftResponse(name=s.name, description=s.description, goal=s.goal) - for s in scenarios - ] - - -@router.post("/generate-test-prompt", response_model=GenerateTestPromptResponse) -async def generate_test_prompt( - data: GenerateTestPromptRequest, - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - """Stage 1: generate foundational test agent prompt from production prompt.""" - from app.services.testing.agent_test_setup_generation import ( - generate_test_prompt_from_production, - ) - - if not data.production_prompt.strip(): - raise HTTPException(400, "Production prompt is required") - - provider_enum, model_str = _get_llm_provider_and_model( - organization_id, db, data.provider, data.model, data.credential_id - ) - - try: - result = generate_test_prompt_from_production( - data.production_prompt, - agent_name=data.agent_name, - language=data.language, - call_type=data.call_type, - additional_context=data.additional_context, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - llm_config=data.llm_config, - credential_id=data.credential_id, - ) - return GenerateTestPromptResponse( - sections=_test_prompt_section_responses(result.sections), - test_agent_prompt=result.test_agent_prompt, - provider=result.provider, - model=result.model, - ) - except ValueError as e: - raise HTTPException(400, str(e)) from e - except Exception as e: - logger.error(f"[Agents] Test prompt generation failed: {repr(e)}") - raise HTTPException(500, f"AI generation failed: {str(e)}") from e - - -@router.post("/generate-scenarios-from-prompt", response_model=GenerateScenariosFromPromptResponse) -async def generate_scenarios_from_prompt( - data: GenerateScenariosFromPromptRequest, - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - """Stage 2: generate scenario drafts from a test agent prompt.""" - from app.services.testing.agent_test_setup_generation import ( - generate_scenarios_from_test_prompt, - ) - - if not data.test_agent_prompt.strip(): - raise HTTPException(400, "Test agent prompt is required") - - provider_enum, model_str = _get_llm_provider_and_model( - organization_id, db, data.provider, data.model, data.credential_id - ) - - try: - result = generate_scenarios_from_test_prompt( - data.test_agent_prompt, - agent_name=data.agent_name, - scenario_count=data.scenario_count, - language=data.language, - call_type=data.call_type, - additional_context=data.additional_context, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - llm_config=data.llm_config, - credential_id=data.credential_id, - ) - return GenerateScenariosFromPromptResponse( - scenarios=_scenario_draft_responses(result.scenarios), - provider=result.provider, - model=result.model, - ) - except ValueError as e: - raise HTTPException(400, str(e)) from e - except Exception as e: - logger.error(f"[Agents] Scenario generation failed: {repr(e)}") - raise HTTPException(500, f"AI generation failed: {str(e)}") from e - - -@router.post("/generate-test-setup", response_model=GenerateTestSetupResponse) -async def generate_test_setup( - data: GenerateTestSetupRequest, - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - """Run stage 1 then stage 2: foundational test prompt + scenario drafts.""" - from app.services.testing.agent_test_setup_generation import ( - generate_scenarios_from_test_prompt, - generate_test_prompt_from_production, - ) - - if not data.production_prompt.strip(): - raise HTTPException(400, "Production prompt is required") - - provider_enum, model_str = _get_llm_provider_and_model( - organization_id, db, data.provider, data.model, data.credential_id - ) - - try: - prompt_result = generate_test_prompt_from_production( - data.production_prompt, - agent_name=data.agent_name, - language=data.language, - call_type=data.call_type, - additional_context=data.additional_context, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - llm_config=data.llm_config, - credential_id=data.credential_id, - ) - scenario_result = generate_scenarios_from_test_prompt( - prompt_result.test_agent_prompt, - agent_name=data.agent_name, - scenario_count=data.scenario_count, - language=data.language, - call_type=data.call_type, - additional_context=data.additional_context, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - llm_config=data.llm_config, - credential_id=data.credential_id, - ) - return GenerateTestSetupResponse( - sections=_test_prompt_section_responses(prompt_result.sections), - test_agent_prompt=prompt_result.test_agent_prompt, - scenarios=_scenario_draft_responses(scenario_result.scenarios), - provider=prompt_result.provider, - model=prompt_result.model, - ) - except ValueError as e: - raise HTTPException(400, str(e)) from e - except Exception as e: - logger.error(f"[Agents] Test setup generation failed: {repr(e)}") - raise HTTPException(500, f"AI generation failed: {str(e)}") from e - - -def generate_unique_agent_id(db: Session) -> str: - """Generate a unique 6-digit agent ID.""" - max_attempts = 100 - for _ in range(max_attempts): - agent_id = f"{random.randint(100000, 999999)}" - existing = db.query(Agent).filter(Agent.agent_id == agent_id).first() - if not existing: - return agent_id - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Failed to generate unique agent ID" - ) - - -def get_agent_dependencies(db: Session, organization_id: UUID, agent_uuid: UUID) -> dict: - """Return dependency counts that block non-force delete.""" - evaluators_count = db.query(Evaluator).filter( - Evaluator.agent_id == agent_uuid, - Evaluator.organization_id == organization_id, - ).count() - - evaluator_results_count = db.query(EvaluatorResult).filter( - EvaluatorResult.agent_id == agent_uuid, - EvaluatorResult.organization_id == organization_id, - ).count() - - call_recordings_count = db.query(CallRecording).filter( - CallRecording.agent_id == agent_uuid, - CallRecording.organization_id == organization_id, - ).count() - - conversation_evaluations_count = db.query(ConversationEvaluation).filter( - ConversationEvaluation.agent_id == agent_uuid, - ConversationEvaluation.organization_id == organization_id, - ).count() - - test_conversations_count = db.query(TestAgentConversation).filter( - TestAgentConversation.agent_id == agent_uuid, - TestAgentConversation.organization_id == organization_id, - ).count() - - dependencies = {} - if evaluators_count > 0: - dependencies["evaluators"] = evaluators_count - if evaluator_results_count > 0: - dependencies["evaluator_results"] = evaluator_results_count - if call_recordings_count > 0: - dependencies["call_recordings"] = call_recordings_count - if conversation_evaluations_count > 0: - dependencies["conversation_evaluations"] = conversation_evaluations_count - if test_conversations_count > 0: - dependencies["test_conversations"] = test_conversations_count - - return dependencies - - -@router.post("", response_model=AgentResponse, status_code=status.HTTP_201_CREATED) -async def create_agent( - agent: AgentCreate, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db) -): - """Create a new test agent. - - The agent is stamped with the active workspace from the - ``X-Workspace-Id`` header (falling back to the org's Default). - """ - # Validate phone_number is provided when call_medium is phone_call - if agent.call_medium == CallMediumEnumSchema.PHONE_CALL and not agent.phone_number: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="phone_number is required when call_medium is phone_call" - ) - - # Validate voice_bundle_id exists, is active, and belongs to organization - voice_bundle = db.query(VoiceBundle).filter( - and_( - VoiceBundle.id == agent.voice_bundle_id, - VoiceBundle.organization_id == organization_id, - VoiceBundle.is_active == True, - ) - ).first() - if not voice_bundle: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Active voice bundle not found", - ) - - # Validate voice_ai_integration_id exists and belongs to organization - if agent.voice_ai_integration_id: - integration = db.query(Integration).filter( - and_( - Integration.id == agent.voice_ai_integration_id, - Integration.organization_id == organization_id, - Integration.is_active == True - ) - ).first() - if not integration: - raise HTTPException(status_code=404, detail="Integration not found or inactive") - - if integration.platform not in [ - IntegrationPlatform.RETELL, - IntegrationPlatform.VAPI, - IntegrationPlatform.ELEVENLABS, - IntegrationPlatform.SMALLEST, - ]: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Integration platform {integration.platform.value} is not supported for Voice AI agents. " - "Only Retell, Vapi, ElevenLabs, and Smallest are supported." - ) - ) - - if not agent.voice_ai_agent_id: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="voice_ai_agent_id is required when voice_ai_integration_id is provided" - ) - - _validate_agent_phone_assignment( - db, - organization_id=organization_id, - call_medium=agent.call_medium, - phone_number=agent.phone_number, - telephony_phone_number_id=agent.telephony_phone_number_id, - ) - - # Generate unique 6-digit agent_id - agent_id = generate_unique_agent_id(db) - - db_agent = Agent( - agent_id=agent_id, - organization_id=organization_id, - workspace_id=workspace_id, - name=agent.name, - phone_number=agent.phone_number, - language=agent.language, - description=agent.description, - call_type=agent.call_type, - call_medium=agent.call_medium, - telephony_phone_number_id=agent.telephony_phone_number_id, - voice_bundle_id=agent.voice_bundle_id, - ai_provider_id=agent.ai_provider_id, - voice_ai_integration_id=agent.voice_ai_integration_id, - voice_ai_agent_id=agent.voice_ai_agent_id, - provider_prompt=agent.provider_prompt, - silence_hangup_secs=agent.silence_hangup_secs, - ) - db.add(db_agent) - db.commit() - db.refresh(db_agent) - - from app.services.telephony.phone_routing import sync_agent_telephony_number_link - - sync_agent_telephony_number_link(db, db_agent) - db.refresh(db_agent) - - has_provider_prompt = isinstance(agent.provider_prompt, str) and bool(agent.provider_prompt.strip()) - if agent.voice_ai_integration_id and agent.voice_ai_agent_id and not has_provider_prompt: - try: - from app.services.voice_providers.prompt_sync import sync_provider_prompt - integration = db.query(Integration).filter(Integration.id == agent.voice_ai_integration_id).first() - if integration: - sync_provider_prompt(db_agent, integration, db) - db.refresh(db_agent) - except Exception as e: - logger.warning(f"[Agents] Best-effort provider prompt sync failed on create: {e}") - - return db_agent - - -@router.get("", response_model=List[AgentResponse]) -async def list_agents( - skip: int = 0, - limit: int = 100, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db) -): - """Get list of all agents for the active workspace. - - Scoped to (organization_id, workspace_id) so users only see agents - in the workspace they're currently in. - """ - agents = db.query(Agent).filter( - Agent.organization_id == organization_id, - Agent.workspace_id == workspace_id, - ).offset(skip).limit(limit).all() - return agents - - -@router.get("/check-phone-assignment", response_model=AgentPhoneAssignmentCheckResponse) -async def check_phone_assignment( - phone_number: Optional[str] = Query(None), - telephony_phone_number_id: Optional[UUID] = Query(None), - exclude_agent_id: Optional[UUID] = Query(None), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -): - """Check whether a phone number is available for agent assignment in this org.""" - if not phone_number and not telephony_phone_number_id: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="phone_number or telephony_phone_number_id is required", - ) - - from app.services.telephony.phone_routing import find_agent_phone_assignment_conflict - - conflict = find_agent_phone_assignment_conflict( - db, - organization_id=organization_id, - phone_number=phone_number, - telephony_phone_number_id=telephony_phone_number_id, - exclude_agent_id=exclude_agent_id, - ) - if conflict and conflict.get("error") == "telephony_not_found": - raise HTTPException(status_code=404, detail="Telephony phone number not found") - if conflict: - return AgentPhoneAssignmentCheckResponse( - available=False, - phone_number=conflict["phone_number"], - conflict=AgentPhoneAssignmentConflict(**conflict), - ) - - resolved_phone = phone_number - if telephony_phone_number_id: - from app.models.database import TelephonyPhoneNumber - - row = ( - db.query(TelephonyPhoneNumber) - .filter( - TelephonyPhoneNumber.id == telephony_phone_number_id, - TelephonyPhoneNumber.organization_id == organization_id, - ) - .first() - ) - if row: - resolved_phone = row.phone_number - - return AgentPhoneAssignmentCheckResponse( - available=True, - phone_number=resolved_phone, - ) - - -@router.get("/{agent_id}", response_model=AgentResponse) -async def get_agent( - agent_id: str, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db) -): - """Get a specific agent by ID (UUID) or agent_id (6-digit) within the active workspace.""" - try: - # Try as UUID first - agent_uuid = UUID(agent_id) - agent = db.query(Agent).filter( - and_( - Agent.id == agent_uuid, - Agent.organization_id == organization_id, - Agent.workspace_id == workspace_id, - ) - ).first() - except ValueError: - # Try as 6-digit ID - agent = db.query(Agent).filter( - and_( - Agent.agent_id == agent_id, - Agent.organization_id == organization_id, - Agent.workspace_id == workspace_id, - ) - ).first() - - if not agent: - raise HTTPException(status_code=404, detail=f"Agent {agent_id} not found") - return agent - - -@router.put("/{agent_id}", response_model=AgentResponse) -async def update_agent( - agent_id: str, - agent_update: AgentUpdate, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db) -): - """Update an existing agent by ID (UUID) or agent_id (6-digit) within the active workspace.""" - try: - # Try as UUID first - agent_uuid = UUID(agent_id) - db_agent = db.query(Agent).filter( - and_( - Agent.id == agent_uuid, - Agent.organization_id == organization_id, - Agent.workspace_id == workspace_id, - ) - ).first() - except ValueError: - # Try as 6-digit ID - db_agent = db.query(Agent).filter( - and_( - Agent.agent_id == agent_id, - Agent.organization_id == organization_id, - Agent.workspace_id == workspace_id, - ) - ).first() - - if not db_agent: - raise HTTPException(status_code=404, detail=f"Agent {agent_id} not found") - - # Determine the call_medium to validate - call_medium = agent_update.call_medium if agent_update.call_medium is not None else db_agent.call_medium - - # Validate phone_number is provided when call_medium is phone_call - if call_medium == CallMediumEnum.PHONE_CALL: - phone_number = agent_update.phone_number if agent_update.phone_number is not None else db_agent.phone_number - if not phone_number: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="phone_number is required when call_medium is phone_call" - ) - - # Validate voice_bundle_id if provided - if agent_update.voice_bundle_id: - voice_bundle = db.query(VoiceBundle).filter( - and_( - VoiceBundle.id == agent_update.voice_bundle_id, - VoiceBundle.organization_id == organization_id - ) - ).first() - if not voice_bundle: - raise HTTPException(status_code=404, detail="Voice bundle not found") - - # Validate voice_ai_integration_id if provided - if agent_update.voice_ai_integration_id: - integration = db.query(Integration).filter( - and_( - Integration.id == agent_update.voice_ai_integration_id, - Integration.organization_id == organization_id, - Integration.is_active == True - ) - ).first() - if not integration: - raise HTTPException(status_code=404, detail="Integration not found or inactive") - - if integration.platform not in [ - IntegrationPlatform.RETELL, - IntegrationPlatform.VAPI, - IntegrationPlatform.ELEVENLABS, - IntegrationPlatform.SMALLEST, - ]: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Integration platform {integration.platform.value} is not supported for Voice AI agents. " - "Only Retell, Vapi, ElevenLabs, and Smallest are supported." - ) - ) - - if not agent_update.voice_ai_agent_id: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="voice_ai_agent_id is required when voice_ai_integration_id is provided" - ) - - update_data = agent_update.model_dump(exclude_unset=True, exclude_none=False) - - effective_call_medium = ( - agent_update.call_medium if agent_update.call_medium is not None else db_agent.call_medium - ) - effective_phone_number = ( - agent_update.phone_number - if "phone_number" in update_data - else db_agent.phone_number - ) - effective_telephony_id = ( - agent_update.telephony_phone_number_id - if "telephony_phone_number_id" in update_data - else db_agent.telephony_phone_number_id - ) - - _validate_agent_phone_assignment( - db, - organization_id=organization_id, - call_medium=effective_call_medium, - phone_number=effective_phone_number, - telephony_phone_number_id=effective_telephony_id, - exclude_agent_id=db_agent.id, - ) - - # Convert the update model to dict, handling None values properly - # Use model_dump with exclude_unset to only get fields that were explicitly provided - - # Apply updates - for field, value in update_data.items(): - setattr(db_agent, field, value) - - db.commit() - db.refresh(db_agent) - - from app.services.telephony.phone_routing import sync_agent_telephony_number_link - - if "telephony_phone_number_id" in update_data or "phone_number" in update_data: - sync_agent_telephony_number_link(db, db_agent) - db.refresh(db_agent) - - if "voice_ai_agent_id" in update_data or "voice_ai_integration_id" in update_data: - integration_id = db_agent.voice_ai_integration_id - provider_prompt_updated = "provider_prompt" in update_data - if integration_id and db_agent.voice_ai_agent_id and not provider_prompt_updated: - try: - from app.services.voice_providers.prompt_sync import sync_provider_prompt - integration = db.query(Integration).filter(Integration.id == integration_id).first() - if integration: - sync_provider_prompt(db_agent, integration, db) - db.refresh(db_agent) - except Exception as e: - logger.warning(f"[Agents] Best-effort provider prompt sync failed on update: {e}") - - return db_agent - - -@router.post("/{agent_id}/sync-provider-prompt") -async def sync_agent_provider_prompt( - agent_id: str, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - """Fetch and store the current system prompt from the voice provider.""" - try: - agent_uuid = UUID(agent_id) - db_agent = db.query(Agent).filter( - and_( - Agent.id == agent_uuid, - Agent.organization_id == organization_id, - Agent.workspace_id == workspace_id, - ) - ).first() - except ValueError: - db_agent = db.query(Agent).filter( - and_( - Agent.agent_id == agent_id, - Agent.organization_id == organization_id, - Agent.workspace_id == workspace_id, - ) - ).first() - - if not db_agent: - raise HTTPException(status_code=404, detail=f"Agent {agent_id} not found") - - if not db_agent.voice_ai_integration_id or not db_agent.voice_ai_agent_id: - raise HTTPException( - status_code=400, - detail="Agent is not linked to an external voice provider", - ) - - integration = db.query(Integration).filter( - and_( - Integration.id == db_agent.voice_ai_integration_id, - Integration.organization_id == organization_id, - Integration.is_active == True, - ) - ).first() - if not integration: - raise HTTPException(status_code=404, detail="Integration not found or inactive") - - try: - from app.services.voice_providers.prompt_sync import sync_provider_prompt - prompt = sync_provider_prompt(db_agent, integration, db) - except Exception as e: - raise HTTPException(status_code=502, detail=f"Failed to fetch prompt from provider: {str(e)}") - - db.refresh(db_agent) - synced = isinstance(prompt, str) and bool(prompt.strip()) - if not synced: - raise HTTPException( - status_code=422, - detail="Provider returned no prompt. Verify the external agent has a system prompt configured.", - ) - return { - "synced": synced, - "provider_prompt": db_agent.provider_prompt, - "provider_prompt_synced_at": db_agent.provider_prompt_synced_at.isoformat() if db_agent.provider_prompt_synced_at else None, - } - - -@router.delete("/{agent_id}") -async def delete_agent( - agent_id: str, - force: bool = Query(False, description="Force delete with all dependent records"), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db) -): - """Delete an agent (scoped to the active workspace). Returns 409 if dependent records exist unless force=true.""" - try: - agent_uuid = UUID(agent_id) - db_agent = db.query(Agent).filter( - and_( - Agent.id == agent_uuid, - Agent.organization_id == organization_id, - Agent.workspace_id == workspace_id, - ) - ).first() - except ValueError: - db_agent = db.query(Agent).filter( - and_( - Agent.agent_id == agent_id, - Agent.organization_id == organization_id, - Agent.workspace_id == workspace_id, - ) - ).first() - - if not db_agent: - raise HTTPException(status_code=404, detail=f"Agent {agent_id} not found") - - agent_uuid = db_agent.id - dependencies = get_agent_dependencies(db, organization_id, agent_uuid) - - if dependencies and not force: - parts = [] - if dependencies.get("evaluators"): - parts.append(f"{dependencies['evaluators']} evaluator(s)") - if dependencies.get("evaluator_results"): - parts.append(f"{dependencies['evaluator_results']} evaluator result(s)") - if dependencies.get("call_recordings"): - parts.append(f"{dependencies['call_recordings']} call recording(s)") - if dependencies.get("conversation_evaluations"): - parts.append(f"{dependencies['conversation_evaluations']} conversation evaluation(s)") - if dependencies.get("test_conversations"): - parts.append(f"{dependencies['test_conversations']} test conversation(s)") - - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail={ - "message": f"Cannot delete agent. It is referenced by: {', '.join(parts)}.", - "dependencies": dependencies, - "hint": "Use force=true to delete this agent and all its dependent records.", - }, - ) - - if dependencies: - # Delete in FK-safe order: - # 1. EvaluatorResults (references evaluators and agents) - db.query(EvaluatorResult).filter( - EvaluatorResult.agent_id == agent_uuid, - ).delete(synchronize_session=False) - - # 2. Evaluators (references agents) - db.query(Evaluator).filter( - Evaluator.agent_id == agent_uuid, - ).delete(synchronize_session=False) - - # 3. Nullify call recordings (keep recordings, unlink agent) - db.query(CallRecording).filter( - CallRecording.agent_id == agent_uuid, - ).update({CallRecording.agent_id: None}, synchronize_session=False) - - # 4. ConversationEvaluations - db.query(ConversationEvaluation).filter( - ConversationEvaluation.agent_id == agent_uuid, - ).delete(synchronize_session=False) - - # 5. TestAgentConversations - db.query(TestAgentConversation).filter( - TestAgentConversation.agent_id == agent_uuid, - ).delete(synchronize_session=False) - - db.delete(db_agent) - db.commit() - - if dependencies: - return JSONResponse( - status_code=200, - content={ - "message": "Agent and all dependent records deleted successfully.", - "deleted": dependencies, - }, - ) - - return Response(status_code=204) - - -@router.get("/{agent_id}/delete-impact") -async def get_agent_delete_impact( - agent_id: str, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db) -): - """Preview dependent records that would be affected by force delete (scoped to the active workspace).""" - try: - agent_uuid = UUID(agent_id) - db_agent = db.query(Agent).filter( - and_( - Agent.id == agent_uuid, - Agent.organization_id == organization_id, - Agent.workspace_id == workspace_id, - ) - ).first() - except ValueError: - db_agent = db.query(Agent).filter( - and_( - Agent.agent_id == agent_id, - Agent.organization_id == organization_id, - Agent.workspace_id == workspace_id, - ) - ).first() - - if not db_agent: - raise HTTPException(status_code=404, detail=f"Agent {agent_id} not found") - - dependencies = get_agent_dependencies(db, organization_id, db_agent.id) - return { - "agent_id": str(db_agent.id), - "agent_name": db_agent.name, - "dependencies": dependencies, - "can_delete_without_force": len(dependencies) == 0, - } - - -from app.core.auth.capabilities import SIM_MANAGE, SIM_VIEW -from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities - -apply_workspace_route_capabilities( - router, - view_capability=SIM_VIEW, - manage_capability=SIM_MANAGE, -) - +""" +Agents API Routes +Complete CRUD operations for test agents +""" +from fastapi import APIRouter, Depends, HTTPException, status, Query +from fastapi.responses import JSONResponse, Response +from sqlalchemy.orm import Session +from typing import List, Optional +from uuid import UUID +import random +from pydantic import BaseModel +from loguru import logger + +from app.dependencies import get_db, get_organization_id, get_workspace_id, get_api_key +from app.models.database import ( + Agent, ConversationEvaluation, TestAgentConversation, VoiceBundle, + AIProvider, Integration, IntegrationPlatform, CallMediumEnum, + Evaluator, EvaluatorResult, CallRecording, Scenario, +) +from sqlalchemy import and_ +from app.models.schemas import ( + AgentCreate, + AgentUpdate, + AgentResponse, + AgentPhoneAssignmentCheckResponse, + AgentPhoneAssignmentConflict, + CallMediumEnum as CallMediumEnumSchema, + GenerateTestPromptRequest, + GenerateTestPromptResponse, + GenerateScenariosFromPromptRequest, + GenerateScenariosFromPromptResponse, + GenerateTestSetupRequest, + GenerateTestSetupResponse, + GeneratedScenarioDraftResponse, + TestPromptSectionResponse, +) + +router = APIRouter(prefix="/agents", tags=["agents"]) + + +def _validate_agent_phone_assignment( + db: Session, + *, + organization_id: UUID, + call_medium, + phone_number: Optional[str], + telephony_phone_number_id: Optional[UUID], + exclude_agent_id: Optional[UUID] = None, +) -> None: + """Raise HTTPException if phone assignment conflicts with another agent.""" + if call_medium != CallMediumEnum.PHONE_CALL: + return + if not phone_number and not telephony_phone_number_id: + return + + from app.services.telephony.phone_routing import find_agent_phone_assignment_conflict + + conflict = find_agent_phone_assignment_conflict( + db, + organization_id=organization_id, + phone_number=phone_number, + telephony_phone_number_id=telephony_phone_number_id, + exclude_agent_id=exclude_agent_id, + ) + if not conflict: + return + if conflict.get("error") == "telephony_not_found": + raise HTTPException(status_code=404, detail="Telephony phone number not found") + + agent_name = conflict["agent_name"] + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "message": f'This number is already assigned to agent "{agent_name}".', + "agent_id": str(conflict["agent_id"]), + "agent_name": agent_name, + "phone_number": conflict["phone_number"], + }, + ) + + +def resolve_agent_by_path_id( + db: Session, + *, + organization_id: UUID, + workspace_id: UUID, + agent_id: str, +) -> Agent: + """Resolve agent by UUID primary key or 6-digit agent_id (same as GET /agents/{id}).""" + try: + agent_uuid = UUID(agent_id) + agent = db.query(Agent).filter( + and_( + Agent.id == agent_uuid, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ) + ).first() + except ValueError: + agent = db.query(Agent).filter( + and_( + Agent.agent_id == agent_id, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ) + ).first() + if not agent: + raise HTTPException(status_code=404, detail=f"Agent {agent_id} not found") + return agent + + +# ====================================================================== +# AI Generation for agent descriptions +# ====================================================================== + +class GenerateAgentDescriptionRequest(BaseModel): + description: str + tone: Optional[str] = "professional" + format_style: Optional[str] = "structured" + provider: Optional[str] = None + model: Optional[str] = None + agent_id: Optional[UUID] = None + include_linked_scenarios: bool = True + append_scenarios_to_output: bool = False + + +GENERATE_AGENT_DESCRIPTION_SYSTEM = ( + "You are an expert at writing clear, well-structured descriptions for voice AI test agents. " + "The user will describe what they need the agent to do, and you will generate a comprehensive, " + "well-formatted agent description in markdown.\n\n" + "Guidelines:\n" + "- Use clear markdown structure: headings, bullet points, numbered lists\n" + "- Include sections for: Purpose, Behavior, Expected Interactions, Personality Traits, and Constraints\n" + "- Be specific about the agent's role, tone of voice, and how it should handle conversations\n" + "- Include example scenarios or edge cases where helpful\n" + "- Return ONLY the description in markdown, no preamble or explanation about what you did" +) + + +from app.services.ai.llm_resolver import get_llm_provider_and_model as _get_llm_provider_and_model + + +@router.post("/generate-description") +async def generate_agent_description( + data: GenerateAgentDescriptionRequest, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """Generate an agent description using AI from a brief description.""" + from app.services.ai.llm_service import llm_service + from app.services.testing.test_agent_simulation_prompt import ( + format_scenarios_for_generation_context, + format_scenarios_reference_appendix, + load_linked_scenarios_for_agent, + merge_generated_description_with_scenario_appendix, + ) + + if not data.description.strip(): + raise HTTPException(400, "Description is required") + + provider_enum, model_str = _get_llm_provider_and_model( + organization_id, db, data.provider, data.model + ) + + linked_scenarios = [] + if data.agent_id and data.include_linked_scenarios: + agent = db.query(Agent).filter( + Agent.id == data.agent_id, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ).first() + if not agent: + raise HTTPException(status_code=404, detail=f"Agent {data.agent_id} not found") + linked_scenarios = load_linked_scenarios_for_agent( + db, + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=agent.id, + ) + + user_prompt_parts = [ + "Create a detailed agent description for the following:", + "", + f"Description: {data.description}", + f"Tone: {data.tone or 'professional'}", + f"Format: {data.format_style or 'structured'}", + ] + scenario_context = format_scenarios_for_generation_context(linked_scenarios) + if scenario_context: + user_prompt_parts.extend(["", scenario_context]) + user_prompt_parts.extend([ + "", + "Generate a comprehensive, well-formatted agent description in markdown.", + ]) + user_prompt = "\n".join(user_prompt_parts) + + messages = [ + {"role": "system", "content": GENERATE_AGENT_DESCRIPTION_SYSTEM}, + {"role": "user", "content": user_prompt}, + ] + + try: + result = llm_service.generate_response( + messages=messages, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + temperature=0.7, + max_tokens=4000, + ) + content = result["text"] + if data.append_scenarios_to_output and linked_scenarios: + appendix = format_scenarios_reference_appendix(linked_scenarios) + content = merge_generated_description_with_scenario_appendix(content, appendix) + return {"content": content, "provider": provider_enum.value, "model": model_str} + except Exception as e: + logger.error(f"[Agents] AI description generation failed: {repr(e)}") + raise HTTPException(500, f"AI generation failed: {str(e)}") + + +def _test_prompt_section_responses(sections) -> list[TestPromptSectionResponse]: + return [ + TestPromptSectionResponse(key=s.key, title=s.title, content=s.content) + for s in sections + ] + + +def _scenario_draft_responses(scenarios) -> list[GeneratedScenarioDraftResponse]: + return [ + GeneratedScenarioDraftResponse(name=s.name, description=s.description, goal=s.goal) + for s in scenarios + ] + + +@router.post("/generate-test-prompt", response_model=GenerateTestPromptResponse) +async def generate_test_prompt( + data: GenerateTestPromptRequest, + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """Stage 1: generate foundational test agent prompt from production prompt.""" + from app.services.testing.agent_test_setup_generation import ( + generate_test_prompt_from_production, + ) + + if not data.production_prompt.strip(): + raise HTTPException(400, "Production prompt is required") + + provider_enum, model_str = _get_llm_provider_and_model( + organization_id, db, data.provider, data.model, data.credential_id + ) + + try: + result = generate_test_prompt_from_production( + data.production_prompt, + agent_name=data.agent_name, + language=data.language, + call_type=data.call_type, + additional_context=data.additional_context, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + llm_config=data.llm_config, + credential_id=data.credential_id, + ) + return GenerateTestPromptResponse( + sections=_test_prompt_section_responses(result.sections), + test_agent_prompt=result.test_agent_prompt, + provider=result.provider, + model=result.model, + ) + except ValueError as e: + raise HTTPException(400, str(e)) from e + except Exception as e: + logger.error(f"[Agents] Test prompt generation failed: {repr(e)}") + raise HTTPException(500, f"AI generation failed: {str(e)}") from e + + +@router.post("/generate-scenarios-from-prompt", response_model=GenerateScenariosFromPromptResponse) +async def generate_scenarios_from_prompt( + data: GenerateScenariosFromPromptRequest, + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """Stage 2: generate scenario drafts from a test agent prompt.""" + from app.services.testing.agent_test_setup_generation import ( + generate_scenarios_from_test_prompt, + ) + + if not data.test_agent_prompt.strip(): + raise HTTPException(400, "Test agent prompt is required") + + provider_enum, model_str = _get_llm_provider_and_model( + organization_id, db, data.provider, data.model, data.credential_id + ) + + try: + result = generate_scenarios_from_test_prompt( + data.test_agent_prompt, + agent_name=data.agent_name, + scenario_count=data.scenario_count, + language=data.language, + call_type=data.call_type, + additional_context=data.additional_context, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + llm_config=data.llm_config, + credential_id=data.credential_id, + ) + return GenerateScenariosFromPromptResponse( + scenarios=_scenario_draft_responses(result.scenarios), + provider=result.provider, + model=result.model, + ) + except ValueError as e: + raise HTTPException(400, str(e)) from e + except Exception as e: + logger.error(f"[Agents] Scenario generation failed: {repr(e)}") + raise HTTPException(500, f"AI generation failed: {str(e)}") from e + + +@router.post("/generate-test-setup", response_model=GenerateTestSetupResponse) +async def generate_test_setup( + data: GenerateTestSetupRequest, + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """Run stage 1 then stage 2: foundational test prompt + scenario drafts.""" + from app.services.testing.agent_test_setup_generation import ( + generate_scenarios_from_test_prompt, + generate_test_prompt_from_production, + ) + + if not data.production_prompt.strip(): + raise HTTPException(400, "Production prompt is required") + + provider_enum, model_str = _get_llm_provider_and_model( + organization_id, db, data.provider, data.model, data.credential_id + ) + + try: + prompt_result = generate_test_prompt_from_production( + data.production_prompt, + agent_name=data.agent_name, + language=data.language, + call_type=data.call_type, + additional_context=data.additional_context, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + llm_config=data.llm_config, + credential_id=data.credential_id, + ) + scenario_result = generate_scenarios_from_test_prompt( + prompt_result.test_agent_prompt, + agent_name=data.agent_name, + scenario_count=data.scenario_count, + language=data.language, + call_type=data.call_type, + additional_context=data.additional_context, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + llm_config=data.llm_config, + credential_id=data.credential_id, + ) + return GenerateTestSetupResponse( + sections=_test_prompt_section_responses(prompt_result.sections), + test_agent_prompt=prompt_result.test_agent_prompt, + scenarios=_scenario_draft_responses(scenario_result.scenarios), + provider=prompt_result.provider, + model=prompt_result.model, + ) + except ValueError as e: + raise HTTPException(400, str(e)) from e + except Exception as e: + logger.error(f"[Agents] Test setup generation failed: {repr(e)}") + raise HTTPException(500, f"AI generation failed: {str(e)}") from e + + +def generate_unique_agent_id(db: Session) -> str: + """Generate a unique 6-digit agent ID.""" + max_attempts = 100 + for _ in range(max_attempts): + agent_id = f"{random.randint(100000, 999999)}" + existing = db.query(Agent).filter(Agent.agent_id == agent_id).first() + if not existing: + return agent_id + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to generate unique agent ID" + ) + + +def get_agent_dependencies(db: Session, organization_id: UUID, agent_uuid: UUID) -> dict: + """Return dependency counts that block non-force delete.""" + evaluators_count = db.query(Evaluator).filter( + Evaluator.agent_id == agent_uuid, + Evaluator.organization_id == organization_id, + ).count() + + evaluator_results_count = db.query(EvaluatorResult).filter( + EvaluatorResult.agent_id == agent_uuid, + EvaluatorResult.organization_id == organization_id, + ).count() + + call_recordings_count = db.query(CallRecording).filter( + CallRecording.agent_id == agent_uuid, + CallRecording.organization_id == organization_id, + ).count() + + conversation_evaluations_count = db.query(ConversationEvaluation).filter( + ConversationEvaluation.agent_id == agent_uuid, + ConversationEvaluation.organization_id == organization_id, + ).count() + + test_conversations_count = db.query(TestAgentConversation).filter( + TestAgentConversation.agent_id == agent_uuid, + TestAgentConversation.organization_id == organization_id, + ).count() + + dependencies = {} + if evaluators_count > 0: + dependencies["evaluators"] = evaluators_count + if evaluator_results_count > 0: + dependencies["evaluator_results"] = evaluator_results_count + if call_recordings_count > 0: + dependencies["call_recordings"] = call_recordings_count + if conversation_evaluations_count > 0: + dependencies["conversation_evaluations"] = conversation_evaluations_count + if test_conversations_count > 0: + dependencies["test_conversations"] = test_conversations_count + + return dependencies + + +@router.post("", response_model=AgentResponse, status_code=status.HTTP_201_CREATED) +async def create_agent( + agent: AgentCreate, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db) +): + """Create a new test agent. + + The agent is stamped with the active workspace from the + ``X-Workspace-Id`` header (falling back to the org's Default). + """ + # Validate phone_number is provided when call_medium is phone_call + if agent.call_medium == CallMediumEnumSchema.PHONE_CALL and not agent.phone_number: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="phone_number is required when call_medium is phone_call" + ) + + # Validate voice_bundle_id exists, is active, and belongs to organization + voice_bundle = db.query(VoiceBundle).filter( + and_( + VoiceBundle.id == agent.voice_bundle_id, + VoiceBundle.organization_id == organization_id, + VoiceBundle.is_active == True, + ) + ).first() + if not voice_bundle: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Active voice bundle not found", + ) + + # Validate voice_ai_integration_id exists and belongs to organization + if agent.voice_ai_integration_id: + integration = db.query(Integration).filter( + and_( + Integration.id == agent.voice_ai_integration_id, + Integration.organization_id == organization_id, + Integration.is_active == True + ) + ).first() + if not integration: + raise HTTPException(status_code=404, detail="Integration not found or inactive") + + if integration.platform not in [ + IntegrationPlatform.RETELL, + IntegrationPlatform.VAPI, + IntegrationPlatform.ELEVENLABS, + IntegrationPlatform.SMALLEST, + ]: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Integration platform {integration.platform.value} is not supported for Voice AI agents. " + "Only Retell, Vapi, ElevenLabs, and Smallest are supported." + ) + ) + + if not agent.voice_ai_agent_id: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="voice_ai_agent_id is required when voice_ai_integration_id is provided" + ) + + _validate_agent_phone_assignment( + db, + organization_id=organization_id, + call_medium=agent.call_medium, + phone_number=agent.phone_number, + telephony_phone_number_id=agent.telephony_phone_number_id, + ) + + # Generate unique 6-digit agent_id + agent_id = generate_unique_agent_id(db) + + db_agent = Agent( + agent_id=agent_id, + organization_id=organization_id, + workspace_id=workspace_id, + name=agent.name, + phone_number=agent.phone_number, + language=agent.language, + description=agent.description, + call_type=agent.call_type, + call_medium=agent.call_medium, + telephony_phone_number_id=agent.telephony_phone_number_id, + voice_bundle_id=agent.voice_bundle_id, + ai_provider_id=agent.ai_provider_id, + voice_ai_integration_id=agent.voice_ai_integration_id, + voice_ai_agent_id=agent.voice_ai_agent_id, + provider_prompt=agent.provider_prompt, + silence_hangup_secs=agent.silence_hangup_secs, + ) + db.add(db_agent) + db.commit() + db.refresh(db_agent) + + from app.services.telephony.phone_routing import sync_agent_telephony_number_link + + sync_agent_telephony_number_link(db, db_agent) + db.refresh(db_agent) + + has_provider_prompt = isinstance(agent.provider_prompt, str) and bool(agent.provider_prompt.strip()) + if agent.voice_ai_integration_id and agent.voice_ai_agent_id and not has_provider_prompt: + try: + from app.services.voice_providers.prompt_sync import sync_provider_prompt + integration = db.query(Integration).filter(Integration.id == agent.voice_ai_integration_id).first() + if integration: + sync_provider_prompt(db_agent, integration, db) + db.refresh(db_agent) + except Exception as e: + logger.warning(f"[Agents] Best-effort provider prompt sync failed on create: {e}") + + return db_agent + + +@router.get("", response_model=List[AgentResponse]) +async def list_agents( + skip: int = 0, + limit: int = 100, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db) +): + """Get list of all agents for the active workspace. + + Scoped to (organization_id, workspace_id) so users only see agents + in the workspace they're currently in. + """ + agents = db.query(Agent).filter( + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ).offset(skip).limit(limit).all() + return agents + + +@router.get("/check-phone-assignment", response_model=AgentPhoneAssignmentCheckResponse) +async def check_phone_assignment( + phone_number: Optional[str] = Query(None), + telephony_phone_number_id: Optional[UUID] = Query(None), + exclude_agent_id: Optional[UUID] = Query(None), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +): + """Check whether a phone number is available for agent assignment in this org.""" + if not phone_number and not telephony_phone_number_id: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="phone_number or telephony_phone_number_id is required", + ) + + from app.services.telephony.phone_routing import find_agent_phone_assignment_conflict + + conflict = find_agent_phone_assignment_conflict( + db, + organization_id=organization_id, + phone_number=phone_number, + telephony_phone_number_id=telephony_phone_number_id, + exclude_agent_id=exclude_agent_id, + ) + if conflict and conflict.get("error") == "telephony_not_found": + raise HTTPException(status_code=404, detail="Telephony phone number not found") + if conflict: + return AgentPhoneAssignmentCheckResponse( + available=False, + phone_number=conflict["phone_number"], + conflict=AgentPhoneAssignmentConflict(**conflict), + ) + + resolved_phone = phone_number + if telephony_phone_number_id: + from app.models.database import TelephonyPhoneNumber + + row = ( + db.query(TelephonyPhoneNumber) + .filter( + TelephonyPhoneNumber.id == telephony_phone_number_id, + TelephonyPhoneNumber.organization_id == organization_id, + ) + .first() + ) + if row: + resolved_phone = row.phone_number + + return AgentPhoneAssignmentCheckResponse( + available=True, + phone_number=resolved_phone, + ) + + +@router.get("/{agent_id}", response_model=AgentResponse) +async def get_agent( + agent_id: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db) +): + """Get a specific agent by ID (UUID) or agent_id (6-digit) within the active workspace.""" + try: + # Try as UUID first + agent_uuid = UUID(agent_id) + agent = db.query(Agent).filter( + and_( + Agent.id == agent_uuid, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ) + ).first() + except ValueError: + # Try as 6-digit ID + agent = db.query(Agent).filter( + and_( + Agent.agent_id == agent_id, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ) + ).first() + + if not agent: + raise HTTPException(status_code=404, detail=f"Agent {agent_id} not found") + return agent + + +@router.put("/{agent_id}", response_model=AgentResponse) +async def update_agent( + agent_id: str, + agent_update: AgentUpdate, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db) +): + """Update an existing agent by ID (UUID) or agent_id (6-digit) within the active workspace.""" + try: + # Try as UUID first + agent_uuid = UUID(agent_id) + db_agent = db.query(Agent).filter( + and_( + Agent.id == agent_uuid, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ) + ).first() + except ValueError: + # Try as 6-digit ID + db_agent = db.query(Agent).filter( + and_( + Agent.agent_id == agent_id, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ) + ).first() + + if not db_agent: + raise HTTPException(status_code=404, detail=f"Agent {agent_id} not found") + + # Determine the call_medium to validate + call_medium = agent_update.call_medium if agent_update.call_medium is not None else db_agent.call_medium + + # Validate phone_number is provided when call_medium is phone_call + if call_medium == CallMediumEnum.PHONE_CALL: + phone_number = agent_update.phone_number if agent_update.phone_number is not None else db_agent.phone_number + if not phone_number: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="phone_number is required when call_medium is phone_call" + ) + + # Validate voice_bundle_id if provided + if agent_update.voice_bundle_id: + voice_bundle = db.query(VoiceBundle).filter( + and_( + VoiceBundle.id == agent_update.voice_bundle_id, + VoiceBundle.organization_id == organization_id + ) + ).first() + if not voice_bundle: + raise HTTPException(status_code=404, detail="Voice bundle not found") + + # Validate voice_ai_integration_id if provided + if agent_update.voice_ai_integration_id: + integration = db.query(Integration).filter( + and_( + Integration.id == agent_update.voice_ai_integration_id, + Integration.organization_id == organization_id, + Integration.is_active == True + ) + ).first() + if not integration: + raise HTTPException(status_code=404, detail="Integration not found or inactive") + + if integration.platform not in [ + IntegrationPlatform.RETELL, + IntegrationPlatform.VAPI, + IntegrationPlatform.ELEVENLABS, + IntegrationPlatform.SMALLEST, + ]: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Integration platform {integration.platform.value} is not supported for Voice AI agents. " + "Only Retell, Vapi, ElevenLabs, and Smallest are supported." + ) + ) + + if not agent_update.voice_ai_agent_id: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="voice_ai_agent_id is required when voice_ai_integration_id is provided" + ) + + update_data = agent_update.model_dump(exclude_unset=True, exclude_none=False) + + effective_call_medium = ( + agent_update.call_medium if agent_update.call_medium is not None else db_agent.call_medium + ) + effective_phone_number = ( + agent_update.phone_number + if "phone_number" in update_data + else db_agent.phone_number + ) + effective_telephony_id = ( + agent_update.telephony_phone_number_id + if "telephony_phone_number_id" in update_data + else db_agent.telephony_phone_number_id + ) + + _validate_agent_phone_assignment( + db, + organization_id=organization_id, + call_medium=effective_call_medium, + phone_number=effective_phone_number, + telephony_phone_number_id=effective_telephony_id, + exclude_agent_id=db_agent.id, + ) + + # Convert the update model to dict, handling None values properly + # Use model_dump with exclude_unset to only get fields that were explicitly provided + + # Apply updates + for field, value in update_data.items(): + setattr(db_agent, field, value) + + db.commit() + db.refresh(db_agent) + + from app.services.telephony.phone_routing import sync_agent_telephony_number_link + + if "telephony_phone_number_id" in update_data or "phone_number" in update_data: + sync_agent_telephony_number_link(db, db_agent) + db.refresh(db_agent) + + if "voice_ai_agent_id" in update_data or "voice_ai_integration_id" in update_data: + integration_id = db_agent.voice_ai_integration_id + provider_prompt_updated = "provider_prompt" in update_data + if integration_id and db_agent.voice_ai_agent_id and not provider_prompt_updated: + try: + from app.services.voice_providers.prompt_sync import sync_provider_prompt + integration = db.query(Integration).filter(Integration.id == integration_id).first() + if integration: + sync_provider_prompt(db_agent, integration, db) + db.refresh(db_agent) + except Exception as e: + logger.warning(f"[Agents] Best-effort provider prompt sync failed on update: {e}") + + return db_agent + + +@router.post("/{agent_id}/sync-provider-prompt") +async def sync_agent_provider_prompt( + agent_id: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """Fetch and store the current system prompt from the voice provider.""" + try: + agent_uuid = UUID(agent_id) + db_agent = db.query(Agent).filter( + and_( + Agent.id == agent_uuid, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ) + ).first() + except ValueError: + db_agent = db.query(Agent).filter( + and_( + Agent.agent_id == agent_id, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ) + ).first() + + if not db_agent: + raise HTTPException(status_code=404, detail=f"Agent {agent_id} not found") + + if not db_agent.voice_ai_integration_id or not db_agent.voice_ai_agent_id: + raise HTTPException( + status_code=400, + detail="Agent is not linked to an external voice provider", + ) + + integration = db.query(Integration).filter( + and_( + Integration.id == db_agent.voice_ai_integration_id, + Integration.organization_id == organization_id, + Integration.is_active == True, + ) + ).first() + if not integration: + raise HTTPException(status_code=404, detail="Integration not found or inactive") + + try: + from app.services.voice_providers.prompt_sync import sync_provider_prompt + prompt = sync_provider_prompt(db_agent, integration, db) + except Exception as e: + raise HTTPException(status_code=502, detail=f"Failed to fetch prompt from provider: {str(e)}") + + db.refresh(db_agent) + synced = isinstance(prompt, str) and bool(prompt.strip()) + if not synced: + raise HTTPException( + status_code=422, + detail="Provider returned no prompt. Verify the external agent has a system prompt configured.", + ) + return { + "synced": synced, + "provider_prompt": db_agent.provider_prompt, + "provider_prompt_synced_at": db_agent.provider_prompt_synced_at.isoformat() if db_agent.provider_prompt_synced_at else None, + } + + +@router.delete("/{agent_id}") +async def delete_agent( + agent_id: str, + force: bool = Query(False, description="Force delete with all dependent records"), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db) +): + """Delete an agent (scoped to the active workspace). Returns 409 if dependent records exist unless force=true.""" + try: + agent_uuid = UUID(agent_id) + db_agent = db.query(Agent).filter( + and_( + Agent.id == agent_uuid, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ) + ).first() + except ValueError: + db_agent = db.query(Agent).filter( + and_( + Agent.agent_id == agent_id, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ) + ).first() + + if not db_agent: + raise HTTPException(status_code=404, detail=f"Agent {agent_id} not found") + + agent_uuid = db_agent.id + dependencies = get_agent_dependencies(db, organization_id, agent_uuid) + + if dependencies and not force: + parts = [] + if dependencies.get("evaluators"): + parts.append(f"{dependencies['evaluators']} evaluator(s)") + if dependencies.get("evaluator_results"): + parts.append(f"{dependencies['evaluator_results']} evaluator result(s)") + if dependencies.get("call_recordings"): + parts.append(f"{dependencies['call_recordings']} call recording(s)") + if dependencies.get("conversation_evaluations"): + parts.append(f"{dependencies['conversation_evaluations']} conversation evaluation(s)") + if dependencies.get("test_conversations"): + parts.append(f"{dependencies['test_conversations']} test conversation(s)") + + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "message": f"Cannot delete agent. It is referenced by: {', '.join(parts)}.", + "dependencies": dependencies, + "hint": "Use force=true to delete this agent and all its dependent records.", + }, + ) + + if dependencies: + # Delete in FK-safe order: + # 1. EvaluatorResults (references evaluators and agents) + db.query(EvaluatorResult).filter( + EvaluatorResult.agent_id == agent_uuid, + ).delete(synchronize_session=False) + + # 2. Evaluators (references agents) + db.query(Evaluator).filter( + Evaluator.agent_id == agent_uuid, + ).delete(synchronize_session=False) + + # 3. Nullify call recordings (keep recordings, unlink agent) + db.query(CallRecording).filter( + CallRecording.agent_id == agent_uuid, + ).update({CallRecording.agent_id: None}, synchronize_session=False) + + # 4. ConversationEvaluations + db.query(ConversationEvaluation).filter( + ConversationEvaluation.agent_id == agent_uuid, + ).delete(synchronize_session=False) + + # 5. TestAgentConversations + db.query(TestAgentConversation).filter( + TestAgentConversation.agent_id == agent_uuid, + ).delete(synchronize_session=False) + + db.delete(db_agent) + db.commit() + + if dependencies: + return JSONResponse( + status_code=200, + content={ + "message": "Agent and all dependent records deleted successfully.", + "deleted": dependencies, + }, + ) + + return Response(status_code=204) + + +@router.get("/{agent_id}/delete-impact") +async def get_agent_delete_impact( + agent_id: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db) +): + """Preview dependent records that would be affected by force delete (scoped to the active workspace).""" + try: + agent_uuid = UUID(agent_id) + db_agent = db.query(Agent).filter( + and_( + Agent.id == agent_uuid, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ) + ).first() + except ValueError: + db_agent = db.query(Agent).filter( + and_( + Agent.agent_id == agent_id, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ) + ).first() + + if not db_agent: + raise HTTPException(status_code=404, detail=f"Agent {agent_id} not found") + + dependencies = get_agent_dependencies(db, organization_id, db_agent.id) + return { + "agent_id": str(db_agent.id), + "agent_name": db_agent.name, + "dependencies": dependencies, + "can_delete_without_force": len(dependencies) == 0, + } + + +from app.core.auth.capabilities import SIM_MANAGE, SIM_VIEW +from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities + +apply_workspace_route_capabilities( + router, + view_capability=SIM_VIEW, + manage_capability=SIM_MANAGE, +) + diff --git a/app/api/v1/routes/aiproviders.py b/app/api/v1/routes/aiproviders.py index 7104deb2..b7f8b9e6 100644 --- a/app/api/v1/routes/aiproviders.py +++ b/app/api/v1/routes/aiproviders.py @@ -67,12 +67,12 @@ def _scrub_for_response( effective_routing = get_credential_effective_routing_label( organization_id, db, - instance.routing_mode, + instance, ) effective_gateway_interface = get_credential_effective_gateway_interface( organization_id, db, - getattr(instance, "gateway_interface", None), + instance, ) has_gateway_auth_secret = bool(getattr(instance, "gateway_auth_secret", None)) db.expunge(instance) diff --git a/app/api/v1/routes/auth.py b/app/api/v1/routes/auth.py index 97dbc838..2a4437c5 100644 --- a/app/api/v1/routes/auth.py +++ b/app/api/v1/routes/auth.py @@ -54,6 +54,10 @@ provision_billing_customer, provision_default_workspace, ) +from app.services.signup_reference_codes import ( + consume_reference_code, + validate_reference_code_for_signup, +) router = APIRouter(prefix="/auth", tags=["Authentication"]) @@ -81,6 +85,7 @@ class AuthProviderConfig(BaseModel): class AuthConfigResponse(BaseModel): providers: List[AuthProviderConfig] tier: str # "oss" | "enterprise" + gated_signup: bool = False class SignupRequest(BaseModel): @@ -89,6 +94,7 @@ class SignupRequest(BaseModel): organization_name: Optional[str] = Field(default=None, max_length=255) first_name: Optional[str] = Field(default=None, max_length=255) last_name: Optional[str] = Field(default=None, max_length=255) + reference_code: Optional[str] = Field(default=None, max_length=64) class LoginRequest(BaseModel): @@ -200,7 +206,15 @@ def get_auth_config() -> AuthConfigResponse: ) ) - return AuthConfigResponse(providers=providers, tier=tier) + return AuthConfigResponse( + providers=providers, + tier=tier, + gated_signup=( + settings.AUTH_GATED_SIGNUP_ENABLED + and settings.AUTH_LOCAL_ALLOW_SIGNUP + and "local_password" in enabled + ), + ) # --------------------------------------------------------------------------- @@ -292,6 +306,10 @@ def signup(payload: SignupRequest, db: Session = Depends(get_db)) -> TokenRespon detail="Self-service signup is disabled. Contact your administrator for access.", ) + reference_row = None + if settings.AUTH_GATED_SIGNUP_ENABLED: + reference_row = validate_reference_code_for_signup(db, payload.reference_code) + existing = db.query(User).filter(User.email == payload.email).first() if existing: raise HTTPException( @@ -332,6 +350,8 @@ def signup(payload: SignupRequest, db: Session = Depends(get_db)) -> TokenRespon name=org_name, email=payload.email, ) + if reference_row is not None: + consume_reference_code(db, reference_row) user.last_login_at = datetime.now(timezone.utc) db.commit() db.refresh(user) @@ -368,14 +388,17 @@ def login(payload: LoginRequest, db: Session = Depends(get_db)) -> LoginResponse memberships = ( db.query(OrganizationMember, Organization) .join(Organization, Organization.id == OrganizationMember.organization_id) - .filter(OrganizationMember.user_id == user.id) + .filter( + OrganizationMember.user_id == user.id, + Organization.is_active == True, # noqa: E712 + ) .order_by(OrganizationMember.joined_at.asc()) .all() ) if not memberships: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, - detail="Your account is not a member of any organization. Contact your administrator.", + detail="Your account is not a member of any active organization. Contact your administrator.", ) if len(memberships) > 1 and not payload.organization_id: @@ -512,6 +535,13 @@ def refresh_session(payload: RefreshRequest, db: Session = Depends(get_db)) -> T detail="User is not a member of this organization.", ) + org = db.query(Organization).filter(Organization.id == row.organization_id).first() + if org is None or not org.is_active: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Organization disabled.", + ) + revoke_refresh_token(db, payload.refresh_token) role_value = membership.role.value if hasattr(membership.role, "value") else membership.role return _issue_session_tokens( @@ -596,6 +626,13 @@ def switch_organization( detail="You are not a member of that organization.", ) + org = db.query(Organization).filter(Organization.id == target_org_id).first() + if org is None or not org.is_active: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Organization disabled.", + ) + user = db.query(User).filter(User.id == principal.user_id).first() if user is None or not user.is_active: raise HTTPException( diff --git a/app/api/v1/routes/call_import_evaluations.py b/app/api/v1/routes/call_import_evaluations.py index 109004a3..a57e2e58 100644 --- a/app/api/v1/routes/call_import_evaluations.py +++ b/app/api/v1/routes/call_import_evaluations.py @@ -1,9309 +1,9452 @@ -"""Evaluation routes scoped to a Call Import batch.""" - -from __future__ import annotations - -import asyncio -import csv -import base64 -import io -import json -import math -import re -import statistics -from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple -from uuid import UUID, uuid4 - -from datetime import date, datetime, timedelta, timezone - -from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status -from fastapi.responses import StreamingResponse -from loguru import logger -from pydantic import BaseModel, Field, field_validator -from sqlalchemy import desc, func, or_, text -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Session -from sqlalchemy.orm.attributes import flag_modified - -from app.core.auth import Principal, get_principal -from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message -from app.database import get_db -from app.dependencies import ( - get_api_key, - get_organization_id, - get_workspace_id, - require_enterprise_feature, -) -from app.services.call_imports.audit import ( - actor_emails_for_evaluation, - emails_for_user_ids, - stamp_call_import_actor, - stamp_evaluation_actor, - user_ids_from_evaluations, -) -from app.services.workspace_rbac import resolve_workspace_capabilities -from app.models.database import ( - AIProvider, - CallImport, - CallImportEvaluation, - CallImportEvaluationReportSnapshot, - CallImportEvaluationPdfReport, - CallImportEvaluationRow, - CallImportRow, - Metric, - PromptPartial, - Workspace, -) -from app.models.enums import CallImportRowStatus, ModelProvider -from app.models.schemas import ( - CallImportEvaluationAggregateResponse, - CallImportEvaluationBulkDelete, - CallImportEvaluationBulkActionResponse, - CallImportEvaluationCreate, - CallImportEvaluationListResponse, - CallImportEvaluationResponse, - CallImportEvaluationRetryRequest, - CallImportEvaluationRetryResponse, - CallImportEvaluationRetrySkippedItem, - CallImportEvaluationRowListResponse, - CallImportEvaluationRowResponse, - CallImportEvaluationUpdate, - CallImportMetricAggregate, - CallImportMetricHistogramBucket, - CallImportMetricLabelPair, - CallImportMetricSummary, - CallImportMetricValueCount, - DiscoveredLabelDeleteRequest, - DiscoveredLabelItem, - DiscoveredLabelMergeRequest, - DiscoveredLabelsResponse, - DiscoveredMetricDeleteRequest, - DiscoveredMetricItem, - DiscoveredMetricMergeRequest, - DiscoveredMetricsResponse, - EvaluationInsightsRequest, - EvaluationTldrSummary, - EvaluationMetricClustersRequest, - EvaluationMetricClustersState, - EvaluationPromptImprovementsRequest, - EvaluationPromptImprovementsState, - MetricFailurePoliciesResponse, - MetricFailurePoliciesSaveRequest, - MetricFailurePolicy, - MetricClusterEligibleRow, - MetricClusterEligibleRowsResponse, - EvaluationUserInsightsRequest, - EvaluationUserInsightsState, - MetricFlowEdge, - MetricPeriodDelta, - MetricFlowNode, - MetricFlowResponse, -) -from app.services.reporting.call_import_evaluation_pdf_report import ( - call_import_evaluation_pdf_report_service, -) -from app.services.reporting.call_import_pdf_report_storage import ( - build_pdf_report_s3_key, - compute_pdf_report_cache_fingerprint, - compute_pdf_report_config_fingerprint, - compute_pdf_report_content_fingerprint, - config_summary_from_report_config, - find_cached_pdf_report, - presigned_urls_for_pdf_report, -) -from app.services.call_import_metric_clusters import ( - METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, - estimate_metric_clusters_llm_calls, - filter_completed_row_pairs, - list_eligible_cluster_rows, - metric_clusters_raw_is_cancelled, - metric_clusters_state_from_raw, - metric_clusters_state_to_db, -) -from app.services.metric_failure_policy import ( - aggregate_primary_percent, - build_failure_policy_previews, - effective_policies, - failure_rate_percent_from_rows, - failure_policies_to_db, - has_clusterable_metrics, - merge_clustering_policies, - merge_failure_policies_into_raw, - policies_from_evaluation_raw, - validate_failure_policies_for_metrics, -) -from app.services.call_import_user_insights import ( - normalize_max_llm_calls, - total_llm_calls_for_rows, - user_insights_state_from_raw, -) - -router = APIRouter( - prefix="/call-imports/{call_import_id}/evaluations", - tags=["Call Import Evaluations"], - dependencies=[Depends(require_enterprise_feature("call_imports"))], -) - - -class CallImportEvaluationPdfReportRequest(BaseModel): - vendor_name: str = Field(..., min_length=1, max_length=120) - report_type: Literal["external", "internal"] = "external" - include_weekly_delta: bool = False - include_period_delta: bool = False - baseline_evaluation_id: Optional[str] = None - period_label: Optional[str] = Field(default=None, max_length=64) - use_case: Optional[str] = Field(default=None, max_length=120) - internal_brand_image_id: Optional[str] = None - external_brand_image_id: Optional[str] = None - report_config: Dict[str, Any] = Field(default_factory=dict) - platform_base_url: Optional[str] = Field( - default=None, - max_length=512, - description="Frontend origin for deep links to example calls in internal PDFs.", - ) - - @field_validator("vendor_name") - @classmethod - def _clean_vendor_name(cls, value: str) -> str: - cleaned = value.strip() - if not cleaned: - raise ValueError("Vendor name is required.") - return cleaned - - -class CallImportEvaluationPdfReportResponse(BaseModel): - id: str - filename: str - preview_url: Optional[str] = None - download_url: Optional[str] = None - created_at: datetime - created_by: Optional[str] = None - report_type: str - vendor_name: str - config_summary: Optional[str] = None - storage_available: bool = True - cache_hit: bool = False - - -class CallImportEvaluationPdfReportListItem(BaseModel): - id: str - filename: Optional[str] = None - vendor_name: str - report_type: str - created_by: Optional[str] = None - created_at: datetime - config_summary: Optional[str] = None - cache_fingerprint: Optional[str] = None - - -class CallImportEvaluationPdfReportListResponse(BaseModel): - items: List[CallImportEvaluationPdfReportListItem] - - -class CallImportEvaluationBaselineCandidate(BaseModel): - evaluation_id: str - name: str - dataset: str - period_label: Optional[str] = None - period_start: Optional[date] = None - period_end: Optional[date] = None - period_display: str - completed_rows: int - created_at: datetime - is_default: bool = False - - -class CallImportEvaluationBaselineCandidatesResponse(BaseModel): - items: List[CallImportEvaluationBaselineCandidate] - default_evaluation_id: Optional[str] = None - - -def _require_import( - db: Session, - call_import_id: UUID, - organization_id: UUID, -) -> CallImport: - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException(status_code=404, detail="Call import not found") - return call_import - - -def require_call_import_capability(capability: str): - """Ensure the caller has *capability* in the call import's workspace (not just the header).""" - - def _dep( - call_import_id: UUID, - principal: Principal = Depends(get_principal), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), - ) -> CallImport: - call_import = _require_import(db, call_import_id, organization_id) - caps, _, role = resolve_workspace_capabilities( - db, - principal=principal, - workspace_id=call_import.workspace_id, - organization_id=organization_id, - ) - if capability not in caps: - raise HTTPException( - status_code=403, - detail=capability_denied_message( - capability, - role_name=role.name if role else None, - workspace_label="the active workspace", - ), - ) - return call_import - - return _dep - - -def _flatten_transcript(text: Optional[str]) -> str: - """Collapse a multi-line transcript onto a single line for spreadsheet export. - - The diarised transcript is stored as ``: `` lines joined - by ``\\n`` because the in-app ``TranscriptView`` parses those line - breaks to render chat bubbles. In Excel / Google Sheets that same - newline-per-turn formatting causes each cell to balloon vertically, - which the user reads as "lots of empty space on top of the cell". - Flattening at export time keeps the DB shape intact while giving the - spreadsheet a single-line cell per row. - """ - if not text: - return "" - parts = [ - segment.strip() - for segment in text.replace("\r\n", "\n").replace("\r", "\n").split("\n") - ] - return " ".join(p for p in parts if p) - - -def _evaluated_transcript_source_label( - evaluation: CallImportEvaluation, - source_row: CallImportRow, -) -> str: - """Label which transcript source this row was scored against.""" - source = (evaluation.transcript_source or "diarised").strip().lower() - if source == "production": - if not (source_row.transcript or "").strip(): - return "" - return "Production" - if not (source_row.diarised_transcript or "").strip(): - return "" - return "Diarised" - - -def _pick_evaluation_row_transcript( - source_row: Optional[CallImportRow], - evaluation: Optional[CallImportEvaluation] = None, -) -> Optional[str]: - """Transcript shown in evaluation row detail for the run's source.""" - if source_row is None: - return None - source = ( - (evaluation.transcript_source or "diarised").strip().lower() - if evaluation is not None - else "diarised" - ) - if source == "production": - raw = (source_row.transcript or "").strip() - return raw or None - diarised = (source_row.diarised_transcript or "").strip() - if diarised: - return diarised - raw = (source_row.transcript or "").strip() - return raw or None - - -def _to_evaluation_row_response( - eval_row_obj: CallImportEvaluationRow, - source_row: Optional[CallImportRow], - evaluation: Optional[CallImportEvaluation] = None, -) -> CallImportEvaluationRowResponse: - """Serialize one evaluation row plus joined source-row metadata.""" - return CallImportEvaluationRowResponse( - id=eval_row_obj.id, - evaluation_id=eval_row_obj.evaluation_id, - call_import_row_id=eval_row_obj.call_import_row_id, - row_index=source_row.row_index if source_row else None, - conversation_id=source_row.conversation_id if source_row else None, - transcript=_pick_evaluation_row_transcript(source_row, evaluation), - raw_columns=source_row.raw_columns if source_row else None, - recording_url=source_row.recording_url if source_row else None, - recording_date=source_row.recording_date if source_row else None, - recording_s3_key=source_row.recording_s3_key if source_row else None, - diarised_transcript_status=( - source_row.diarised_transcript_status if source_row else None - ), - diarised_transcript_error=( - source_row.diarised_transcript_error if source_row else None - ), - status=eval_row_obj.status, - metric_scores=eval_row_obj.metric_scores or {}, - error_message=eval_row_obj.error_message, - started_at=eval_row_obj.started_at, - finished_at=eval_row_obj.finished_at, - created_at=eval_row_obj.created_at, - updated_at=eval_row_obj.updated_at, - ) - - -def _serialize_selected_metric_ids(value) -> List[UUID]: - result: List[UUID] = [] - if not isinstance(value, list): - return result - for item in value: - try: - result.append(UUID(str(item))) - except (TypeError, ValueError): - continue - return result - - -def _metrics_for_ids(db: Session, org_id: UUID, ids: List[UUID]) -> List[Metric]: - if not ids: - return [] - rows = ( - db.query(Metric) - .filter( - Metric.organization_id == org_id, - Metric.id.in_(ids), - ) - .all() - ) - by_id = {row.id: row for row in rows} - return [by_id[mid] for mid in ids if mid in by_id] - - -def _expand_metric_selection( - db: Session, - org_id: UUID, - selected_ids: List[UUID], -) -> Tuple[List[Metric], Dict[UUID, List[Metric]]]: - """Resolve user-supplied metric ids into actual leaves + parent grouping. - - Rules: - * If a parent id is in ``selected_ids`` and no specific children of - that parent are also listed, include EVERY enabled child of that - parent. - * If a parent id AND some of its children are listed, include only - the listed children (treat the parent selection as the - "container" so users can deselect labels). - * Standalone metrics (no parent, no children) pass through - unchanged. - * Disabled metrics are filtered out at this layer so the caller - doesn't have to repeat the check. - - Returns: - (effective_metrics, parent_to_children) - - ``effective_metrics`` is the deduplicated list of metrics the - worker will actually score (children + standalone). Order is - preserved from ``selected_ids`` for display stability. - - ``parent_to_children`` maps each parent metric id (UUID) to the - list of its selected children. Useful for grouping in the LLM - prompt builder. - """ - if not selected_ids: - return [], {} - - requested = list(selected_ids) - initial_rows = ( - db.query(Metric) - .filter( - Metric.organization_id == org_id, - Metric.id.in_(requested), - ) - .all() - ) - initial_by_id = {row.id: row for row in initial_rows} - - parent_ids_requested = { - m.id for m in initial_rows if m.selection_mode and not m.parent_metric_id - } - # Map parent id -> children explicitly requested by the user. - explicit_children_by_parent: Dict[UUID, List[Metric]] = {} - for m in initial_rows: - if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: - explicit_children_by_parent.setdefault( - m.parent_metric_id, [] - ).append(m) - - # For parents without explicit children, hydrate every enabled child. - parents_needing_full_expansion = [ - pid - for pid in parent_ids_requested - if pid not in explicit_children_by_parent - ] - auto_expanded_children: Dict[UUID, List[Metric]] = {} - if parents_needing_full_expansion: - for pid in parents_needing_full_expansion: - child_rows = ( - db.query(Metric) - .filter( - Metric.organization_id == org_id, - Metric.parent_metric_id == pid, - Metric.enabled.is_(True), - ) - .order_by(Metric.created_at.asc()) - .all() - ) - auto_expanded_children[pid] = child_rows - - parent_to_children: Dict[UUID, List[Metric]] = {} - for pid in parent_ids_requested: - children = explicit_children_by_parent.get( - pid - ) or auto_expanded_children.get(pid, []) - # Drop disabled children so the worker doesn't waste a slot on - # them. Empty parents (no enabled children) are still tracked - # because the UI may want to show "0 of 0" rather than swallow - # them silently. - parent_to_children[pid] = [c for c in children if c.enabled] - - effective: List[Metric] = [] - seen: set[UUID] = set() - for mid in requested: - m = initial_by_id.get(mid) - if m is None: - continue - if m.selection_mode and not m.parent_metric_id: - # Parent row itself is not scored — only its children. - for child in parent_to_children.get(m.id, []): - if child.id in seen or not child.enabled: - continue - seen.add(child.id) - effective.append(child) - continue - if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: - # Already accounted for via the parent expansion above. - continue - if not m.enabled: - continue - if m.id in seen: - continue - seen.add(m.id) - effective.append(m) - - return effective, parent_to_children - - -def _evaluation_bulk_operation_for_response( - evaluation_id: UUID, -) -> Optional[str]: - from app.services.call_imports.evaluation_bulk_op import ( - get_evaluation_bulk_operation, - ) - - return get_evaluation_bulk_operation(evaluation_id) - - -def _serialize_eval( - db: Session, - row: CallImportEvaluation, - *, - sibling_evaluation_ids: Optional[List[UUID]] = None, - user_emails: Optional[Dict[UUID, str]] = None, -) -> CallImportEvaluationResponse: - selected_ids = _serialize_selected_metric_ids(row.selected_metric_ids) - - # Pull every metric referenced anywhere in the run's grouping (leaves, - # standalone, AND parents from selected_metric_groups) so the UI can - # render parent labels even when only children were materialized into - # selected_metric_ids. - groups_raw: Dict[str, List[str]] = {} - if isinstance(row.selected_metric_groups, dict): - for parent_str, children in row.selected_metric_groups.items(): - if not isinstance(children, list): - continue - cleaned: List[str] = [] - for c in children: - try: - UUID(str(c)) - cleaned.append(str(c)) - except (TypeError, ValueError): - continue - try: - UUID(parent_str) - groups_raw[parent_str] = cleaned - except (TypeError, ValueError): - continue - - metric_ids_for_lookup: List[UUID] = list(selected_ids) - for parent_str in groups_raw.keys(): - try: - pid = UUID(parent_str) - if pid not in metric_ids_for_lookup: - metric_ids_for_lookup.append(pid) - except (TypeError, ValueError): - continue - - metrics = _metrics_for_ids( - db, row.organization_id, metric_ids_for_lookup - ) - - from app.services.call_imports.progress_counters import merge_eval_counters_for_ui - - ui_completed_raw, ui_failed_raw = merge_eval_counters_for_ui(row) - total = int(row.total_rows or 0) - ui_completed = ( - min(ui_completed_raw, total) if total else ui_completed_raw - ) - ui_failed = min(ui_failed_raw, total) if total else ui_failed_raw - - if user_emails is None: - user_emails = emails_for_user_ids(db, user_ids_from_evaluations([row])) - created_email, updated_email = actor_emails_for_evaluation(row, user_emails) - - return CallImportEvaluationResponse( - id=row.id, - call_import_id=row.call_import_id, - organization_id=row.organization_id, - name=row.name, - selected_metric_ids=selected_ids, - selected_metric_groups=groups_raw or None, - metrics=[ - CallImportMetricSummary( - id=metric.id, - name=metric.name, - metric_type=metric.metric_type, - description=metric.description, - parent_metric_id=metric.parent_metric_id, - selection_mode=metric.selection_mode, - # Required by the Flow tab to know whether a parent - # opted into discovery; without it the - # DiscoveredLabelsPanel stays hidden even when the - # worker is actively producing discovered_labels. - allow_discovery=bool( - getattr(metric, "allow_discovery", False) - ), - ) - for metric in metrics - ], - status=row.status, - total_rows=row.total_rows, - completed_rows=ui_completed, - failed_rows=ui_failed, - error_message=row.error_message, - llm_provider=row.llm_provider, - llm_model=row.llm_model, - llm_credential_id=row.llm_credential_id, - llm_config=( - row.llm_config if isinstance(getattr(row, "llm_config", None), dict) else None - ), - metric_llm_overrides=( - row.metric_llm_overrides - if isinstance(row.metric_llm_overrides, dict) - else None - ), - stt_provider=row.stt_provider, - stt_model=row.stt_model, - stt_credential_id=row.stt_credential_id, - diarisation_llm_provider=getattr(row, "diarisation_llm_provider", None), - diarisation_llm_model=getattr(row, "diarisation_llm_model", None), - diarisation_llm_credential_id=getattr( - row, "diarisation_llm_credential_id", None - ), - diarisation_prompt=getattr(row, "diarisation_prompt", None), - transcribe_mode=( - (getattr(row, "transcribe_mode", None) or "stt_llm") - ), - transcript_source=(row.transcript_source or "diarised"), - sibling_evaluation_ids=list(sibling_evaluation_ids or []), - started_at=row.started_at, - finished_at=row.finished_at, - created_at=row.created_at, - updated_at=row.updated_at, - created_by_email=created_email, - last_updated_by_email=updated_email, - tldr_summary=_tldr_summary_payload(row), - user_insights=_user_insights_payload(row), - metric_clusters=_metric_clusters_payload(row), - discover_new_metrics=bool( - getattr(row, "discover_new_metrics", False) - ), - bulk_operation=_evaluation_bulk_operation_for_response(row.id), - ) - - -def _normalize_name(value: Optional[str]) -> Optional[str]: - """Trim user-supplied name; empty string becomes ``NULL``.""" - if value is None: - return None - trimmed = value.strip() - return trimmed or None - - -def _rollup_evaluation_status(evaluation: CallImportEvaluation, db: Session) -> None: - """Recompute counters + terminal status after rows are added/removed. - - Uses a single aggregate query instead of loading every row status. - """ - from app.workers.tasks.evaluate_call_import_row_core import ( - _apply_parent_status_from_counters, - reconcile_evaluation_counters, - ) - - reconcile_evaluation_counters(db, evaluation) - _apply_parent_status_from_counters(evaluation) - db.flush() - - if evaluation.status in {"completed", "failed", "partial"}: - from app.models.database import CallImport - from app.services.call_imports.bulk_ops import rollup_call_import_batch_status - - call_import = ( - db.query(CallImport) - .filter(CallImport.id == evaluation.call_import_id) - .first() - ) - if call_import is not None: - rollup_call_import_batch_status(db, call_import) - - -@router.post( - "", - response_model=CallImportEvaluationResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="createCallImportEvaluation", -) -async def create_call_import_evaluation( - call_import_id: UUID, - payload: CallImportEvaluationCreate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportEvaluationResponse: - del api_key - call_import = _require_import(db, call_import_id, organization_id) - - metric_ids = payload.metric_ids - if not metric_ids: - raise HTTPException( - status_code=400, - detail="Select at least one metric to run the evaluation against.", - ) - - org_metrics = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.id.in_(metric_ids), - ) - .all() - ) - by_id = {metric.id: metric for metric in org_metrics} - unknown_ids = [mid for mid in metric_ids if mid not in by_id] - if unknown_ids: - raise HTTPException( - status_code=400, - detail=( - "These metric ids do not exist in your organization: " - f"{', '.join(str(mid) for mid in unknown_ids)}. " - "Refresh the metrics list and try again." - ), - ) - # Parents themselves are containers, not scored rows, so a disabled - # parent shouldn't block the run as long as it has enabled children. - # We only reject disabled rows that the worker will actually try to - # evaluate (children + standalone leaves). - disabled_leaves = [ - metric - for metric in org_metrics - if not metric.enabled - and not (metric.selection_mode and not metric.parent_metric_id) - ] - if disabled_leaves: - names = ", ".join(metric.name for metric in disabled_leaves) - raise HTTPException( - status_code=400, - detail=( - f"These metrics are disabled and cannot be evaluated: {names}. " - "Enable them on the Metrics page (or pick different ones) and " - "try again." - ), - ) - - # Expand hierarchical selection: parents auto-include their enabled - # children, mixed parent+child selections respect the user's subset. - effective_metrics, parent_to_children = _expand_metric_selection( - db, organization_id, metric_ids - ) - if not effective_metrics: - raise HTTPException( - status_code=400, - detail=( - "None of the selected metrics yielded an enabled leaf to " - "evaluate. Check that parent categories have enabled " - "children, then try again." - ), - ) - - # The effective list (children + standalone leaves) is what gets - # persisted to ``selected_metric_ids`` and scored by the worker. - # The original parents are preserved in ``selected_metric_groups`` - # so the UI can rebuild the tree later. - leaf_metric_ids: List[UUID] = [m.id for m in effective_metrics] - selected_metric_groups: Dict[str, List[str]] = { - str(pid): [str(c.id) for c in children] - for pid, children in parent_to_children.items() - } - metric_rows = effective_metrics - valid_metric_id_strs = {str(m.id) for m in metric_rows} - - # ----- Validate run-level + per-metric LLM config ----- - llm_provider_norm: Optional[str] = None - llm_model_norm: Optional[str] = None - if payload.llm_provider or payload.llm_model: - if not (payload.llm_provider and payload.llm_model): - raise HTTPException( - status_code=400, - detail="Both llm_provider and llm_model are required when overriding the run LLM.", - ) - try: - llm_provider_norm = ModelProvider( - payload.llm_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Unknown LLM provider '{payload.llm_provider}'. " - "Valid keys are documented in ModelProvider." - ), - ) - llm_model_norm = payload.llm_model.strip() or None - if not llm_model_norm: - raise HTTPException( - status_code=400, detail="llm_model cannot be empty." - ) - - if payload.llm_credential_id is not None: - cred = ( - db.query(AIProvider) - .filter( - AIProvider.id == payload.llm_credential_id, - AIProvider.organization_id == organization_id, - ) - .first() - ) - if not cred: - raise HTTPException( - status_code=400, - detail=( - "The provided llm_credential_id does not exist in this " - "organization." - ), - ) - - # Per-metric overrides: keys can be either a leaf metric id (applies - # to that metric only) or a parent metric id (applies to every - # child of that parent). Parent keys are expanded to their - # children so the worker only sees concrete leaf ids. - metric_overrides_payload: Optional[Dict[str, Dict[str, Any]]] = None - if payload.metric_llm_overrides: - metric_overrides_payload = {} - for metric_id, override in payload.metric_llm_overrides.items(): - target_leaf_ids: List[str] = [] - if metric_id in valid_metric_id_strs: - target_leaf_ids = [metric_id] - else: - # Maybe it's a parent id — expand to the children that - # are part of THIS run. - try: - parent_uuid = UUID(metric_id) - except (TypeError, ValueError): - raise HTTPException( - status_code=400, - detail=( - "metric_llm_overrides references metric " - f"{metric_id} which is not a valid UUID." - ), - ) - children_for_parent = parent_to_children.get(parent_uuid) - if not children_for_parent: - raise HTTPException( - status_code=400, - detail=( - "metric_llm_overrides references metric " - f"{metric_id} which is not in metric_ids." - ), - ) - target_leaf_ids = [str(c.id) for c in children_for_parent] - - override_dict: Dict[str, Any] = {} - if override.provider is not None: - if not override.model: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} has a provider " - "but no model." - ), - ) - try: - override_dict["provider"] = ModelProvider( - override.provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} uses unknown " - f"provider '{override.provider}'." - ), - ) - override_dict["model"] = override.model.strip() - elif override.model: - # Model without provider doesn't make sense — treat as 400 - # so the UI can fix it instead of silently falling back. - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} has a model but " - "no provider." - ), - ) - if override.credential_id is not None: - override_dict["credential_id"] = str(override.credential_id) - if override.llm_config is not None: - override_dict["llm_config"] = override.llm_config - if override_dict: - for leaf_id in target_leaf_ids: - metric_overrides_payload[leaf_id] = override_dict - - # ----- Validate auto-transcribe settings ----- - # Diarised runs auto-diarise rows missing a diarised transcript and - # require STT + diariser LLM config. Production runs score the CSV - # transcript directly and skip diarisation entirely. - use_diarised = payload.transcript_sources[0] == "diarised" - auto_transcribe = use_diarised - - transcribe_mode_norm: Optional[str] = None - stt_provider_norm: Optional[str] = None - stt_model_norm: Optional[str] = None - diarisation_llm_provider_norm: Optional[str] = None - diarisation_llm_model_norm: Optional[str] = None - diarisation_prompt_norm: Optional[str] = None - - if use_diarised: - transcribe_mode_norm = (payload.transcribe_mode or "stt_llm").strip().lower() - if transcribe_mode_norm not in {"stt_llm", "llm_only"}: - raise HTTPException( - status_code=400, - detail=( - f"Unknown transcribe_mode '{payload.transcribe_mode}'. " - "Expected 'stt_llm' or 'llm_only'." - ), - ) - - if transcribe_mode_norm == "stt_llm": - if not payload.stt_provider: - raise HTTPException( - status_code=400, - detail=( - "stt_provider is required when " - "transcribe_mode='stt_llm': every evaluation run " - "auto-diarises rows that are missing a diarised " - "transcript." - ), - ) - if not payload.stt_model: - raise HTTPException( - status_code=400, - detail=( - "stt_model is required when transcribe_mode='stt_llm'." - ), - ) - try: - stt_provider_norm = ModelProvider( - payload.stt_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=f"Unknown STT provider '{payload.stt_provider}'.", - ) - stt_model_norm = payload.stt_model.strip() or None - if not stt_model_norm: - raise HTTPException( - status_code=400, detail="stt_model cannot be empty." - ) - else: - # llm_only — explicitly reject lingering STT inputs so the - # contract is unambiguous (the worker would ignore them but - # silent acceptance hides accidental misconfiguration). - if (payload.stt_provider or "").strip() or ( - payload.stt_model or "" - ).strip(): - raise HTTPException( - status_code=400, - detail=( - "stt_provider / stt_model must be omitted when " - "transcribe_mode='llm_only'; the LLM consumes the " - "audio directly." - ), - ) - - # --- Validate LLM diariser settings ----- - if not payload.diarization_llm_provider: - raise HTTPException( - status_code=400, - detail=( - "diarization_llm_provider is required: every evaluation " - "run diarises STT output with an LLM." - ), - ) - if not payload.diarization_llm_model: - raise HTTPException( - status_code=400, - detail=( - "diarization_llm_model is required: every evaluation " - "run diarises STT output with an LLM." - ), - ) - try: - diarisation_llm_provider_norm = ModelProvider( - payload.diarization_llm_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Unknown diarisation LLM provider " - f"'{payload.diarization_llm_provider}'." - ), - ) - diarisation_llm_model_norm = ( - payload.diarization_llm_model.strip() or None - ) - if not diarisation_llm_model_norm: - raise HTTPException( - status_code=400, - detail="diarization_llm_model cannot be empty.", - ) - diarisation_prompt_norm = ( - payload.diarization_prompt.strip() - if isinstance(payload.diarization_prompt, str) - else None - ) or None - - from app.models.enums import CallImportParameterType, CallImportStatus - from app.services.call_imports.bulk_ops import ( - count_all_source_rows, - count_completed_source_rows, - count_source_rows_with_production_transcript, - ) - - starting_from_mapped = False - if call_import.status == CallImportStatus.MAPPED: - if not call_import.source_s3_key or not call_import.source_format: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This batch has no staged source file. Upload and map " - "a CSV/Excel file before running evaluation." - ), - ) - if not call_import.schema_id: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="Cannot run evaluation without a mapped schema.", - ) - from app.api.v1.routes.call_imports import ( - _ensure_blob_storage_enabled, - _resolve_schema, - _resolve_telephony_integration, - _validate_direct_url_import_ready, - ) - - workspace_id = call_import.workspace_id - schema = _resolve_schema( - db, organization_id, workspace_id, call_import.schema_id - ) - parameters = list(schema.parameters) - if not use_diarised: - transcript_mapped = any( - param.type == CallImportParameterType.TRANSCRIPT - and (call_import.parameter_mapping or {}).get(param.name) - for param in parameters - ) - if not transcript_mapped: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "No transcript column is mapped in this batch. " - "Map a schema transcript parameter to a CSV column, " - "or choose 'Diarize then evaluate'." - ), - ) - if payload.telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, - organization_id, - payload.telephony_integration_id, - payload.provider or "", - ) - else: - _validate_direct_url_import_ready( - parameters, dict(call_import.parameter_mapping or {}) - ) - integration = None - - _ensure_blob_storage_enabled() - - if integration is not None: - call_import.provider = integration.provider - call_import.telephony_integration_id = integration.id - else: - call_import.provider = None - call_import.telephony_integration_id = None - - call_import.total_rows = 0 - call_import.completed_rows = 0 - call_import.failed_rows = 0 - call_import.error_message = None - call_import.status = CallImportStatus.PROCESSING - stamp_call_import_actor(call_import, principal) - db.commit() - db.refresh(call_import) - starting_from_mapped = True - - if use_diarised: - total_row_count = count_completed_source_rows(db, call_import.id) - else: - # Production runs score CSV text — rows need not wait for - # recording fetch to finish before they are evaluable. - total_row_count = count_source_rows_with_production_transcript( - db, call_import.id - ) - - requested_sources: List[str] = list(payload.transcript_sources) - - if ( - not use_diarised - and not starting_from_mapped - and count_all_source_rows(db, call_import.id) > 0 - and total_row_count == 0 - ): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "No rows have a production transcript. " - "Choose 'Diarize then evaluate' or import rows with " - "a transcript column." - ), - ) - - base_name = _normalize_name(payload.name) - - def _name_for_source(source: str) -> Optional[str]: - # Single-source runs preserve the user's chosen name verbatim. - del source - return base_name - - created_evaluations: List[CallImportEvaluation] = [] - - for source in requested_sources: - evaluation = CallImportEvaluation( - call_import_id=call_import.id, - organization_id=organization_id, - # Mirror the parent CallImport's workspace so listings can - # filter on workspace_id directly without joining. - workspace_id=call_import.workspace_id, - name=_name_for_source(source), - selected_metric_ids=[ - str(metric_id) for metric_id in leaf_metric_ids - ], - selected_metric_groups=selected_metric_groups or None, - status="pending", - total_rows=total_row_count, - completed_rows=0, - failed_rows=0, - llm_provider=llm_provider_norm, - llm_model=llm_model_norm, - llm_credential_id=payload.llm_credential_id, - llm_config=payload.llm_config, - metric_llm_overrides=metric_overrides_payload, - stt_provider=stt_provider_norm, - stt_model=stt_model_norm, - stt_credential_id=( - payload.stt_credential_id if auto_transcribe else None - ), - diarisation_llm_provider=diarisation_llm_provider_norm, - diarisation_llm_model=diarisation_llm_model_norm, - diarisation_llm_credential_id=( - payload.diarization_llm_credential_id if auto_transcribe else None - ), - diarisation_prompt=diarisation_prompt_norm, - transcribe_mode=transcribe_mode_norm, - transcript_source=source, - discover_new_metrics=bool( - getattr(payload, "discover_new_metrics", False) - ), - ) - stamp_evaluation_actor(evaluation, principal, creating=True) - db.add(evaluation) - db.flush() - created_evaluations.append(evaluation) - - db.commit() - for evaluation in created_evaluations: - db.refresh(evaluation) - - primary_evaluation = created_evaluations[0] - sibling_ids = [e.id for e in created_evaluations[1:]] - - if not total_row_count and not starting_from_mapped: - for evaluation in created_evaluations: - evaluation.status = "completed" - db.commit() - for evaluation in created_evaluations: - db.refresh(evaluation) - return _serialize_eval( - db, primary_evaluation, sibling_evaluation_ids=sibling_ids - ) - - if starting_from_mapped: - from app.workers.tasks.call_import_bulk_ops import ( - materialize_mapped_call_import_evaluation_task, - ) - - for evaluation in created_evaluations: - materialize_mapped_call_import_evaluation_task.delay( - str(call_import.id), - str(organization_id), - str(call_import.workspace_id), - str(evaluation.id), - transcribe_overwrite=payload.transcribe_overwrite, - ) - else: - from app.workers.tasks.call_import_bulk_ops import ( - materialize_call_import_evaluation_task, - ) - - for evaluation in created_evaluations: - materialize_call_import_evaluation_task.delay( - str(evaluation.id), - transcribe_overwrite=payload.transcribe_overwrite, - ) - - for evaluation in created_evaluations: - db.refresh(evaluation) - - return _serialize_eval( - db, primary_evaluation, sibling_evaluation_ids=sibling_ids - ) - - -@router.get( - "", - response_model=CallImportEvaluationListResponse, - operation_id="listCallImportEvaluations", -) -async def list_call_import_evaluations( - call_import_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationListResponse: - del api_key - _require_import(db, call_import_id, organization_id) - rows = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .order_by(desc(CallImportEvaluation.created_at)) - .all() - ) - email_map = emails_for_user_ids(db, user_ids_from_evaluations(rows)) - return CallImportEvaluationListResponse( - items=[_serialize_eval(db, row, user_emails=email_map) for row in rows], - total=len(rows), - ) - - -@router.get( - "/{eval_id}", - response_model=CallImportEvaluationResponse, - operation_id="getCallImportEvaluation", -) -async def get_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationResponse: - del api_key - _require_import(db, call_import_id, organization_id) - row = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not row: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - return _serialize_eval(db, row) - - -@router.get( - "/{eval_id}/rows", - response_model=CallImportEvaluationRowListResponse, - operation_id="listCallImportEvaluationRows", -) -async def list_call_import_evaluation_rows( - call_import_id: UUID, - eval_id: UUID, - page: int = Query(1, ge=1), - page_size: int = Query(100, ge=1, le=500), - q: Optional[str] = Query( - None, - description=( - "Free-text search across conversation_id and transcript " - "(case-insensitive substring match)." - ), - ), - metric_id: Optional[UUID] = Query( - None, - description=( - "If set, only return rows whose ``metric_scores[metric_id].value`` " - "exactly matches ``metric_value`` (string-compared). " - "Use together with ``metric_value``." - ), - ), - metric_value: Optional[str] = Query( - None, - description="Value to match against metric_id (string compare).", - ), - status_filter: Optional[str] = Query( - None, - alias="status", - description="Restrict to rows with this evaluation row status.", - ), - flow_parent_id: Optional[UUID] = Query( - None, - description=( - "Parent (category) metric whose ``sequence`` array should be " - "checked against ``flow_node`` and ``flow_edge_target``. Used " - "to drill into the calls behind a flow-chart node or edge." - ), - ), - flow_node: Optional[str] = Query( - None, - description=( - "If set together with ``flow_parent_id``, only return rows " - "whose sequence under that parent contains this step. Accepts " - "either a child metric UUID (resolved to slug(name)), a " - "``disc:`` discovered-label id, or a raw slug." - ), - ), - flow_edge_target: Optional[str] = Query( - None, - description=( - "Optional companion to ``flow_node``: when set, restrict to " - "rows whose sequence contains the directed transition " - "``flow_node -> flow_edge_target`` (immediately adjacent). " - "Same id format as ``flow_node``." - ), - ), - discovered_parent_id: Optional[UUID] = Query( - None, - description=( - "Parent (category) metric that defines the discovery scope " - "for ``discovered_label_key`` / ``has_discovered``." - ), - ), - discovered_label_key: Optional[str] = Query( - None, - description=( - "If set together with ``discovered_parent_id``, only return " - "rows whose ``metric_scores[parent].discovered_labels`` " - "list contains an entry with this slug (after applying " - "evaluation-level merge aliases)." - ), - ), - has_discovered: Optional[bool] = Query( - None, - description=( - "If true together with ``discovered_parent_id``, only return " - "rows that have at least one LLM-discovered label for the " - "parent. Useful to triage which calls produced novel labels." - ), - ), - sort_by: Optional[str] = Query( - None, - description=( - "Column to sort by. Accepted values: ``row_index`` (default " - "when omitted), ``conversation_id``, ``status`` (the " - "evaluation-row status), or ``metric:`` to sort " - "by ``metric_scores[].value``. Metric sorts compare " - "the extracted JSON text — adequate for booleans, enum " - "labels, and 0-1 ratings; large integer values may sort " - "lexicographically (10 before 2)." - ), - ), - sort_dir: Optional[str] = Query( - "asc", - description="Sort direction: ``asc`` (default) or ``desc``.", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationRowListResponse: - del api_key - _require_import(db, call_import_id, organization_id) - - eval_row = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not eval_row: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - query = ( - db.query(CallImportEvaluationRow, CallImportRow) - .join(CallImportRow, CallImportRow.id == CallImportEvaluationRow.call_import_row_id) - .filter(CallImportEvaluationRow.evaluation_id == eval_id) - ) - - # --- Filters ---------------------------------------------------------- - if q and q.strip(): - needle = f"%{q.strip()}%" - # Search across both transcript columns so a hit in either the - # production or the diarised version surfaces the row, - # independent of which source the evaluation actually scored. - query = query.filter( - or_( - CallImportRow.conversation_id.ilike(needle), - CallImportRow.transcript.ilike(needle), - CallImportRow.diarised_transcript.ilike(needle), - ) - ) - - if status_filter: - # The CallImportEvaluationRow.status column is a string in PG so a - # plain == filter works; we lowercase to match the stored values. - query = query.filter( - CallImportEvaluationRow.status == status_filter.strip().lower() - ) - - if metric_id is not None and metric_value is not None: - # ``metric_scores`` is a JSONB column shaped like - # ``{"": {"value": , "type": "boolean", ...}}``. We - # extract the nested ``value`` as text and compare to the user - # input as a string — that handles bool/int/enum without needing - # per-type casts. ``metric_value`` is matched case-insensitively - # so chart clicks on labels like "True" survive any casing drift - # between worker output and the chart label. - path_value = func.json_extract_path_text( - CallImportEvaluationRow.metric_scores, - str(metric_id), - "value", - ) - query = query.filter(func.lower(path_value) == metric_value.strip().lower()) - - # --- Flow chart drilldown filter ------------------------------------- - # Translates a clicked node (or edge) on the flow chart into a - # SQL filter against ``metric_scores[].sequence``. The - # frontend sends either a child UUID, a ``disc:`` discovered - # node id, or a raw slug — we normalize all three to the slug that - # actually appears in stored ``sequence`` arrays. - if flow_parent_id is not None and flow_node and flow_node.strip(): - parent_id_str_local = str(flow_parent_id) - alias_map_flow = _alias_map_for_parent(eval_row, flow_parent_id) - - def _flow_node_to_slug(raw: str) -> Optional[str]: - raw_clean = raw.strip() - if not raw_clean: - return None - if raw_clean == _FLOW_START_NODE_ID: - # The synthetic START node isn't a real sequence entry; - # filtering on it is meaningless so we skip silently. - return None - if raw_clean.startswith(_DISCOVERED_NODE_PREFIX): - return _resolve_alias( - alias_map_flow, - _slug_label(raw_clean[len(_DISCOVERED_NODE_PREFIX) :]), - ) - # Try to interpret as a child metric UUID first; fall back - # to treating it as a slug. - try: - child_uuid = UUID(raw_clean) - except (TypeError, ValueError): - return _resolve_alias(alias_map_flow, _slug_label(raw_clean)) - child = ( - db.query(Metric.name) - .filter( - Metric.id == child_uuid, - Metric.organization_id == organization_id, - ) - .first() - ) - if child and child[0]: - return _resolve_alias(alias_map_flow, _slug_label(child[0])) - return _resolve_alias(alias_map_flow, _slug_label(raw_clean)) - - from_slug = _flow_node_to_slug(flow_node) - target_slug: Optional[str] = None - if flow_edge_target and flow_edge_target.strip(): - target_slug = _flow_node_to_slug(flow_edge_target) - - if from_slug: - # The ``metric_scores`` column is declared as ``Column(JSON)`` - # in the model so on databases where the table was created - # from the model (rather than the migration) the physical - # type is ``json``, not ``jsonb``. The JSONB-only operators - # below (``jsonb_exists``, ``jsonb_array_elements_text``, - # ``@>``) require a JSONB input — we cast once up front so - # the same SQL works regardless of which path created the - # table. - scores_jsonb = ( - "(call_import_evaluation_rows.metric_scores)::jsonb" - ) - if target_slug: - # Edge filter: rows whose sequence under this parent - # contains ``from_slug`` immediately followed by - # ``target_slug``. Implemented as a correlated EXISTS - # over ``jsonb_array_elements_text`` with ORDINALITY, - # which is the portable way to express "next array - # index" against a JSONB array in Postgres. - edge_filter_sql = text( - f""" - EXISTS ( - SELECT 1 - FROM jsonb_array_elements_text( - COALESCE( - {scores_jsonb} -> :p_id -> 'sequence', - '[]'::jsonb - ) - ) WITH ORDINALITY AS s1(elem, ord) - JOIN jsonb_array_elements_text( - COALESCE( - {scores_jsonb} -> :p_id -> 'sequence', - '[]'::jsonb - ) - ) WITH ORDINALITY AS s2(elem, ord) - ON s2.ord = s1.ord + 1 - WHERE s1.elem = :from_slug - AND s2.elem = :to_slug - ) - """ - ).bindparams( - p_id=parent_id_str_local, - from_slug=from_slug, - to_slug=target_slug, - ) - query = query.filter(edge_filter_sql) - else: - # Node filter: rows whose ``metric_scores -> parent -> - # 'sequence'`` array contains ``from_slug``. We use the - # function form ``jsonb_exists`` rather than the ``?`` - # operator to avoid psycopg2 mistaking the question - # mark for a parameter placeholder. - node_filter_sql = text( - f""" - jsonb_exists( - COALESCE( - {scores_jsonb} -> :p_id -> 'sequence', - '[]'::jsonb - ), - :slug - ) - """ - ).bindparams(p_id=parent_id_str_local, slug=from_slug) - query = query.filter(node_filter_sql) - - # --- Discovered label filters --------------------------------------- - # Surfaces "which calls produced THIS LLM-discovered label" and the - # broader "which calls produced ANY LLM-discovered label". Both - # operate on ``metric_scores[].discovered_labels`` (a list - # of dicts) plus the same ``sequence`` array — covering both legacy - # rows where the slug only made it into ``sequence`` and newer - # rows where it landed in both. - if discovered_parent_id is not None and ( - discovered_label_key or has_discovered - ): - d_parent_str = str(discovered_parent_id) - alias_map_disc = _alias_map_for_parent(eval_row, discovered_parent_id) - # See note above: cast once so the JSONB operators don't reject - # the column when it's typed as ``json`` in the database. - scores_jsonb = "(call_import_evaluation_rows.metric_scores)::jsonb" - if discovered_label_key and discovered_label_key.strip(): - target = _resolve_alias( - alias_map_disc, _slug_label(discovered_label_key) - ) - if target: - # Match rows whose discovered_labels list has an entry - # ``{"key": }`` OR whose sequence array still - # contains the slug. The latter covers older rows that - # were rewritten by a merge in the discovered_labels - # blob but whose sequence may have lagged. - contains_json = json.dumps( - {d_parent_str: {"discovered_labels": [{"key": target}]}} - ) - disc_filter_sql = text( - f""" - ( - {scores_jsonb} @> CAST(:contains AS JSONB) - OR - jsonb_exists( - COALESCE( - {scores_jsonb} -> :p_id -> 'sequence', - '[]'::jsonb - ), - :slug - ) - ) - """ - ).bindparams( - contains=contains_json, - p_id=d_parent_str, - slug=target, - ) - query = query.filter(disc_filter_sql) - elif has_discovered: - # No specific slug — just rows that surfaced any candidate - # under this parent. We coalesce missing paths to ``[]`` so - # ``jsonb_array_length`` always sees an array (it raises on - # non-array inputs, but our shape guarantees a list when - # the key is present). - has_disc_sql = text( - f""" - jsonb_array_length( - COALESCE( - {scores_jsonb} -> :p_id -> 'discovered_labels', - '[]'::jsonb - ) - ) > 0 - """ - ).bindparams(p_id=d_parent_str) - query = query.filter(has_disc_sql) - - # --- Sorting ---------------------------------------------------------- - # Column-click sorting from the UI. Falls back to ``row_index`` so - # paging stays stable when the user clears the sort. We always add a - # secondary ``row_index`` tiebreaker so duplicate sort keys (e.g. - # many rows with ``status = 'completed'``) keep a deterministic - # order across page boundaries — without this, pagination can - # double-show or skip rows when Postgres picks a different physical - # order on each query. - direction_desc = (sort_dir or "asc").strip().lower() == "desc" - - def _apply_direction(column_expr): - return column_expr.desc() if direction_desc else column_expr.asc() - - # Whether the caller's ``sort_by`` resolved to a known column. We - # use this flag to decide whether ``sort_dir`` is honoured on the - # fallback path: unrecognized columns (typos, stale UI state) fall - # back to the implicit ``row_index ASC`` default and intentionally - # ignore ``sort_dir`` so users don't get a surprise reverse order - # from a typo'd column name. - sort_recognized = False - sort_by_clean = (sort_by or "").strip() - primary_sort = None - metric_uuid: Optional[UUID] = None - if sort_by_clean == "row_index": - sort_recognized = True - # Falls through to the default ``order_by`` below with - # ``primary_sort`` still None — but ``sort_recognized=True`` - # tells the fallback branch to apply the requested direction. - elif sort_by_clean == "conversation_id": - sort_recognized = True - primary_sort = _apply_direction(CallImportRow.conversation_id) - elif sort_by_clean == "status": - sort_recognized = True - primary_sort = _apply_direction(CallImportEvaluationRow.status) - elif sort_by_clean.startswith("metric:"): - raw_metric_id = sort_by_clean.split(":", 1)[1].strip() - try: - metric_uuid = UUID(raw_metric_id) - except (TypeError, ValueError): - metric_uuid = None - if metric_uuid is not None: - sort_recognized = True - # ``metric_scores`` is JSON-typed but the helper functions - # for path extraction differ between Postgres (production) - # and SQLite (default test backend). Branch on the active - # dialect so we can use the right primitive: - # * Postgres → ``json_extract_path_text(col, key, "value")`` - # which returns the value as TEXT for both ``json`` and - # ``jsonb`` columns. - # * SQLite → ``json_extract(col, '$."".value')`` - # using JSONPath syntax. ``metric_uuid`` is already - # validated above (``UUID(raw_metric_id)``), so the - # interpolated path is safe from injection. - # NULL values (rows where the metric wasn't scored) sort - # to the END regardless of direction so un-scored rows - # don't crowd the top of an ascending sort. - dialect_name = ( - db.bind.dialect.name if db.bind is not None else "postgresql" - ) - if dialect_name == "sqlite": - json_path = f'$."{metric_uuid}".value' - path_value = func.json_extract( - CallImportEvaluationRow.metric_scores, - json_path, - ) - else: - path_value = func.json_extract_path_text( - CallImportEvaluationRow.metric_scores, - str(metric_uuid), - "value", - ) - primary_sort = ( - path_value.desc().nullslast() - if direction_desc - else path_value.asc().nullslast() - ) - - if primary_sort is not None: - query = query.order_by(primary_sort, CallImportRow.row_index.asc()) - elif sort_recognized: - # Explicit ``sort_by=row_index`` request — honour direction. - query = query.order_by(_apply_direction(CallImportRow.row_index)) - else: - # No sort requested OR unrecognized column — safe default of - # ``row_index ASC``. We deliberately ignore ``sort_dir`` here - # so a typo'd / stale ``sort_by`` doesn't quietly invert the - # default order. - query = query.order_by(CallImportRow.row_index.asc()) - from app.db_sharding.eval_rows import fetch_evaluation_row_pairs_page - from app.db_sharding.sessions import is_sharding_enabled - - def _pair_row_index( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> int: - return int(pair[1].row_index or 0) - - def _directed_string(value: Optional[str], desc: bool) -> Tuple[int, ...]: - text = value or "" - if not desc: - return (0, *text.encode("utf-8")) - return (1, *(-byte for byte in text.encode("utf-8"))) - - if sort_by_clean == "conversation_id": - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[Any, ...]: - return ( - _directed_string(pair[1].conversation_id, direction_desc), - _pair_row_index(pair), - ) - elif sort_by_clean == "status": - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[Any, ...]: - return ( - _directed_string(pair[0].status, direction_desc), - _pair_row_index(pair), - ) - elif sort_by_clean.startswith("metric:") and metric_uuid is not None: - metric_id_str = str(metric_uuid) - - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[Any, ...]: - scores = pair[0].metric_scores or {} - entry = scores.get(metric_id_str, {}) - raw_value = entry.get("value") if isinstance(entry, dict) else None - null_rank = 1 if raw_value is None else 0 - return ( - null_rank, - _directed_string( - str(raw_value) if raw_value is not None else None, - direction_desc, - ), - _pair_row_index(pair), - ) - elif sort_recognized and sort_by_clean == "row_index": - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[int, ...]: - idx = _pair_row_index(pair) - return (-idx,) if direction_desc else (idx,) - else: - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[int, ...]: - return (_pair_row_index(pair),) - - if is_sharding_enabled(): - def _build_query(session: Session): - return query.with_session(session) - - total, rows = fetch_evaluation_row_pairs_page( - db, - _build_query, - page=page, - page_size=page_size, - sort_key=_pair_sort_key, - bounded_shard_fetch=( - not sort_recognized or sort_by_clean == "row_index" - ), - ) - else: - total = query.count() - rows = query.offset((page - 1) * page_size).limit(page_size).all() - - # Row detail shows the transcript for this run's chosen source. - items: List[CallImportEvaluationRowResponse] = [ - _to_evaluation_row_response(eval_row_obj, source_row, eval_row) - for eval_row_obj, source_row in rows - ] - - return CallImportEvaluationRowListResponse( - items=items, - total=total, - page=page, - page_size=page_size, - ) - - -@router.get( - "/{eval_id}/export", - operation_id="exportCallImportEvaluationCsv", -) -async def export_call_import_evaluation_csv( - call_import_id: UUID, - eval_id: UUID, - format: Literal["csv", "xlsx"] = Query( - "csv", - description=( - "Output format. ``csv`` returns a UTF-8 BOM CSV; ``xlsx`` " - "returns a native Excel workbook (single sheet)." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> StreamingResponse: - del api_key - call_import = _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - selected_metric_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) - # Include parent metric ids referenced in selected_metric_groups so - # the export shows a parent "Chosen Label" column next to its - # children's true/false columns. - lookup_ids: List[UUID] = list(selected_metric_ids) - groups_raw = ( - evaluation.selected_metric_groups - if isinstance(evaluation.selected_metric_groups, dict) - else {} - ) - for parent_str in groups_raw.keys(): - try: - pid = UUID(parent_str) - if pid not in lookup_ids: - lookup_ids.append(pid) - except (TypeError, ValueError): - continue - metrics = _metrics_for_ids(db, organization_id, lookup_ids) - metric_names = {str(metric.id): metric.name for metric in metrics} - metrics_by_id = {str(metric.id): metric for metric in metrics} - - # Two export-time modes depending on how the batch was uploaded: - # - # * Schema-driven (new): ``call_imports.schema_id`` is set, - # ``parameter_mapping`` records which CSV header fed each - # parameter, and ``raw_columns`` on each row is keyed by - # parameter NAME. Export headers are the parameter names. - # * Legacy (pre-schema): ``column_mapping`` / ``extra_columns`` / - # ``custom_column_mapping`` drive the columns and - # ``raw_columns`` is keyed by the original CSV header. - # - # We bucket entries into ``standard_export_headers`` (raw_columns - # key == export header) and ``custom_export`` (export header - # differs from the raw_columns key) so the row-projection loop - # below stays mode-agnostic. - standard_export_headers: List[str] = [] - custom_export: List[tuple[str, str]] = [] # [(export_header, raw_columns_key)] - - if call_import.schema_id is not None: - # Use the live schema parameter list for column ordering. Falls - # back to whatever's in ``parameter_mapping`` if the schema was - # deleted (defensive - the FK is ON DELETE RESTRICT, but tests - # / future cascades may still hit this branch). - from app.models.database import CallImportSchema as _ImportSchema - - schema_obj = ( - db.query(_ImportSchema) - .filter(_ImportSchema.id == call_import.schema_id) - .first() - ) - if schema_obj is not None: - params_sorted = sorted( - schema_obj.parameters, key=lambda p: p.ordering or 0 - ) - for param in params_sorted: - if param.name and param.name not in standard_export_headers: - standard_export_headers.append(param.name) - else: - for param_name in (call_import.parameter_mapping or {}).keys(): - if param_name and param_name not in standard_export_headers: - standard_export_headers.append(param_name) - else: - mapping = call_import.column_mapping or {} - mapped_headers = [ - mapping.get("external_call_id"), - mapping.get("transcript"), - mapping.get("recording_url"), - ] - for header in [*mapped_headers, *(call_import.extra_columns or [])]: - if ( - isinstance(header, str) - and header - and header not in standard_export_headers - ): - standard_export_headers.append(header) - - custom_mapping = call_import.custom_column_mapping or {} - if isinstance(custom_mapping, dict): - for name, csv_header in custom_mapping.items(): - if not isinstance(name, str) or not isinstance(csv_header, str): - continue - if not name or not csv_header: - continue - if name in standard_export_headers: - continue # would clobber a real column - custom_export.append((name, csv_header)) - - if ( - call_import.source_format == "audio" - and "conversation_id" not in standard_export_headers - ): - standard_export_headers.insert(0, "conversation_id") - - # Build the metric columns: each parent (if any) gets a value column - # and (when capture_rationale=true) a " - LLM Rationale" - # column. The per-child boolean columns are intentionally suppressed - # — categorization metrics now collapse to exactly two columns in - # the export, mirroring the in-app table. - child_ids_in_groups: set[str] = set() - for parent_str, child_strs in groups_raw.items(): - for child_str in child_strs: - if isinstance(child_str, str): - child_ids_in_groups.add(child_str) - - metric_headers: List[str] = [] - rationale_headers: Dict[str, str] = {} # metric_id_str -> rationale column name - seen_metric_ids: set[str] = set() - - def _add_metric_column(metric: Metric) -> None: - mid_str = str(metric.id) - if mid_str in seen_metric_ids: - return - # Skip any child whose parent is part of this run — the parent - # column above already shows the chosen child name as its - # value. - if mid_str in child_ids_in_groups: - return - seen_metric_ids.add(mid_str) - header = metric_names[mid_str] - metric_headers.append(header) - if bool(getattr(metric, "capture_rationale", False)): - rationale_header = f"{header} - LLM Rationale" - metric_headers.append(rationale_header) - rationale_headers[mid_str] = rationale_header - - for parent_str in groups_raw.keys(): - parent = metrics_by_id.get(parent_str) - if parent: - _add_metric_column(parent) - # Children of an in-run parent are deliberately not emitted — - # the ``child_ids_in_groups`` guard inside ``_add_metric_column`` - # is what enforces this. We still iterate the keys above (not - # ``.items()``) so the parent-only emission is explicit. - # Append anything left over (standalone metrics not in any group, or - # legacy runs without ``selected_metric_groups``). - for metric in metrics: - if metric.selection_mode and not metric.parent_metric_id: - continue # already handled above - if str(metric.id) in seen_metric_ids: - continue - _add_metric_column(metric) - - # Three new fixed columns surface the two transcript fields and the - # evaluation's transcript_source as live values pulled from the - # ``CallImportRow`` (not from the frozen ``raw_columns`` snapshot). - # The user can now compare "what was in the CSV" vs "what the - # diarisation worker produced" without round-tripping through the - # UI, and downstream tools can verify which transcript the metrics - # were computed against. - PRODUCTION_TRANSCRIPT_HEADER = "Production Transcript" - DIARISED_TRANSCRIPT_HEADER = "Diarised Transcript" - EVAL_SOURCE_HEADER = "Evaluated Transcript Source" - - fieldnames = [ - *standard_export_headers, - *[h for h, _ in custom_export], - PRODUCTION_TRANSCRIPT_HEADER, - DIARISED_TRANSCRIPT_HEADER, - EVAL_SOURCE_HEADER, - *metric_headers, - ] - - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - rows = sorted( - load_evaluation_row_pairs(db, eval_id), - key=lambda pair: int(pair[1].row_index or 0), - ) - else: - rows = ( - db.query(CallImportEvaluationRow, CallImportRow) - .join( - CallImportRow, - CallImportRow.id == CallImportEvaluationRow.call_import_row_id, - ) - .filter(CallImportEvaluationRow.evaluation_id == eval_id) - .order_by(CallImportRow.row_index.asc()) - .all() - ) - - def _project_rows() -> Iterator[Dict[str, str]]: - for eval_row, source_row in rows: - row_out: Dict[str, str] = {} - raw = ( - source_row.raw_columns - if isinstance(source_row.raw_columns, dict) - else {} - ) - for header in standard_export_headers: - value = raw.get(header) - if value is None and header == "conversation_id": - value = source_row.conversation_id - row_out[header] = "" if value is None else str(value) - for export_header, csv_header in custom_export: - value = raw.get(csv_header) - row_out[export_header] = "" if value is None else str(value) - - # Live transcripts pulled from the row, NOT from raw_columns, - # so re-diarised values are always reflected in the export. - # Both transcript columns are flattened to a single line so the - # spreadsheet cell doesn't balloon vertically — the in-app - # ``TranscriptView`` still has the DB copy with line breaks - # intact for chat-bubble rendering. - row_out[PRODUCTION_TRANSCRIPT_HEADER] = _flatten_transcript( - source_row.transcript - ) - row_out[DIARISED_TRANSCRIPT_HEADER] = _flatten_transcript( - source_row.diarised_transcript - ) - row_out[EVAL_SOURCE_HEADER] = _evaluated_transcript_source_label( - evaluation, - source_row, - ) - - scores = ( - eval_row.metric_scores - if isinstance(eval_row.metric_scores, dict) - else {} - ) - for metric in metrics: - metric_score = ( - scores.get(str(metric.id)) - if isinstance(scores, dict) - else None - ) - value = ( - metric_score.get("value") - if isinstance(metric_score, dict) - else None - ) - # Parent metrics (selection_mode set) render the chosen - # child name for single_choice or the ";"-joined list of - # true child names for multi_label. - if ( - metric.selection_mode - and not metric.parent_metric_id - and isinstance(metric_score, dict) - ): - if metric.selection_mode == "multi_label": - selected = metric_score.get("selected_child_names") - if isinstance(selected, list): - value = ";".join(str(s) for s in selected) - else: - value = ( - metric_score.get("chosen_child_name") - or metric_score.get("value") - ) - row_out[metric.name] = "" if value is None else str(value) - rationale_header = rationale_headers.get(str(metric.id)) - if rationale_header is not None: - rationale = ( - metric_score.get("rationale") - if isinstance(metric_score, dict) - else None - ) - row_out[rationale_header] = ( - "" if rationale is None else str(rationale) - ) - yield row_out - - base_filename = f"call-import-{call_import_id}-evaluation-{eval_id}" - - if format == "xlsx": - # xlsx is unicode-native (Hindi/Devanagari, emoji, etc.) so the - # UTF-8-BOM dance isn't needed here. ``write_only`` mode keeps - # peak memory bounded for large evaluations because openpyxl - # only buffers the current row. - try: - from openpyxl import Workbook # type: ignore - from openpyxl.cell import WriteOnlyCell # type: ignore - from openpyxl.styles import Font # type: ignore - except ImportError as exc: # pragma: no cover - exercised by pyproject lock - raise HTTPException( - status_code=500, - detail=( - "Excel export requires the 'openpyxl' package which is " - "not installed." - ), - ) from exc - - workbook = Workbook(write_only=True) - worksheet = workbook.create_sheet(title="Evaluation") - - bold_font = Font(bold=True) - header_cells = [] - for header in fieldnames: - cell = WriteOnlyCell(worksheet, value=header) - cell.font = bold_font - header_cells.append(cell) - worksheet.append(header_cells) - - for row_dict in _project_rows(): - worksheet.append([row_dict.get(h, "") for h in fieldnames]) - - buffer = io.BytesIO() - workbook.save(buffer) - xlsx_bytes = buffer.getvalue() - filename = f"{base_filename}.xlsx" - return StreamingResponse( - iter([xlsx_bytes]), - media_type=( - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ), - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, - ) - - output = io.StringIO() - writer = csv.DictWriter(output, fieldnames=fieldnames, extrasaction="ignore") - writer.writeheader() - for row_dict in _project_rows(): - writer.writerow(row_dict) - - # Excel on Windows defaults to the system ANSI codepage (Windows-1252) - # when a CSV has no encoding marker, which turns UTF-8 Hindi/Devanagari - # / any non-ASCII text into mojibake (e.g. ``ठीक`` → ``ठीक``). - # A UTF-8 BOM tells Excel to switch to UTF-8 decoding and is silently - # skipped by every other UTF-8-aware reader (pandas, LibreOffice, - # Google Sheets, etc.), so the data round-trips correctly everywhere. - csv_text = output.getvalue() - # ``utf-8-sig`` adds the UTF-8 BOM so Excel on Windows decodes the file - # as UTF-8 instead of the system codepage. We also declare the same - # codec in the Content-Type header so well-behaved HTTP clients (incl. - # ``httpx`` / ``requests`` in our tests) strip the BOM during decode. - csv_bytes = csv_text.encode("utf-8-sig") - filename = f"{base_filename}.csv" - return StreamingResponse( - iter([csv_bytes]), - media_type="text/csv; charset=utf-8-sig", - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, - ) - - -def _report_filename_slug(value: str) -> str: - slug = re.sub(r"[^a-zA-Z0-9]+", "-", value.strip().lower()).strip("-") - return slug or "client" - - -def _pdf_report_actor(principal: Principal) -> tuple[Optional[str], Optional[UUID]]: - created_by = principal.email - if not created_by and principal.user_id: - created_by = str(principal.user_id) - return created_by, principal.user_id - - -def _pdf_report_response_from_row( - row: CallImportEvaluationPdfReport, - *, - cache_hit: bool = False, -) -> CallImportEvaluationPdfReportResponse: - filename = row.filename or "report.pdf" - preview_url, download_url = presigned_urls_for_pdf_report( - row.s3_key or "", - filename, - ) - return CallImportEvaluationPdfReportResponse( - id=str(row.id), - filename=filename, - preview_url=preview_url, - download_url=download_url, - created_at=row.created_at or datetime.now(timezone.utc), - created_by=row.created_by, - report_type=row.report_type, - vendor_name=row.vendor_name, - config_summary=config_summary_from_report_config( - row.report_config if isinstance(row.report_config, dict) else {} - ), - storage_available=bool(row.s3_key), - cache_hit=cache_hit, - ) - - -def _pdf_report_list_item_from_row( - row: CallImportEvaluationPdfReport, -) -> CallImportEvaluationPdfReportListItem: - return CallImportEvaluationPdfReportListItem( - id=str(row.id), - filename=row.filename, - vendor_name=row.vendor_name, - report_type=row.report_type, - created_by=row.created_by, - created_at=row.created_at or datetime.now(timezone.utc), - config_summary=config_summary_from_report_config( - row.report_config if isinstance(row.report_config, dict) else {} - ), - cache_fingerprint=row.cache_fingerprint, - ) - - -def _report_branding_for_import_workspace( - db: Session, - organization_id: UUID, - workspace_id: UUID, - *, - internal_brand_image_id: Optional[str] = None, - external_brand_image_id: Optional[str] = None, -) -> tuple[dict[str, str] | list[str], Optional[str]]: - workspace = ( - db.query(Workspace) - .filter( - Workspace.id == workspace_id, - Workspace.organization_id == organization_id, - ) - .first() - ) - raw = workspace.report_branding if workspace and isinstance(workspace.report_branding, dict) else {} - images = raw.get("images") if isinstance(raw.get("images"), list) else [] - loaded_images: list[dict[str, str]] = [] - for item in images: - if not isinstance(item, dict) or not item.get("s3_key"): - continue - content_type = str(item.get("content_type") or "image/png") - try: - from app.services.storage.s3_service import s3_service - - image_bytes = s3_service.download_file_by_key(str(item["s3_key"])) - except Exception as exc: # noqa: BLE001 - logger.warning( - "Unable to load report branding image for workspace {}: {}", - workspace_id, - exc, - ) - continue - encoded = base64.b64encode(image_bytes).decode("ascii") - role = str(item.get("role") or "generic") - if role not in {"internal", "external", "generic"}: - role = "generic" - loaded_images.append( - { - "id": str(item.get("id") or ""), - "role": role, - "data_uri": f"data:{content_type};base64,{encoded}", - } - ) - - def _pick(role: str, selected_id: Optional[str]) -> Optional[str]: - if selected_id: - for loaded in loaded_images: - if loaded["id"] == selected_id: - return loaded["data_uri"] - for loaded in loaded_images: - if loaded["role"] == role: - return loaded["data_uri"] - return None - - logo_data_uris: dict[str, str] = {} - internal_uri = _pick("internal", internal_brand_image_id) - external_uri = _pick("external", external_brand_image_id) - if internal_uri: - logo_data_uris["internal"] = internal_uri - if external_uri: - logo_data_uris["external"] = external_uri - if ( - not logo_data_uris - and not internal_brand_image_id - and not external_brand_image_id - ): - # Backward compatibility for workspaces that only had a generic logo - # library before the two-slot report header existed. - generic_uris = [ - loaded["data_uri"] - for loaded in loaded_images - if loaded.get("data_uri") - ] - if generic_uris: - heading = raw.get("heading") if isinstance(raw.get("heading"), str) else None - return generic_uris[:4], heading - heading = raw.get("heading") if isinstance(raw.get("heading"), str) else None - return logo_data_uris, heading - - -def _display_metrics_for_pdf_report( - db: Session, - organization_id: UUID, - evaluation: CallImportEvaluation, -) -> list[Metric]: - selected_metric_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) - lookup_ids: List[UUID] = list(selected_metric_ids) - groups_raw = ( - evaluation.selected_metric_groups - if isinstance(evaluation.selected_metric_groups, dict) - else {} - ) - for parent_str in groups_raw.keys(): - try: - parent_id = UUID(parent_str) - except (TypeError, ValueError): - continue - if parent_id not in lookup_ids: - lookup_ids.append(parent_id) - - metrics = _metrics_for_ids(db, organization_id, lookup_ids) - child_ids_in_groups: set[str] = set() - for child_strs in groups_raw.values(): - if not isinstance(child_strs, list): - continue - child_ids_in_groups.update(str(child_id) for child_id in child_strs) - - metrics_by_id = {str(metric.id): metric for metric in metrics} - display: list[Metric] = [] - seen: set[str] = set() - - for parent_str in groups_raw.keys(): - parent = metrics_by_id.get(str(parent_str)) - if parent and str(parent.id) not in seen: - display.append(parent) - seen.add(str(parent.id)) - - for metric in metrics: - metric_id = str(metric.id) - if metric_id in seen or metric_id in child_ids_in_groups: - continue - if metric.selection_mode and not metric.parent_metric_id: - continue - display.append(metric) - seen.add(metric_id) - - return display - - -def _metrics_for_clustering( - db: Session, - evaluation: CallImportEvaluation, - eval_rows: List[CallImportEvaluationRow], -) -> List[Metric]: - """All enabled quality metrics scored in this run, normalized for clustering. - - Hierarchical children are collapsed to their parent metric so cluster - groups render at the category level (e.g. ``AI reveal``) instead of the - child label level (e.g. ``Yes`` / ``No``). - """ - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - aggregate_metric_ids: List[UUID] = [] - for agg in aggregates: - if (agg.metric_category or "quality") == "user_insight": - continue - try: - aggregate_metric_ids.append(UUID(agg.metric_id)) - except (TypeError, ValueError): - continue - if not aggregate_metric_ids: - return [] - - aggregate_metrics = _metrics_for_ids( - db, evaluation.organization_id, aggregate_metric_ids - ) - by_id = {metric.id: metric for metric in aggregate_metrics} - - normalized_ids: List[UUID] = [] - seen: set[UUID] = set() - for metric_id in aggregate_metric_ids: - metric = by_id.get(metric_id) - target_id = ( - metric.parent_metric_id - if metric is not None and metric.parent_metric_id - else metric_id - ) - if target_id in seen: - continue - seen.add(target_id) - normalized_ids.append(target_id) - - metrics = _metrics_for_ids(db, evaluation.organization_id, normalized_ids) - return [ - metric - for metric in metrics - if getattr(metric, "enabled", True) and not _metric_is_user_insight(metric) - ] - - -def _metric_is_user_insight(metric: Metric) -> bool: - if (getattr(metric, "metric_category", "quality") or "quality") == "user_insight": - return True - text_value = " ".join( - str(part or "").lower() - for part in (getattr(metric, "name", ""), getattr(metric, "description", "")) - ) - normalized = text_value.replace("-", " ").replace("_", " ") - phrases = ( - "call context", - "caller context", - "product identification", - "out of scope", - "identity match", - "user identity", - "caller identity", - "frustration trigger", - "video call offer", - "video call reception", - ) - return any(phrase in normalized for phrase in phrases) - - -def _evaluation_rows_for_period( - db: Session, - evaluation_id: UUID, -) -> list[tuple[CallImportEvaluationRow, CallImportRow]]: - return ( - db.query(CallImportEvaluationRow, CallImportRow) - .join(CallImportRow, CallImportRow.id == CallImportEvaluationRow.call_import_row_id) - .filter(CallImportEvaluationRow.evaluation_id == evaluation_id) - .order_by(CallImportRow.row_index.asc()) - .all() - ) - - -def _baseline_candidate_evaluations( - db: Session, - organization_id: UUID, - workspace_id: UUID, - current_evaluation: CallImportEvaluation, - current_period_start: Optional[date], - *, - limit: int = 20, -) -> list[dict[str, Any]]: - candidates = ( - db.query(CallImportEvaluation, CallImport) - .join(CallImport, CallImport.id == CallImportEvaluation.call_import_id) - .filter( - CallImportEvaluation.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - CallImportEvaluation.id != current_evaluation.id, - CallImportEvaluation.status == "completed", - CallImportEvaluation.completed_rows > 0, - ) - .order_by(desc(CallImportEvaluation.created_at)) - .limit(limit * 3) - .all() - ) - items: list[dict[str, Any]] = [] - for candidate_eval, candidate_import in candidates: - rows = _evaluation_rows_for_period(db, candidate_eval.id) - period_start, period_end, period_label, period_display = _report_period_from_rows(rows) - if current_period_start and period_start and period_start >= current_period_start: - continue - dataset = ( - (candidate_import.dataset or "").strip() - or (candidate_import.original_filename or candidate_import.filename or "").strip() - or "Unknown dataset" - ) - evaluation_name = ( - (candidate_eval.name or "").strip() - or str(candidate_eval.id)[:8] - ) - items.append( - { - "evaluation_id": str(candidate_eval.id), - "name": evaluation_name, - "dataset": dataset, - "period_label": period_label, - "period_start": period_start, - "period_end": period_end, - "period_display": period_display, - "completed_rows": int(candidate_eval.completed_rows or 0), - "created_at": candidate_eval.created_at, - "is_default": False, - } - ) - if len(items) >= limit: - break - items.sort( - key=lambda item: ( - item["period_start"] or date.min, - item["created_at"] or datetime.min.replace(tzinfo=timezone.utc), - ), - reverse=True, - ) - if items: - items[0]["is_default"] = True - return items - - -def _resolve_baseline_evaluation( - db: Session, - organization_id: UUID, - workspace_id: UUID, - current_evaluation: CallImportEvaluation, - current_period_start: Optional[date], - baseline_evaluation_id: Optional[str], -) -> Optional[CallImportEvaluation]: - candidates = _baseline_candidate_evaluations( - db, - organization_id, - workspace_id, - current_evaluation, - current_period_start, - ) - allowed_ids = {item["evaluation_id"] for item in candidates} - if baseline_evaluation_id: - baseline_id = str(baseline_evaluation_id).strip() - if baseline_id not in allowed_ids: - raise HTTPException( - status_code=400, - detail="Selected baseline evaluation is not a valid prior run for this report.", - ) - return ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == UUID(baseline_id), - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not candidates: - return None - default_id = candidates[0]["evaluation_id"] - return ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == UUID(default_id), - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - - -def _benchmark_context_for_evaluation( - db: Session, - baseline_evaluation: Optional[CallImportEvaluation], -) -> Optional[dict[str, str]]: - if baseline_evaluation is None: - return None - baseline_import = ( - db.query(CallImport) - .filter(CallImport.id == baseline_evaluation.call_import_id) - .first() - ) - rows = _evaluation_rows_for_period(db, baseline_evaluation.id) - period_start, _period_end, period_label, _period_display = _report_period_from_rows(rows) - dataset = ( - (baseline_import.dataset or "").strip() - if baseline_import and baseline_import.dataset - else None - ) - filename = ( - (baseline_import.original_filename or baseline_import.filename or "").strip() - if baseline_import - else None - ) - evaluation_label = ( - (baseline_evaluation.name or "").strip() - if baseline_evaluation.name - else str(baseline_evaluation.id)[:8] - ) - period = period_label or ( - period_start.isoformat() if period_start else "previous report" - ) - return { - "dataset": dataset or filename or "Unknown dataset", - "evaluation": evaluation_label, - "evaluation_id": str(baseline_evaluation.id), - "period": period, - } - - -def _period_deltas_from_evaluation( - db: Session, - baseline_evaluation: CallImportEvaluation, - current_metric_aggregates: list[dict[str, Any]], - current_evaluation: CallImportEvaluation, - current_eval_rows: List[CallImportEvaluationRow], -) -> dict[str, dict[str, str]]: - baseline_rows = _evaluation_rows_for_period(db, baseline_evaluation.id) - baseline_eval_rows = [eval_row for eval_row, _source_row in baseline_rows] - baseline_aggregate_models = _compute_metric_aggregates( - db, - baseline_evaluation, - baseline_eval_rows, - ) - baseline_metric_aggregates = [ - _aggregate_to_dict(aggregate) for aggregate in baseline_aggregate_models - ] - _metrics, _aggs, policies, _source, _child_map = _clustering_context( - db, current_evaluation, current_eval_rows - ) - metric_by_id = {str(m.id): m for m in _metrics} - current_by_id = { - str(item.get("metric_id")): item for item in current_metric_aggregates - } - previous_by_id = { - str(item.get("metric_id")): item - for item in baseline_metric_aggregates - if isinstance(item, dict) - } - deltas: dict[str, dict[str, str]] = {} - for metric_id, current in current_by_id.items(): - metric = metric_by_id.get(metric_id) - policy = policies.get(metric_id) - previous_raw = previous_by_id.get(metric_id) - if metric is None or policy is None: - deltas[metric_id] = { - "label": "No previous-week baseline", - "detail": "No comparable prior report snapshot was found.", - } - continue - current_pct = failure_rate_percent_from_rows( - current_eval_rows, metric, policy - ) - previous_pct = failure_rate_percent_from_rows( - baseline_eval_rows, metric, policy - ) - if current_pct is None or previous_pct is None: - current_pct = current_pct or _aggregate_primary_percent(current, policy) - previous_pct = ( - previous_pct or _aggregate_primary_percent(previous_raw, policy) - if previous_raw - else None - ) - if current_pct is None or previous_pct is None: - deltas[metric_id] = { - "label": "No previous-week baseline", - "detail": "No comparable prior report snapshot was found.", - } - continue - delta = current_pct - previous_pct - sign = "+" if delta >= 0 else "" - deltas[metric_id] = { - "label": f"{sign}{delta:.1f} pp", - "detail": ( - f"Current report {current_pct:.1f}% vs previous report " - f"{previous_pct:.1f}%" - ), - } - return deltas - - -_DELTA_EXPLANATION_SYSTEM_PROMPT = ( - "You are a senior conversation-analytics reviewer. You will receive " - "week-over-week metric failure-rate deltas plus reconciled failure " - "cluster context per metric.\n\n" - "Return STRICT JSON only:\n" - "{\n" - ' "explanations": {"": "<1-2 sentence explanation of why the delta likely occurred>"}\n' - "}\n\n" - "Constraints:\n" - "- Only include metrics supplied in the prompt.\n" - "- Cluster labels are generated independently each run and are NOT stable " - "IDs. Never compare an unmatched current label to 0% baseline.\n" - "- Use matched_theme_shifts for label-aligned comparisons, " - "gap_label_shifts for structural shifts, and new_themes_current_period " - "for themes that emerged without a baseline match.\n" - "- If reconciliation is uncertain, explain using the numeric delta and " - "gap_label_shifts only.\n" - "- Keep each explanation to 1-2 short sentences (~220 chars).\n" - "- Vendor-safe, factual language; no markdown." -) - - -def _period_delta_explanation_cache_key( - baseline_evaluation_id: UUID, - *, - completed_rows: int, - baseline_completed_rows: int, -) -> str: - return ( - f"{baseline_evaluation_id}:{completed_rows}:" - f"{baseline_completed_rows}:reconciled-v2" - ) - - -def _normalize_cluster_label(label: str) -> str: - return re.sub(r"[^a-z0-9]+", " ", (label or "").lower()).strip() - - -_CLUSTER_LABEL_STOPWORDS = frozenset( - { - "a", - "an", - "the", - "and", - "or", - "during", - "while", - "with", - "for", - "from", - "into", - "general", - "user", - "bot", - "agent", - } -) - - -def _cluster_label_tokens(label: str) -> set[str]: - return { - token - for token in _normalize_cluster_label(label).split() - if token and token not in _CLUSTER_LABEL_STOPWORDS and len(token) > 2 - } - - -def _cluster_label_similarity(left: str, right: str) -> float: - tokens_left = _cluster_label_tokens(left) - tokens_right = _cluster_label_tokens(right) - if not tokens_left or not tokens_right: - return 0.0 - intersection = tokens_left & tokens_right - if not intersection: - return 0.0 - union = tokens_left | tokens_right - jaccard = len(intersection) / len(union) - smaller = tokens_left if len(tokens_left) <= len(tokens_right) else tokens_right - overlap_ratio = len(intersection) / len(smaller) - return max(jaccard, overlap_ratio * 0.85) - - -def _group_clusters_by_gap_label( - clusters: list[dict[str, Any]], -) -> dict[str, list[dict[str, Any]]]: - grouped: dict[str, list[dict[str, Any]]] = {} - for cluster in clusters: - gap_label = str(cluster.get("gap_label") or "UNKNOWN") - grouped.setdefault(gap_label, []).append(cluster) - return grouped - - -def _append_matched_cluster_pair( - matched: list[dict[str, Any]], - current: dict[str, Any], - baseline: dict[str, Any], - *, - match_confidence: float, - match_method: str, -) -> None: - matched.append( - { - "current_label": current.get("label"), - "baseline_label": baseline.get("label"), - "gap_label": current.get("gap_label") or baseline.get("gap_label"), - "current_share_pct": current.get("share_pct"), - "baseline_share_pct": baseline.get("share_pct"), - "share_delta_pp": round( - float(current.get("share_pct") or 0.0) - - float(baseline.get("share_pct") or 0.0), - 1, - ), - "match_confidence": round(match_confidence, 2), - "match_method": match_method, - } - ) - - -def _aggregate_share_by_gap_label( - clusters: list[dict[str, Any]], -) -> dict[str, float]: - totals: dict[str, float] = {} - for cluster in clusters: - gap_label = str(cluster.get("gap_label") or "UNKNOWN") - totals[gap_label] = totals.get(gap_label, 0.0) + float( - cluster.get("share_pct") or 0.0 - ) - return {gap: round(share, 1) for gap, share in totals.items()} - - -def _reconcile_cluster_periods( - current_clusters: list[dict[str, Any]], - baseline_clusters: list[dict[str, Any]], - *, - similarity_threshold: float = 0.35, -) -> dict[str, Any]: - """Align independently-generated cluster labels before delta explanation.""" - matched: list[dict[str, Any]] = [] - current_unmatched = list(current_clusters) - remaining_baseline = list(baseline_clusters) - - current_by_gap = _group_clusters_by_gap_label(current_unmatched) - baseline_by_gap = _group_clusters_by_gap_label(remaining_baseline) - for gap_label in list(current_by_gap): - current_group = current_by_gap.get(gap_label) or [] - baseline_group = baseline_by_gap.get(gap_label) or [] - if len(current_group) != 1 or len(baseline_group) != 1: - continue - current = current_group[0] - baseline = baseline_group[0] - _append_matched_cluster_pair( - matched, - current, - baseline, - match_confidence=0.75, - match_method="single_cluster_per_gap_label", - ) - current_unmatched.remove(current) - remaining_baseline.remove(baseline) - current_by_gap[gap_label] = [] - baseline_by_gap[gap_label] = [] - - for current in list(current_unmatched): - best_idx: Optional[int] = None - best_score = 0.0 - for idx, baseline in enumerate(remaining_baseline): - score = _cluster_label_similarity( - str(current.get("label") or ""), - str(baseline.get("label") or ""), - ) - if current.get("gap_label") == baseline.get("gap_label"): - score += 0.1 - if score > best_score: - best_score = score - best_idx = idx - - if best_idx is not None and best_score >= similarity_threshold: - baseline = remaining_baseline.pop(best_idx) - _append_matched_cluster_pair( - matched, - current, - baseline, - match_confidence=best_score, - match_method="label_similarity", - ) - - matched_current_labels = { - str(item.get("current_label") or "") for item in matched - } - matched_baseline_labels = { - str(item.get("baseline_label") or "") for item in matched - } - current_unmatched = [ - cluster - for cluster in current_clusters - if str(cluster.get("label") or "") not in matched_current_labels - ] - remaining_baseline = [ - cluster - for cluster in baseline_clusters - if str(cluster.get("label") or "") not in matched_baseline_labels - ] - - new_themes = [ - { - "label": cluster.get("label"), - "gap_label": cluster.get("gap_label"), - "share_pct": cluster.get("share_pct"), - "note": "New theme in current period (no close baseline match).", - } - for cluster in current_unmatched - ] - - retired_themes = [ - { - "label": baseline.get("label"), - "gap_label": baseline.get("gap_label"), - "share_pct": baseline.get("share_pct"), - "note": "Theme present in baseline only (retired or renamed).", - } - for baseline in remaining_baseline - ] - - current_gap = _aggregate_share_by_gap_label(current_clusters) - baseline_gap = _aggregate_share_by_gap_label(baseline_clusters) - gap_label_shifts: dict[str, dict[str, float]] = {} - for gap_label in set(current_gap) | set(baseline_gap): - current_share = current_gap.get(gap_label, 0.0) - baseline_share = baseline_gap.get(gap_label, 0.0) - if abs(current_share - baseline_share) >= 0.5: - gap_label_shifts[gap_label] = { - "current_share_pct": current_share, - "baseline_share_pct": baseline_share, - "share_delta_pp": round(current_share - baseline_share, 1), - } - - return { - "matched_theme_shifts": matched, - "new_themes_current_period": new_themes, - "retired_themes_baseline_period": retired_themes, - "gap_label_shifts": gap_label_shifts, - "reconciliation_note": ( - "Cluster labels are generated independently each run and may " - "rename the same failure mode. Do not treat unmatched current " - "labels as 0% in the baseline period." - ), - } - - -def _load_period_delta_explanations_cache( - evaluation: CallImportEvaluation, - cache_key: str, -) -> Optional[dict[str, str]]: - raw = getattr(evaluation, "period_delta_explanations", None) - if not isinstance(raw, dict): - return None - entry = raw.get(cache_key) - if not isinstance(entry, dict): - return None - explanations_raw = entry.get("explanations") - if not isinstance(explanations_raw, dict): - return None - return { - str(metric_id): str(why).strip() - for metric_id, why in explanations_raw.items() - if str(metric_id).strip() and isinstance(why, str) and why.strip() - } - - -def _save_period_delta_explanations_cache( - db: Session, - evaluation: CallImportEvaluation, - cache_key: str, - explanations: dict[str, str], -) -> None: - raw = evaluation.period_delta_explanations - if not isinstance(raw, dict): - raw = {} - updated = dict(raw) - updated[cache_key] = { - "explanations": explanations, - "generated_at": datetime.now(timezone.utc).isoformat(), - } - evaluation.period_delta_explanations = updated - flag_modified(evaluation, "period_delta_explanations") - db.commit() - - -def _cluster_summary_for_metric( - state: Optional[EvaluationMetricClustersState], - metric_id: str, -) -> list[dict[str, Any]]: - if state is None or state.status != "completed": - return [] - for group in state.groups: - if str(group.metric_id) != metric_id: - continue - return [ - { - "label": cluster.label, - "gap_label": cluster.gap_label, - "share_pct": round(cluster.share_pct, 1), - "count": cluster.count, - } - for cluster in group.clusters[:5] - ] - return [] - - -def _merge_delta_why( - raw_deltas: dict[str, dict[str, str]], - explanations: dict[str, str], -) -> dict[str, dict[str, str]]: - if not explanations: - return raw_deltas - merged: dict[str, dict[str, str]] = {} - for metric_id, delta in raw_deltas.items(): - updated = dict(delta) - why = explanations.get(metric_id) - if why: - updated["why"] = why - merged[metric_id] = updated - return merged - - -def _explain_period_deltas( - db: Session, - organization_id: UUID, - evaluation: CallImportEvaluation, - baseline_evaluation: CallImportEvaluation, - raw_deltas: dict[str, dict[str, str]], - *, - min_delta_pp: float = 0.5, -) -> dict[str, dict[str, str]]: - """Attach ``why`` explanations to period deltas using cached LLM output.""" - if not raw_deltas: - return raw_deltas - - cache_key = _period_delta_explanation_cache_key( - baseline_evaluation.id, - completed_rows=evaluation.completed_rows, - baseline_completed_rows=baseline_evaluation.completed_rows, - ) - cached = _load_period_delta_explanations_cache(evaluation, cache_key) - if cached is not None: - return _merge_delta_why(raw_deltas, cached) - - current_clusters = _metric_clusters_payload(evaluation) - baseline_clusters = _metric_clusters_payload(baseline_evaluation) - metrics_for_prompt: list[dict[str, Any]] = [] - for metric_id, delta in raw_deltas.items(): - label = delta.get("label") or "" - if "No previous-week baseline" in label: - continue - match = re.search(r"([+-]?\d+(?:\.\d+)?)\s*pp", label) - if match and abs(float(match.group(1))) < min_delta_pp: - continue - current_summary = _cluster_summary_for_metric(current_clusters, metric_id) - baseline_summary = _cluster_summary_for_metric(baseline_clusters, metric_id) - if not current_summary and not baseline_summary: - continue - cluster_reconciliation = _reconcile_cluster_periods( - current_summary, - baseline_summary, - ) - metrics_for_prompt.append( - { - "metric_id": metric_id, - "delta_label": label, - "delta_detail": delta.get("detail") or "", - "cluster_reconciliation": cluster_reconciliation, - } - ) - - if not metrics_for_prompt: - return raw_deltas - - provider_hint: Optional[str] = None - model_hint: Optional[str] = None - tldr_raw = evaluation.tldr_summary - if isinstance(tldr_raw, dict): - if isinstance(tldr_raw.get("provider"), str): - provider_hint = tldr_raw["provider"] - if isinstance(tldr_raw.get("model"), str): - model_hint = tldr_raw["model"] - - from app.services.ai.llm_resolver import get_llm_provider_and_model - from app.services.call_import_user_insights import _call_llm, _parse_json_object - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, provider_hint, model_hint - ) - try: - text = _call_llm( - db, - organization_id, - provider_enum, - model_str, - [ - {"role": "system", "content": _DELTA_EXPLANATION_SYSTEM_PROMPT}, - { - "role": "user", - "content": json.dumps( - {"metrics": metrics_for_prompt}, - ensure_ascii=False, - default=str, - ), - }, - ], - temperature=0.3, - max_tokens=900, - ) - except Exception as exc: - logger.warning("[PeriodDeltaExplain] LLM call failed: {}", exc) - return raw_deltas - - parsed = _parse_json_object(text) - explanations_raw = parsed.get("explanations") - explanations: dict[str, str] = {} - if isinstance(explanations_raw, dict): - for metric_id, why in explanations_raw.items(): - if isinstance(why, str) and why.strip(): - explanations[str(metric_id)] = why.strip() - - if explanations: - _save_period_delta_explanations_cache( - db, evaluation, cache_key, explanations - ) - return _merge_delta_why(raw_deltas, explanations) - - -def _period_deltas_with_explanations( - db: Session, - organization_id: UUID, - evaluation: CallImportEvaluation, - baseline_evaluation: CallImportEvaluation, - raw_deltas: dict[str, dict[str, str]], -) -> dict[str, dict[str, str]]: - return _explain_period_deltas( - db, - organization_id, - evaluation, - baseline_evaluation, - raw_deltas, - ) - - -def _benchmark_context_for_snapshot( - db: Session, - previous_snapshot: Optional[CallImportEvaluationReportSnapshot], -) -> Optional[dict[str, str]]: - if previous_snapshot is None: - return None - previous_import = ( - db.query(CallImport) - .filter(CallImport.id == previous_snapshot.call_import_id) - .first() - ) - previous_eval = ( - db.query(CallImportEvaluation) - .filter(CallImportEvaluation.id == previous_snapshot.evaluation_id) - .first() - ) - dataset = ( - (previous_import.dataset or "").strip() - if previous_import and previous_import.dataset - else None - ) - filename = ( - (previous_import.original_filename or previous_import.filename or "").strip() - if previous_import - else None - ) - evaluation_label = ( - (previous_eval.name or "").strip() - if previous_eval and previous_eval.name - else str(previous_snapshot.evaluation_id)[:8] - ) - period = previous_snapshot.period_label or ( - previous_snapshot.period_start.isoformat() - if previous_snapshot.period_start - else "previous report" - ) - return { - "dataset": dataset or filename or "Unknown dataset", - "evaluation": evaluation_label, - "evaluation_id": str(previous_snapshot.evaluation_id), - "period": period, - } - - -def _clamp_prose_to_sentences( - text: str, - *, - max_sentences: int = 3, - max_chars: int = 300, -) -> str: - """Keep concise audit/TLDR prose within sentence and character limits.""" - cleaned = (text or "").strip() - if not cleaned: - return cleaned - cleaned = re.sub(r"\s*\n+\s*", " ", cleaned).strip() - sentences = [ - sentence.strip() - for sentence in re.split(r"(?<=[.!?])\s+", cleaned) - if sentence.strip() - ] - if sentences: - result = " ".join(sentences[:max_sentences]).strip() - else: - result = cleaned - if len(result) > max_chars: - trimmed = result[: max_chars - 3].rsplit(" ", 1)[0].rstrip(".,;:") - result = f"{trimmed}..." if trimmed else result[:max_chars] - return result - - -def _audit_summary_text_from_tldr( - summary: Optional[EvaluationTldrSummary], -) -> Optional[str]: - if summary is None: - return None - narrative = _clamp_prose_to_sentences(summary.narrative.strip()) - return narrative or None - - -def _metric_insights_from_tldr( - summary: Optional[EvaluationTldrSummary], -) -> dict[str, str]: - if summary is None: - return {} - return { - str(metric_id): insight.strip() - for metric_id, insight in summary.metric_insights.items() - if str(metric_id).strip() and insight.strip() - } - - -def _report_period_from_rows( - rows: list[tuple[CallImportEvaluationRow, CallImportRow]], -) -> tuple[Optional[date], Optional[date], Optional[str], str]: - dates = [ - source_row.recording_date - for eval_row, source_row in rows - if eval_row.status == "completed" and source_row.recording_date - ] - if not dates: - return None, None, None, "Not specified" - start = min(dates) - end = max(dates) - week_anchor = max(dates) - week_start = week_anchor - timedelta(days=week_anchor.weekday()) - week_end = week_start + timedelta(days=6) - iso_year, iso_week, _ = week_anchor.isocalendar() - label = f"{iso_year}-W{iso_week:02d}" - if week_start.year == week_end.year: - week_range = f"{week_start.strftime('%b %d')}–{week_end.strftime('%b %d, %Y')}" - else: - week_range = ( - f"{week_start.strftime('%b %d, %Y')}–{week_end.strftime('%b %d, %Y')}" - ) - display = f"W{iso_week:02d} · {week_range}" - return start, end, label, display - - -def _aggregate_to_dict(aggregate: CallImportMetricAggregate) -> dict[str, Any]: - if hasattr(aggregate, "model_dump"): - return aggregate.model_dump(mode="json") - return aggregate.dict() - - -def _aggregate_primary_percent( - raw: dict[str, Any], - policy: Optional[MetricFailurePolicy] = None, -) -> Optional[float]: - return aggregate_primary_percent(raw, policy) - - -def _child_names_by_parent( - db: Session, - organization_id: UUID, - parent_metric_ids: Sequence[UUID], -) -> Dict[str, List[str]]: - if not parent_metric_ids: - return {} - children = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.parent_metric_id.in_(list(parent_metric_ids)), - ) - .all() - ) - out: Dict[str, List[str]] = {} - for child in children: - pid = str(child.parent_metric_id) - out.setdefault(pid, []).append(child.name) - return out - - -def _clustering_context( - db: Session, - evaluation: CallImportEvaluation, - eval_rows: List[CallImportEvaluationRow], -) -> Tuple[ - List[Metric], - List[CallImportMetricAggregate], - Dict[str, MetricFailurePolicy], - Literal["inferred", "user"], - Dict[str, List[str]], -]: - metrics = _metrics_for_clustering(db, evaluation, eval_rows) - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - parent_ids = [ - m.id - for m in metrics - if getattr(m, "selection_mode", None) and not getattr(m, "parent_metric_id", None) - ] - child_names_by_parent = _child_names_by_parent( - db, evaluation.organization_id, parent_ids - ) - policies, source = effective_policies( - evaluation, - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - ) - return metrics, aggregates, policies, source, child_names_by_parent - - -def _period_deltas_from_aggregates( - previous_metric_aggregates: list[dict[str, Any]], - current_metric_aggregates: list[dict[str, Any]], - policies: Optional[Dict[str, MetricFailurePolicy]] = None, -) -> dict[str, dict[str, str]]: - current_by_id = {str(item.get("metric_id")): item for item in current_metric_aggregates} - previous_by_id = { - str(item.get("metric_id")): item - for item in previous_metric_aggregates - if isinstance(item, dict) - } - deltas: dict[str, dict[str, str]] = {} - for metric_id, current in current_by_id.items(): - previous_raw = previous_by_id.get(metric_id) - policy = (policies or {}).get(metric_id) - current_pct = _aggregate_primary_percent(current, policy) - previous_pct = ( - _aggregate_primary_percent(previous_raw, policy) - if previous_raw - else None - ) - if current_pct is None or previous_pct is None: - deltas[metric_id] = { - "label": "No previous-week baseline", - "detail": "No comparable prior report snapshot was found.", - } - continue - delta = current_pct - previous_pct - sign = "+" if delta >= 0 else "" - deltas[metric_id] = { - "label": f"{sign}{delta:.1f} pp", - "detail": f"Current report {current_pct:.1f}% vs previous report {previous_pct:.1f}%", - } - return deltas - - -def _period_deltas_from_snapshot( - previous: Optional[CallImportEvaluationReportSnapshot], - current_metric_aggregates: list[dict[str, Any]], -) -> dict[str, dict[str, str]]: - previous_items = ( - previous.metric_aggregates - if previous and isinstance(previous.metric_aggregates, list) - else [] - ) - return _period_deltas_from_aggregates(previous_items, current_metric_aggregates) - - -def _sample_evidence_for_metrics( - rows: list[tuple[CallImportEvaluationRow, CallImportRow]], - metric_ids: set[str], -) -> dict[str, list[dict[str, str]]]: - samples: dict[str, list[dict[str, str]]] = {metric_id: [] for metric_id in metric_ids} - for eval_row, source_row in rows: - scores = eval_row.metric_scores if isinstance(eval_row.metric_scores, dict) else {} - for metric_id in metric_ids: - if len(samples.get(metric_id, [])) >= 4: - continue - score = scores.get(metric_id) - if not isinstance(score, dict): - continue - rationale = score.get("rationale") - transcript = source_row.diarised_transcript or source_row.transcript or "" - quote = rationale if isinstance(rationale, str) and rationale.strip() else transcript[:350] - if quote: - samples.setdefault(metric_id, []).append( - { - "conversation_id": source_row.conversation_id, - "quote": str(quote).strip()[:500], - } - ) - return samples - - -def _fallback_report_narrative( - insight_aggregates: list[dict[str, Any]], - evidence_samples: dict[str, list[dict[str, str]]], -) -> dict[str, Any]: - observations: dict[str, str] = {} - evidence: dict[str, dict[str, str]] = {} - design_notes: list[str] = [] - for aggregate in insight_aggregates: - metric_id = str(aggregate.get("metric_id") or "") - name = str(aggregate.get("metric_name") or "Insight") - counts = aggregate.get("value_counts") if isinstance(aggregate.get("value_counts"), list) else [] - if counts: - top = counts[0] - total = int(aggregate.get("count") or 0) or sum( - int(item.get("count") or 0) for item in counts if isinstance(item, dict) - ) - pct = (int(top.get("count") or 0) / total) * 100 if total else 0 - observations[metric_id] = ( - f"{top.get('label')} is the dominant {name.lower()} category at {pct:.1f}% of classified calls." - ) - design_notes.append( - f"{name}: {top.get('label')} is the largest segment and should be reviewed for workflow or prompt improvements." - ) - sample = (evidence_samples.get(metric_id) or [{}])[0] - if sample: - evidence[metric_id] = sample - return { - "observations": observations, - "evidence": evidence, - "design_notes": design_notes[:7], - "audit_summary": None, - } - - -def _generate_report_narrative( - db: Session, - organization_id: UUID, - *, - metric_aggregates: list[dict[str, Any]], - insight_aggregates: list[dict[str, Any]], - period_delta_by_metric: dict[str, dict[str, str]], - evidence_samples: dict[str, list[dict[str, str]]], - report_config: dict[str, Any], -) -> dict[str, Any]: - if not insight_aggregates: - return {"observations": {}, "evidence": {}, "design_notes": [], "audit_summary": None} - try: - from app.services.ai.llm_resolver import get_llm_provider_and_model - from app.services.ai.llm_service import llm_service - - provider_enum, model_str = get_llm_provider_and_model(organization_id, db, None, None) - prompt = ( - "You are writing a vendor-safe external call quality audit report. " - "Return strict JSON with keys observations (object keyed by metric_id), " - "evidence (object keyed by metric_id with conversation_id and quote), " - "design_notes (array of concise numbered-note strings), and audit_summary (string). " - "Use only the supplied aggregates and evidence samples.\n\n" - + json.dumps( - { - "metric_aggregates": metric_aggregates[:30], - "insight_aggregates": insight_aggregates, - "period_deltas": period_delta_by_metric, - "evidence_samples": evidence_samples, - "report_config": report_config, - }, - default=str, - ) - ) - llm_result = llm_service.generate_response( - messages=[ - {"role": "system", "content": "Return JSON only. No markdown."}, - {"role": "user", "content": prompt}, - ], - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - temperature=0.2, - max_tokens=1200, - ) - parsed = json.loads(str(llm_result.content or "{}")) - if isinstance(parsed, dict): - fallback = _fallback_report_narrative(insight_aggregates, evidence_samples) - return { - "observations": parsed.get("observations") or fallback["observations"], - "evidence": parsed.get("evidence") or fallback["evidence"], - "design_notes": parsed.get("design_notes") or fallback["design_notes"], - "audit_summary": parsed.get("audit_summary") or fallback["audit_summary"], - } - except Exception as exc: # noqa: BLE001 - logger.warning("Report narrative LLM generation fell back to deterministic text: {}", exc) - return _fallback_report_narrative(insight_aggregates, evidence_samples) - - -@router.get( - "/{eval_id}/baseline-candidates", - response_model=CallImportEvaluationBaselineCandidatesResponse, - operation_id="listCallImportEvaluationBaselineCandidates", -) -async def list_call_import_evaluation_baseline_candidates( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationBaselineCandidatesResponse: - del api_key - call_import = _require_import(db, call_import_id, organization_id) - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - rows = _evaluation_rows_for_period(db, evaluation.id) - period_start, _period_end, _derived_period_label, _period_display = _report_period_from_rows( - rows - ) - candidates = _baseline_candidate_evaluations( - db, - organization_id, - call_import.workspace_id, - evaluation, - period_start, - ) - default_evaluation_id = next( - (item["evaluation_id"] for item in candidates if item.get("is_default")), - None, - ) - return CallImportEvaluationBaselineCandidatesResponse( - items=[CallImportEvaluationBaselineCandidate(**item) for item in candidates], - default_evaluation_id=default_evaluation_id, - ) - - -@router.post( - "/{eval_id}/pdf-report", - operation_id="generateCallImportEvaluationPdfReport", - dependencies=[Depends(require_call_import_capability(REPORTS_GENERATE))], -) -async def generate_call_import_evaluation_pdf_report( - call_import_id: UUID, - eval_id: UUID, - payload: CallImportEvaluationPdfReportRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -): - del api_key - call_import = _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - is_internal = payload.report_type == "internal" - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - rows = sorted( - load_evaluation_row_pairs(db, eval_id), - key=lambda pair: int(pair[1].row_index or 0), - ) - else: - rows = ( - db.query(CallImportEvaluationRow, CallImportRow) - .join( - CallImportRow, - CallImportRow.id == CallImportEvaluationRow.call_import_row_id, - ) - .filter(CallImportEvaluationRow.evaluation_id == eval_id) - .order_by(CallImportRow.row_index.asc()) - .all() - ) - report_config = payload.report_config if isinstance(payload.report_config, dict) else {} - metrics = _display_metrics_for_pdf_report(db, organization_id, evaluation) - configured_quality_ids = { - str(item) - for item in report_config.get("quality_metric_ids", []) - if item - } - configured_insight_ids = { - str(item.get("metric_id") or item) - for item in report_config.get("insights", []) - if item - } - if configured_quality_ids or configured_insight_ids: - allowed_ids = configured_quality_ids | configured_insight_ids - metrics = [metric for metric in metrics if str(metric.id) in allowed_ids] - - eval_rows = [eval_row for eval_row, _source_row in rows] - aggregate_models = _compute_metric_aggregates(db, evaluation, eval_rows) - selected_report_metric_ids = {str(metric.id) for metric in metrics} - aggregate_dicts = [ - _aggregate_to_dict(aggregate) - for aggregate in aggregate_models - if aggregate.metric_id in selected_report_metric_ids - ] - insight_metric_ids = { - str(metric.id) - for metric in metrics - if _metric_is_user_insight(metric) - } - metric_aggregates = [ - item for item in aggregate_dicts if str(item.get("metric_id")) not in insight_metric_ids - ] - insight_aggregates = [ - item for item in aggregate_dicts if str(item.get("metric_id")) in insight_metric_ids - ] - period_start, period_end, derived_period_label, period_display = _report_period_from_rows(rows) - period_label = (payload.period_label or derived_period_label or "").strip() or None - include_period_delta = ( - payload.include_period_delta or payload.include_weekly_delta - ) - previous_snapshot = None - period_delta_by_metric: dict[str, dict[str, str]] = {} - baseline_evaluation: Optional[CallImportEvaluation] = None - if include_period_delta and period_start: - baseline_evaluation = _resolve_baseline_evaluation( - db, - organization_id, - call_import.workspace_id, - evaluation, - period_start, - payload.baseline_evaluation_id, - ) - if baseline_evaluation: - period_delta_by_metric = _period_deltas_from_evaluation( - db, - baseline_evaluation, - metric_aggregates, - evaluation, - [eval_row for eval_row, _ in rows], - ) - period_delta_by_metric = _period_deltas_with_explanations( - db, - organization_id, - evaluation, - baseline_evaluation, - period_delta_by_metric, - ) - benchmark_context = _benchmark_context_for_evaluation(db, baseline_evaluation) - evidence_samples = _sample_evidence_for_metrics(rows, insight_metric_ids) - cached_tldr_summary = _tldr_summary_payload(evaluation) - cached_user_insights = _user_insights_payload(evaluation) - cached_metric_clusters = _metric_clusters_payload(evaluation) - cached_prompt_improvements = _prompt_improvements_payload(evaluation) - generated_insights_for_pdf = _selected_generated_user_insights( - cached_user_insights, - report_config, - ) - metric_clusters_for_pdf = _selected_metric_clusters_for_pdf( - cached_metric_clusters, - report_config, - ) - prompt_improvements_for_pdf = _selected_prompt_improvements_for_pdf( - cached_prompt_improvements, - report_config, - ) - branding_images, custom_heading = _report_branding_for_import_workspace( - db, - organization_id, - call_import.workspace_id, - internal_brand_image_id=payload.internal_brand_image_id, - external_brand_image_id=payload.external_brand_image_id, - ) - eval_row_list = [eval_row for eval_row, _ in rows] - pdf_aggregates = _compute_metric_aggregates(db, evaluation, eval_row_list) - pdf_parent_ids = [ - m.id - for m in metrics - if getattr(m, "selection_mode", None) - and not getattr(m, "parent_metric_id", None) - ] - pdf_child_map = _child_names_by_parent( - db, evaluation.organization_id, pdf_parent_ids - ) - failure_policies_for_pdf, _fp_source = effective_policies( - evaluation, - metrics, - pdf_aggregates, - child_names_by_parent=pdf_child_map, - ) - - from app.services.storage.s3_service import s3_service - - config_fingerprint = compute_pdf_report_config_fingerprint( - report_type=payload.report_type, - include_period_delta=bool(payload.include_period_delta), - include_weekly_delta=bool(payload.include_weekly_delta), - baseline_evaluation_id=payload.baseline_evaluation_id, - internal_brand_image_id=payload.internal_brand_image_id, - external_brand_image_id=payload.external_brand_image_id, - use_case=payload.use_case, - report_config=report_config, - report_heading=custom_heading, - vendor_name=payload.vendor_name, - platform_base_url=payload.platform_base_url, - period_label=period_label, - ) - content_fingerprint = compute_pdf_report_content_fingerprint( - evaluation_status=evaluation.status, - completed_rows=int(evaluation.completed_rows or 0), - total_rows=int(evaluation.total_rows or 0), - failed_rows=int(evaluation.failed_rows or 0), - metric_aggregates=metric_aggregates, - insight_aggregates=insight_aggregates, - period_delta_by_metric=period_delta_by_metric, - benchmark_context=benchmark_context, - metric_metadata=[ - { - "id": str(metric.id), - "name": metric.name, - "description": metric.description, - } - for metric in metrics - ], - failure_policies=failure_policies_for_pdf, - tldr_summary=cached_tldr_summary, - user_insights_for_pdf=generated_insights_for_pdf, - metric_clusters_for_pdf=metric_clusters_for_pdf, - prompt_improvements_for_pdf=prompt_improvements_for_pdf, - ) - cache_fingerprint = compute_pdf_report_cache_fingerprint( - config_fingerprint=config_fingerprint, - content_fingerprint=content_fingerprint, - ) - if s3_service.is_enabled(): - cached_pdf_report = find_cached_pdf_report( - db, - evaluation_id=evaluation.id, - organization_id=organization_id, - cache_fingerprint=cache_fingerprint, - ) - if cached_pdf_report is not None: - logger.info( - "Reusing stored PDF report {} for evaluation {} (cache fingerprint match)", - cached_pdf_report.id, - eval_id, - ) - return _pdf_report_response_from_row(cached_pdf_report, cache_hit=True) - - narrative = _generate_report_narrative( - db, - organization_id, - metric_aggregates=metric_aggregates, - insight_aggregates=insight_aggregates if is_internal else [], - period_delta_by_metric=period_delta_by_metric, - evidence_samples=evidence_samples if is_internal else {}, - report_config=report_config, - ) - - generated_at = datetime.now(timezone.utc) - try: - pdf_started = datetime.now(timezone.utc) - pdf_bytes = await asyncio.to_thread( - call_import_evaluation_pdf_report_service.render_pdf, - vendor_name=payload.vendor_name, - call_import=call_import, - evaluation=evaluation, - metrics=metrics, - rows=rows, - failure_policies=failure_policies_for_pdf, - generated_at=generated_at, - internal=is_internal, - logo_data_uris=branding_images, - custom_heading=custom_heading, - include_weekly_delta=include_period_delta, - period_delta_by_metric=period_delta_by_metric, - use_case=payload.use_case, - period_display=period_display, - total_metric_count=db.query(Metric) - .filter(Metric.organization_id == organization_id, Metric.enabled.is_(True)) - .count(), - report_config=report_config, - narrative=narrative, - audit_summary=_audit_summary_text_from_tldr(cached_tldr_summary), - metric_insights=_metric_insights_from_tldr(cached_tldr_summary), - benchmark_context=benchmark_context, - generated_user_insights=generated_insights_for_pdf, - user_insights_overview=( - cached_user_insights.overview if cached_user_insights else None - ), - metric_clusters=metric_clusters_for_pdf, - metric_clusters_overview=( - cached_metric_clusters.overview if cached_metric_clusters else None - ), - prompt_improvements=prompt_improvements_for_pdf, - platform_base_url=payload.platform_base_url, - ) - logger.info( - "PDF report render finished in {:.1f}s for evaluation {}", - (datetime.now(timezone.utc) - pdf_started).total_seconds(), - eval_id, - ) - except Exception as exc: # noqa: BLE001 - logger.exception( - "Failed to generate PDF report for call import {} evaluation {}", - call_import_id, - eval_id, - ) - raise HTTPException( - status_code=500, - detail=f"Failed to generate PDF report: {exc}", - ) from exc - - snapshot = CallImportEvaluationReportSnapshot( - evaluation_id=evaluation.id, - call_import_id=call_import.id, - organization_id=organization_id, - workspace_id=call_import.workspace_id, - period_label=period_label, - period_start=period_start, - period_end=period_end, - report_config=report_config, - selected_metric_ids=[str(metric.id) for metric in metrics], - metric_aggregates=metric_aggregates, - insight_aggregates=insight_aggregates, - narrative=narrative, - total_calls=evaluation.total_rows, - selected_metric_count=len(metrics), - total_metric_count=db.query(Metric) - .filter(Metric.organization_id == organization_id, Metric.enabled.is_(True)) - .count(), - ) - db.add(snapshot) - db.flush() - - filename = ( - f"{_report_filename_slug(payload.vendor_name)}-" - f"{payload.report_type}-quality-metric-audit-{eval_id}.pdf" - ) - - if not s3_service.is_enabled(): - db.commit() - return StreamingResponse( - iter([pdf_bytes]), - media_type="application/pdf", - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, - ) - - report_id = uuid4() - s3_key = build_pdf_report_s3_key( - organization_id=organization_id, - call_import_id=call_import.id, - evaluation_id=evaluation.id, - report_id=report_id, - ) - try: - s3_service.upload_file_by_key( - file_content=pdf_bytes, - key=s3_key, - content_type="application/pdf", - ) - except Exception as exc: # noqa: BLE001 - logger.exception( - "Failed to upload PDF report for evaluation {} to object storage", - eval_id, - ) - db.rollback() - raise HTTPException( - status_code=500, - detail=f"Failed to store PDF report: {exc}", - ) from exc - - created_by, created_by_user_id = _pdf_report_actor(principal) - pdf_report = CallImportEvaluationPdfReport( - id=report_id, - evaluation_id=evaluation.id, - call_import_id=call_import.id, - organization_id=organization_id, - workspace_id=call_import.workspace_id, - snapshot_id=snapshot.id, - vendor_name=payload.vendor_name, - report_type=payload.report_type, - filename=filename, - s3_key=s3_key, - report_config=report_config, - cache_fingerprint=cache_fingerprint, - created_by=created_by, - created_by_user_id=created_by_user_id, - ) - db.add(pdf_report) - try: - db.commit() - except IntegrityError: - db.rollback() - try: - s3_service.delete_file_by_key(s3_key) - except Exception: # noqa: BLE001 - logger.warning( - "Failed to delete orphan PDF after cache race for evaluation {}", - eval_id, - ) - raced_winner = find_cached_pdf_report( - db, - evaluation_id=evaluation.id, - organization_id=organization_id, - cache_fingerprint=cache_fingerprint, - ) - if raced_winner is not None: - logger.info( - "PDF report cache race resolved for evaluation {} (winner {})", - eval_id, - raced_winner.id, - ) - return _pdf_report_response_from_row(raced_winner, cache_hit=True) - raise HTTPException( - status_code=500, - detail="Failed to store PDF report due to a concurrent duplicate request.", - ) from None - db.refresh(pdf_report) - return _pdf_report_response_from_row(pdf_report) - - -@router.get( - "/{eval_id}/pdf-reports", - response_model=CallImportEvaluationPdfReportListResponse, - operation_id="listCallImportEvaluationPdfReports", -) -async def list_call_import_evaluation_pdf_reports( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationPdfReportListResponse: - del api_key - _require_import(db, call_import_id, organization_id) - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - rows = ( - db.query(CallImportEvaluationPdfReport) - .filter( - CallImportEvaluationPdfReport.evaluation_id == eval_id, - CallImportEvaluationPdfReport.organization_id == organization_id, - ) - .order_by(desc(CallImportEvaluationPdfReport.created_at)) - .all() - ) - return CallImportEvaluationPdfReportListResponse( - items=[_pdf_report_list_item_from_row(row) for row in rows], - ) - - -@router.get( - "/{eval_id}/pdf-reports/{report_id}", - response_model=CallImportEvaluationPdfReportResponse, - operation_id="getCallImportEvaluationPdfReport", -) -async def get_call_import_evaluation_pdf_report( - call_import_id: UUID, - eval_id: UUID, - report_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationPdfReportResponse: - del api_key - _require_import(db, call_import_id, organization_id) - row = ( - db.query(CallImportEvaluationPdfReport) - .filter( - CallImportEvaluationPdfReport.id == report_id, - CallImportEvaluationPdfReport.evaluation_id == eval_id, - CallImportEvaluationPdfReport.call_import_id == call_import_id, - CallImportEvaluationPdfReport.organization_id == organization_id, - ) - .first() - ) - if not row: - raise HTTPException(status_code=404, detail="PDF report not found") - if not row.s3_key: - raise HTTPException( - status_code=404, - detail="PDF report file is not available in object storage", - ) - return _pdf_report_response_from_row(row) - - -@router.get( - "/{eval_id}/pdf-reports/{report_id}/download", - operation_id="downloadCallImportEvaluationPdfReport", -) -async def download_call_import_evaluation_pdf_report( - call_import_id: UUID, - eval_id: UUID, - report_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -): - del api_key - _require_import(db, call_import_id, organization_id) - row = ( - db.query(CallImportEvaluationPdfReport) - .filter( - CallImportEvaluationPdfReport.id == report_id, - CallImportEvaluationPdfReport.evaluation_id == eval_id, - CallImportEvaluationPdfReport.call_import_id == call_import_id, - CallImportEvaluationPdfReport.organization_id == organization_id, - ) - .first() - ) - if not row: - raise HTTPException(status_code=404, detail="PDF report not found") - if not row.s3_key: - raise HTTPException( - status_code=404, - detail="PDF report file is not available in object storage", - ) - from app.services.storage.s3_service import s3_service - - if not s3_service.is_enabled(): - raise HTTPException( - status_code=503, - detail="Object storage is not enabled or not configured.", - ) - try: - file_bytes = s3_service.download_file_by_key(row.s3_key) - except Exception as exc: # noqa: BLE001 - logger.exception( - "Failed to download PDF report {} for evaluation {}", - report_id, - eval_id, - ) - raise HTTPException( - status_code=500, - detail=f"Failed to download PDF report: {exc}", - ) from exc - filename = row.filename or "report.pdf" - safe_name = filename.replace('"', "'") - return StreamingResponse( - iter([file_bytes]), - media_type="application/pdf", - headers={"Content-Disposition": f'attachment; filename="{safe_name}"'}, - ) - - -@router.patch( - "/{eval_id}", - response_model=CallImportEvaluationResponse, - operation_id="updateCallImportEvaluation", -) -async def update_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - payload: CallImportEvaluationUpdate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportEvaluationResponse: - """Edit metadata on an existing evaluation run (currently just ``name``).""" - - del api_key - _require_import(db, call_import_id, organization_id) - - row = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not row: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - # Treat unset vs explicit ``None`` differently: unset = leave alone, - # explicit ``None`` or empty string = clear the name. - payload_data = payload.model_dump(exclude_unset=True) - if "name" in payload_data: - row.name = _normalize_name(payload_data["name"]) - - stamp_evaluation_actor(row, principal) - db.commit() - db.refresh(row) - return _serialize_eval(db, row) - - -def _revoke_pending_tasks(evaluation: CallImportEvaluation) -> None: - """Best-effort cancel of any in-flight Celery tasks for an evaluation.""" - - if not evaluation.celery_group_id and not any( - r.celery_task_id for r in evaluation.row_results - ): - return - try: - from app.workers.celery_app import celery_app - - pending_task_ids = [ - eval_row.celery_task_id - for eval_row in evaluation.row_results - if eval_row.celery_task_id - and eval_row.status in {"pending", "running"} - ] - if pending_task_ids: - celery_app.control.revoke(pending_task_ids, terminate=False) - except Exception: - # Best effort — DB delete remains the source of truth. - pass - - -# --------------------------------------------------------------------------- -# User-initiated cancel for in-flight evaluation rows -# --------------------------------------------------------------------------- -# -# Evaluation rows can sit in ``running`` for many minutes when the underlying -# LLM / audio metric call is slow or wedged (the worker carries an 8 min -# soft / 10 min hard time limit). Without a cancel affordance the operator's -# only recourse is to wait for Celery's time limit to fire — or to manually -# mutate the DB. These helpers + the two endpoints below give the UI a -# first-class "Abort" button mirroring the diarisation cancel pattern at -# ``app.api.v1.routes.call_imports`` (``_apply_diarisation_cancel`` etc.). -# -# Why ``terminate=True``: the legacy ``_revoke_pending_tasks`` above uses -# ``terminate=False`` because it's called from delete-flow paths where the -# task may simply not get to run (a worker pulls it off the queue and drops -# it). For a user-initiated cancel we want SIGTERM to interrupt the worker -# mid-LLM/audio call so the in-flight HTTP request actually aborts. -# ``terminate=True`` routes the signal to the executing process; we spell -# ``signal="SIGTERM"`` out for clarity even though it's the default. - -# Sentinel error message stamped on cancelled rows. Read by the eval worker's -# ``_was_cancelled_externally`` guard (see -# :mod:`app.workers.tasks.evaluate_call_import_row`) so a worker that's already -# past its slowest operation can't overwrite the cancelled state with its own -# terminal status. Touching either copy means touching both. -EVAL_CANCELLED_BY_USER_ERROR: str = "Evaluation cancelled by user" - - -def _cancellable_eval_states() -> Tuple[str, ...]: - """States that an evaluation row can be cancelled from. - - Kept as a tiny helper so adding a future ``"queued"`` / ``"retrying"`` - state only needs one edit. - """ - return ("pending", "running") - - -def _revoke_eval_task(eval_row: CallImportEvaluationRow) -> None: - """Best-effort revoke of a single eval row's Celery task. - - Always swallows control-plane exceptions — Celery's control bus is - inherently best-effort and a missed revoke is not catastrophic - because the DB row is already flipped to ``failed`` by the caller - before this runs (so the UI immediately reflects the cancel; if - the task happens to finish anyway, the worker's finaliser skips - over the row via :data:`EVAL_CANCELLED_BY_USER_ERROR`). - """ - task_id = (eval_row.celery_task_id or "").strip() - if not task_id: - return - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke( - task_id, terminate=True, signal="SIGTERM" - ) - logger.info( - "Revoked evaluation task {} for eval row {}", - task_id, - eval_row.id, - ) - except Exception as exc: # noqa: BLE001 — revoke is best-effort - logger.warning( - "Failed to revoke evaluation task {} for eval row {}: {}", - task_id, - eval_row.id, - exc, - ) - - -def _apply_evaluation_cancel( - eval_rows: List[CallImportEvaluationRow], -) -> Tuple[int, int]: - """Cancel every cancellable row in ``eval_rows``. - - Returns ``(cancelled, skipped)`` so the caller can build a typed - response without re-querying the DB. The caller is responsible for - ``db.commit()`` after this returns — we deliberately don't commit - here so a batch endpoint can flush all rows in one transaction. - """ - cancellable_states = _cancellable_eval_states() - cancelled = 0 - skipped = 0 - now = datetime.now(timezone.utc) - for eval_row in eval_rows: - if (eval_row.status or "").lower() not in cancellable_states: - skipped += 1 - continue - # Flip the row state BEFORE we revoke so the UI's next poll - # already shows the cancel, even if Celery's control plane is - # slow to ack. - eval_row.status = "failed" - eval_row.error_message = EVAL_CANCELLED_BY_USER_ERROR - eval_row.finished_at = now - _revoke_eval_task(eval_row) - # Drop the task id so a follow-up retry (or a stale poll) can't - # accidentally re-revoke or get confused. - eval_row.celery_task_id = None - cancelled += 1 - return cancelled, skipped - - -def _claim_evaluation_bulk_operation( - evaluation_id: UUID, - operation: str, -) -> None: - """Reserve the run for a single bulk worker pass; 409 if one is active.""" - from app.services.call_imports.evaluation_bulk_op import ( - get_evaluation_bulk_operation, - try_set_evaluation_bulk_operation, - ) - - if try_set_evaluation_bulk_operation(evaluation_id, operation): # type: ignore[arg-type] - return - existing = get_evaluation_bulk_operation(evaluation_id) or operation - raise HTTPException( - status_code=409, - detail=( - f"A bulk {existing.replace('_', ' ')} operation is already in " - "progress for this evaluation. Wait for it to finish before " - "starting another action." - ), - ) - - -def _require_no_evaluation_bulk_operation(evaluation_id: UUID) -> None: - from app.services.call_imports.evaluation_bulk_op import ( - get_evaluation_bulk_operation, - ) - - existing = get_evaluation_bulk_operation(evaluation_id) - if existing: - raise HTTPException( - status_code=409, - detail=( - f"A bulk {existing.replace('_', ' ')} operation is already in " - "progress for this evaluation. Wait for it to finish before " - "starting another action." - ), - ) - - -@router.post( - "/{eval_id}/cancel", - response_model=CallImportEvaluationBulkActionResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="cancelCallImportEvaluation", -) -async def cancel_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportEvaluationBulkActionResponse: - """Abort all in-flight (or queued) rows in a single evaluation run. - - Idempotent: calling on a run whose rows are already terminal returns - ``target_count=0`` with 202 so the UI can fire this from an - "Abort" button without having to pre-check the state. - - Heavy row resets and Celery revokes run in a background worker so - large batches do not block the API thread. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - from app.services.call_imports.bulk_ops import count_evaluation_cancel_targets - - target_count = count_evaluation_cancel_targets(db, eval_id, mode="abort") - if target_count == 0: - return CallImportEvaluationBulkActionResponse( - accepted=True, - target_count=0, - evaluation_id=eval_id, - ) - - _claim_evaluation_bulk_operation(eval_id, "abort") - evaluation.status = "cancelled" - stamp_evaluation_actor(evaluation, principal) - db.commit() - - from app.workers.tasks.call_import_bulk_ops import ( - cancel_call_import_evaluation_task, - ) - - cancel_call_import_evaluation_task.delay(str(eval_id), mode="abort") - return CallImportEvaluationBulkActionResponse( - accepted=True, - target_count=target_count, - evaluation_id=eval_id, - ) - - -@router.post( - "/{eval_id}/force-fail-pending", - response_model=CallImportEvaluationBulkActionResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="forceFailCallImportEvaluationPending", -) -async def force_fail_pending_call_import_evaluation_rows( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportEvaluationBulkActionResponse: - """Force-fail only rows currently in ``pending`` for a single run. - - This is narrower than :func:`cancel_call_import_evaluation`: it leaves - ``running`` rows untouched so operators can clear permanently queued rows - without interrupting in-flight evaluations. - - Row updates run in a background worker so large batches do not block - the API thread. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - from app.services.call_imports.bulk_ops import count_evaluation_cancel_targets - - target_count = count_evaluation_cancel_targets( - db, eval_id, mode="force_fail_pending" - ) - if target_count == 0: - return CallImportEvaluationBulkActionResponse( - accepted=True, - target_count=0, - evaluation_id=eval_id, - ) - - _claim_evaluation_bulk_operation(eval_id, "force_fail_pending") - stamp_evaluation_actor(evaluation, principal) - db.commit() - - from app.workers.tasks.call_import_bulk_ops import ( - cancel_call_import_evaluation_task, - ) - - cancel_call_import_evaluation_task.delay( - str(eval_id), mode="force_fail_pending" - ) - return CallImportEvaluationBulkActionResponse( - accepted=True, - target_count=target_count, - evaluation_id=eval_id, - ) - - -@router.post( - "/{eval_id}/rows/{eval_row_id}/cancel", - response_model=CallImportEvaluationRowResponse, - status_code=status.HTTP_200_OK, - operation_id="cancelCallImportEvaluationRow", -) -async def cancel_call_import_evaluation_row( - call_import_id: UUID, - eval_id: UUID, - eval_row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportEvaluationRowResponse: - """Abort an in-flight (or queued) evaluation for a single row. - - Idempotent: calling on a row that's already terminal (``completed`` - / ``failed``) returns the row unchanged with a 200 so the UI can - wire this to a "Stop" button without having to pre-check the - state. Updates the parent run's rollup so its counters reflect - the cancel immediately. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - _require_no_evaluation_bulk_operation(eval_id) - - from app.db_sharding.eval_rows import evaluation_row_session - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - try: - with evaluation_row_session(eval_row_id) as ( - row_db, - _catalog_db, - eval_row, - source_row, - _shard_id, - ): - if eval_row.evaluation_id != eval_id: - raise HTTPException( - status_code=404, - detail="Evaluation row not found in this run", - ) - _apply_evaluation_cancel([eval_row]) - row_db.commit() - _rollup_evaluation_status(evaluation, db) - stamp_evaluation_actor(evaluation, principal) - db.commit() - row_db.refresh(eval_row) - return _to_evaluation_row_response(eval_row, source_row, evaluation) - except LookupError as exc: - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) from exc - - eval_row = ( - db.query(CallImportEvaluationRow) - .filter( - CallImportEvaluationRow.id == eval_row_id, - CallImportEvaluationRow.evaluation_id == eval_id, - ) - .first() - ) - if not eval_row: - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) - - _apply_evaluation_cancel([eval_row]) - db.flush() - _rollup_evaluation_status(evaluation, db) - stamp_evaluation_actor(evaluation, principal) - db.commit() - db.refresh(eval_row) - - source_row = ( - db.query(CallImportRow) - .filter(CallImportRow.id == eval_row.call_import_row_id) - .first() - ) - - return _to_evaluation_row_response(eval_row, source_row, evaluation) - - -@router.delete( - "/{eval_id}", - status_code=status.HTTP_204_NO_CONTENT, - operation_id="deleteCallImportEvaluation", -) -async def delete_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> Response: - del api_key - _require_import(db, call_import_id, organization_id) - - row = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not row: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - _revoke_pending_tasks(row) - - db.delete(row) - db.commit() - return Response(status_code=status.HTTP_204_NO_CONTENT) - - -@router.post( - "/bulk-delete", - status_code=status.HTTP_200_OK, - operation_id="bulkDeleteCallImportEvaluations", -) -async def bulk_delete_call_import_evaluations( - call_import_id: UUID, - payload: CallImportEvaluationBulkDelete, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> Dict[str, int]: - """Delete multiple evaluation runs scoped to one call import. - - Mirrors :func:`delete_call_import_evaluation` but in bulk so the UI - can clear out a multi-select. Unknown ids (already deleted, or - belonging to a different org/import) are silently skipped — the - response just reports how many actually went away. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - if not payload.evaluation_ids: - return {"deleted": 0} - - rows = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id.in_(payload.evaluation_ids), - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .all() - ) - deleted = 0 - for row in rows: - _revoke_pending_tasks(row) - db.delete(row) - deleted += 1 - db.commit() - return {"deleted": deleted} - - -# --------------------------------------------------------------------------- -# Aggregation: turns per-row metric scores into histograms / value counts. -# -# Designed to be cheap enough to call on every page load: we read each -# evaluation row once, bucket numeric values into a fixed 10-bin -# histogram, and tally the top categorical values. Scaling concerns -# (millions of rows) are deferred — at that point we'd push this into a -# Postgres aggregate query, but for typical CSV imports (<10k rows) the -# Python pass is fast enough and dramatically simpler. -# --------------------------------------------------------------------------- - - -_HISTOGRAM_BUCKETS = 10 -_TOP_VALUE_COUNTS = 10 - - -def _coerce_numeric(value: Any) -> Optional[float]: - """Return ``value`` as ``float`` when it's numeric; ``None`` otherwise.""" - if isinstance(value, bool): - # Booleans are ints in Python; treat them as categorical so - # pass/fail metrics show up in value_counts instead of becoming - # a degenerate {0,1} histogram. - return None - if isinstance(value, (int, float)) and math.isfinite(value): - return float(value) - if isinstance(value, str): - try: - f = float(value) - if math.isfinite(f): - return f - except ValueError: - return None - return None - - -def _coerce_category(value: Any) -> Optional[str]: - """Render ``value`` as a label suitable for a value_counts bucket.""" - if value is None: - return None - if isinstance(value, bool): - return "true" if value else "false" - if isinstance(value, (int, float)): - return str(value) - if isinstance(value, str): - text = value.strip() - return text or None - # Lists / dicts: stringify so they still group sensibly without - # exploding the cardinality (worst case: everything is "[…]" once). - return str(value) - - -def _build_histogram( - values: List[float], -) -> List[CallImportMetricHistogramBucket]: - """Fixed-bin histogram over ``values``; returns [] for <2 values.""" - if len(values) < 2: - return [] - lo = min(values) - hi = max(values) - if lo == hi: - # All values identical — render a single bucket so the UI shows a - # spike rather than empty space. - return [ - CallImportMetricHistogramBucket(x0=lo, x1=hi, count=len(values)) - ] - width = (hi - lo) / _HISTOGRAM_BUCKETS - buckets: List[List[float]] = [[] for _ in range(_HISTOGRAM_BUCKETS)] - for v in values: - # Right-edge inclusive on the last bucket so ``hi`` doesn't fall - # off into a non-existent bucket index. - idx = int((v - lo) / width) - if idx >= _HISTOGRAM_BUCKETS: - idx = _HISTOGRAM_BUCKETS - 1 - buckets[idx].append(v) - return [ - CallImportMetricHistogramBucket( - x0=lo + i * width, - x1=lo + (i + 1) * width, - count=len(bucket), - ) - for i, bucket in enumerate(buckets) - ] - - -def _percentile(values: List[float], pct: float) -> Optional[float]: - """Linear-interpolated percentile compatible with NumPy default.""" - if not values: - return None - sorted_vals = sorted(values) - if len(sorted_vals) == 1: - return sorted_vals[0] - rank = (pct / 100.0) * (len(sorted_vals) - 1) - lo = int(math.floor(rank)) - hi = int(math.ceil(rank)) - if lo == hi: - return sorted_vals[lo] - frac = rank - lo - return sorted_vals[lo] + (sorted_vals[hi] - sorted_vals[lo]) * frac - - -def _compute_metric_aggregates( - db: Session, - evaluation: CallImportEvaluation, - eval_rows: List[CallImportEvaluationRow], -) -> List[CallImportMetricAggregate]: - """Collapse per-row ``metric_scores`` into one aggregate per metric. - - Selected metrics are read fresh from the DB so the response always - surfaces the current ``metric.name`` / ``metric_type`` even when a - metric was renamed after the run finished. - """ - - selected_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) - # Include parent metrics from selected_metric_groups so they appear - # alongside their children in the aggregate response. Use ``getattr`` - # with a default so the helper still works for callers that pass - # lightweight objects (tests, in-memory shims) that don't carry the - # attribute at all. - groups_raw_candidate = getattr(evaluation, "selected_metric_groups", None) - groups_raw = ( - groups_raw_candidate if isinstance(groups_raw_candidate, dict) else {} - ) - for parent_str in groups_raw.keys(): - try: - pid = UUID(parent_str) - if pid not in selected_ids: - selected_ids.append(pid) - except (TypeError, ValueError): - continue - - metrics = _metrics_for_ids(db, evaluation.organization_id, selected_ids) - metric_meta: Dict[str, Metric] = {str(m.id): m for m in metrics} - - # Default to selected metrics, but also include any metric ids that - # surface in row scores even if missing from the metric registry — - # otherwise renaming/deleting a metric mid-run would silently drop - # results from the chart. - discovered_ids: List[str] = list(metric_meta.keys()) - for row in eval_rows: - scores = row.metric_scores if isinstance(row.metric_scores, dict) else {} - for metric_id_str in scores.keys(): - if metric_id_str not in metric_meta and metric_id_str not in discovered_ids: - discovered_ids.append(metric_id_str) - - results: List[CallImportMetricAggregate] = [] - - for metric_id_str in discovered_ids: - meta = metric_meta.get(metric_id_str) - numeric_values: List[float] = [] - category_counts: Dict[str, int] = {} - # For multi-label parents we still need to know how many rows - # were scored (each row votes for >=1 label) so the n-badge in - # the UI shows "n=50" instead of the misleading "n=208" sum. - multi_label_rows_scored = 0 - # Unordered pair tally for the co-occurrence heatmap. Keys are - # ``(label_a, label_b)`` with ``a < b`` so we never double-count - # the same unordered pair. Only populated for multi-label - # parents — every other metric leaves this empty. - multi_label_pair_counts: Dict[Tuple[str, str], int] = {} - skipped = 0 - errored = 0 - observed_metric_type: Optional[str] = None - observed_name: Optional[str] = None - - # ``meta`` is a real ``Metric`` row in production, but tests - # frequently pass a lightweight stub. Pull the two attributes - # we need via ``getattr`` so a stub that only sets ``id`` / - # ``name`` / ``metric_type`` doesn't blow up here. - is_multi_label_parent = bool( - meta - and getattr(meta, "selection_mode", None) == "multi_label" - and not getattr(meta, "parent_metric_id", None) - ) - - for row in eval_rows: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else {} - ) - entry = scores.get(metric_id_str) - if not isinstance(entry, dict): - continue - if entry.get("metric_name"): - observed_name = entry.get("metric_name") - if entry.get("type"): - observed_metric_type = entry.get("type") - if entry.get("skipped"): - skipped += 1 - continue - if entry.get("error"): - errored += 1 - continue - - # Multi-label parents store a comma-joined value that - # isn't useful as a single category; instead tally each - # selected child individually so the chart shows per-label - # counts that mirror the children's own boolean histograms. - if is_multi_label_parent: - selected = entry.get("selected_child_names") - if isinstance(selected, list) and selected: - multi_label_rows_scored += 1 - cleaned: List[str] = [] - for label in selected: - text_label = str(label).strip() or None - if text_label: - cleaned.append(text_label) - category_counts[text_label] = ( - category_counts.get(text_label, 0) + 1 - ) - # Emit one increment per unordered pair of distinct - # labels that fired together on this row. ``cleaned`` - # is deduplicated first because the LLM occasionally - # repeats a label inside ``selected_child_names``. - distinct = sorted(set(cleaned)) - for i in range(len(distinct)): - for j in range(i + 1, len(distinct)): - pair = (distinct[i], distinct[j]) - multi_label_pair_counts[pair] = ( - multi_label_pair_counts.get(pair, 0) + 1 - ) - continue - - value = entry.get("value") - numeric = _coerce_numeric(value) - if numeric is not None: - numeric_values.append(numeric) - continue - category = _coerce_category(value) - if category is not None: - category_counts[category] = category_counts.get(category, 0) + 1 - - # ``count`` is "rows scored". For numeric / single-choice - # metrics that's the same as ``len(numeric) + sum(categories)`` - # because each scored row contributes exactly one observation. - # Multi-label parents however contribute one observation per - # selected child, so summing ``category_counts`` over-counts — - # we tracked rows-scored separately above and use it here. - rows_scored = ( - multi_label_rows_scored - if is_multi_label_parent - else len(numeric_values) + sum(category_counts.values()) - ) - - # Build numeric stats first, then categorical (both can coexist). - agg = CallImportMetricAggregate( - metric_id=metric_id_str, - metric_name=( - (meta.name if meta else observed_name) or "Unknown metric" - ), - metric_type=( - meta.metric_type if meta else observed_metric_type - ), - metric_category=( - "user_insight" - if meta is not None and _metric_is_user_insight(meta) - else "quality" - ) - or "quality", - is_multi_label_parent=is_multi_label_parent, - count=rows_scored, - skipped_count=skipped, - error_count=errored, - ) - if numeric_values: - agg.mean = float(statistics.fmean(numeric_values)) - agg.median = float(statistics.median(numeric_values)) - agg.min = min(numeric_values) - agg.max = max(numeric_values) - agg.stddev = ( - float(statistics.pstdev(numeric_values)) - if len(numeric_values) > 1 - else 0.0 - ) - agg.p25 = _percentile(numeric_values, 25) - agg.p75 = _percentile(numeric_values, 75) - agg.p95 = _percentile(numeric_values, 95) - agg.histogram_buckets = _build_histogram(numeric_values) - if category_counts: - sorted_counts = sorted( - category_counts.items(), key=lambda kv: kv[1], reverse=True - ) - agg.value_counts = [ - CallImportMetricValueCount(label=label, count=count) - for label, count in sorted_counts[:_TOP_VALUE_COUNTS] - ] - # Restrict the heatmap to pairs of labels we actually - # rendered above so the frontend never has to match - # against truncated/missing rows. Sorted desc by pair - # count to keep the most informative cells in the - # response when ``_TOP_VALUE_COUNTS`` clipped the matrix. - if is_multi_label_parent and multi_label_pair_counts: - kept_labels = { - label for label, _ in sorted_counts[:_TOP_VALUE_COUNTS] - } - pair_items = [ - (a, b, count) - for (a, b), count in multi_label_pair_counts.items() - if a in kept_labels and b in kept_labels - ] - pair_items.sort(key=lambda t: t[2], reverse=True) - agg.co_occurrence = [ - CallImportMetricLabelPair(a=a, b=b, count=count) - for a, b, count in pair_items - ] - - results.append(agg) - - # Sort so each parent metric immediately precedes its children. - # The Visualizations grid renders metrics top-to-bottom in this - # order, so multi-label parents (the "summary" chart) sit above - # the per-child boolean histograms that drill into them. Metrics - # whose ``meta`` row was deleted mid-run (``meta is None``) sink - # to the bottom but keep their relative order. - enumerated = list(enumerate(results)) - - def _sort_key(item: Tuple[int, CallImportMetricAggregate]): - original_idx, agg = item - meta = metric_meta.get(agg.metric_id) - if meta is None: - return (1, "", 1, "", original_idx) - parent_id = getattr(meta, "parent_metric_id", None) - # Group key: a child shares its parent's UUID; a parent - # uses its own UUID. Within a group, depth=0 (parent) sorts - # before depth=1 (child); ties break alphabetically by name - # so children render in a stable order regardless of which - # row scored which label first. - if parent_id is None: - group_key = str(meta.id) - depth = 0 - else: - group_key = str(parent_id) - depth = 1 - return ( - 0, - group_key, - depth, - (getattr(meta, "name", "") or "").lower(), - original_idx, - ) - - enumerated.sort(key=_sort_key) - return [agg for _idx, agg in enumerated] - - -@router.get( - "/{eval_id}/aggregate", - response_model=CallImportEvaluationAggregateResponse, - operation_id="getCallImportEvaluationAggregate", -) -async def get_call_import_evaluation_aggregate( - call_import_id: UUID, - eval_id: UUID, - baseline_evaluation_id: Optional[UUID] = Query(None), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationAggregateResponse: - """Return per-metric distributions for the Visualizations tab. - - The shape is intentionally chart-friendly: histograms for numeric - metrics, top-N value counts for categorical/text metrics, plus - summary stats (mean/p50/p95) so the UI can render summary cards - without recomputing on the client. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - eval_rows = _load_eval_rows(db, eval_id) - - metrics = _compute_metric_aggregates(db, evaluation, eval_rows) - - period_deltas: dict[str, MetricPeriodDelta] = {} - resolved_baseline_id: Optional[UUID] = None - if baseline_evaluation_id is not None: - call_import = _require_import(db, call_import_id, organization_id) - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - - rows = load_evaluation_row_pairs(db, eval_id) - period_start, _, _, _ = _report_period_from_rows(rows) - baseline_evaluation = _resolve_baseline_evaluation( - db, - organization_id, - call_import.workspace_id, - evaluation, - period_start, - str(baseline_evaluation_id), - ) - if baseline_evaluation: - resolved_baseline_id = baseline_evaluation.id - metric_aggregates_dicts = [ - _aggregate_to_dict(agg) for agg in metrics - ] - raw_deltas = _period_deltas_from_evaluation( - db, - baseline_evaluation, - metric_aggregates_dicts, - evaluation, - eval_rows, - ) - raw_deltas = _period_deltas_with_explanations( - db, - organization_id, - evaluation, - baseline_evaluation, - raw_deltas, - ) - period_deltas = { - metric_id: MetricPeriodDelta( - label=delta.get("label") or "", - detail=delta.get("detail") or "", - why=(delta.get("why") or "").strip() or None, - ) - for metric_id, delta in raw_deltas.items() - } - - _fp_stored, failure_policies_source = policies_from_evaluation_raw( - evaluation.metric_clusters - ) - return CallImportEvaluationAggregateResponse( - evaluation_id=eval_id, - total_rows=evaluation.total_rows, - completed_rows=evaluation.completed_rows, - failed_rows=evaluation.failed_rows, - metrics=metrics, - period_deltas=period_deltas, - baseline_evaluation_id=resolved_baseline_id, - failure_policies_source=failure_policies_source, - ) - - -# --------------------------------------------------------------------------- -# TLDR insights: LLM-generated narrative + bullet patterns rendered above -# the Visualizations charts. Cached on ``CallImportEvaluation.tldr_summary`` -# so the page never auto-burns LLM tokens; the user explicitly clicks -# "Generate summary" or "Regenerate" from the empty-state CTA. -# --------------------------------------------------------------------------- - - -_INSIGHTS_SYSTEM_PROMPT = ( - "You are a senior conversation-analytics reviewer. You will be " - "given aggregated metric statistics + a sample of rationales for " - "the rows of a single call-import evaluation. Identify the most " - "useful PATTERNS that hold ACROSS the calls -- not just per-metric " - "numbers. Look for combinations (e.g. `when X happens, Y also " - "tends to happen`), notable outliers, frequent failure modes, and " - "any signal that would change how a reviewer triages the run.\n\n" - "Return STRICT JSON only, with this shape and no extra keys:\n" - "{\n" - ' "narrative": "",\n' - ' "patterns": ["", "", ...],\n' - ' "metric_insights": {"": "<2-3 line business meaning>"}\n' - "}\n\n" - "Constraints:\n" - "- narrative is the ONLY text shown in the external audit summary and " - "Visualizations TLDR; keep it to at most 3 short sentences (~300 chars).\n" - "- patterns are optional supporting notes and are NOT rendered in the " - "audit summary; keep 0 to 3 bullets if supplied, each <= 120 characters.\n" - "- metric_insights must include one entry for each top-level metric id supplied.\n" - "- Each metric insight should explain what the metric means for the business and what the current distribution suggests, not restate the metric rubric.\n" - "- Avoid restating raw counts unless they reveal a pattern.\n" - "- Use neutral, factual language ('frustration appeared in...') " - "rather than judgemental ('the agents failed to...')." -) - - -def _tldr_summary_payload( - evaluation: CallImportEvaluation, -) -> Optional[EvaluationTldrSummary]: - """Return the cached TLDR (with ``is_stale`` set) or ``None``. - - ``CallImportEvaluation.tldr_summary`` is a ``JSON`` column so we - have to validate shape defensively -- a half-written or hand-edited - blob should not break the aggregate response. Returns ``None`` when - no cached summary exists. - """ - raw = evaluation.tldr_summary - if not isinstance(raw, dict): - return None - narrative = raw.get("narrative") - if not isinstance(narrative, str) or not narrative.strip(): - return None - patterns_raw = raw.get("patterns") - patterns = ( - [str(p) for p in patterns_raw if isinstance(p, str) and p.strip()] - if isinstance(patterns_raw, list) - else [] - ) - metric_insights_raw = raw.get("metric_insights") - metric_insights = ( - { - str(metric_id): str(insight).strip() - for metric_id, insight in metric_insights_raw.items() - if str(metric_id).strip() - and isinstance(insight, str) - and insight.strip() - } - if isinstance(metric_insights_raw, dict) - else {} - ) - generated_at_raw = raw.get("generated_at") - try: - generated_at = ( - datetime.fromisoformat(generated_at_raw) - if isinstance(generated_at_raw, str) - else evaluation.updated_at or datetime.now(timezone.utc) - ) - except ValueError: - generated_at = evaluation.updated_at or datetime.now(timezone.utc) - snapshot = raw.get("generated_at_completed_rows") - snapshot_int = int(snapshot) if isinstance(snapshot, (int, float)) else 0 - return EvaluationTldrSummary( - narrative=_clamp_prose_to_sentences(narrative.strip()), - patterns=patterns, - metric_insights=metric_insights, - generated_at=generated_at, - generated_at_completed_rows=snapshot_int, - provider=raw.get("provider") if isinstance(raw.get("provider"), str) else None, - model=raw.get("model") if isinstance(raw.get("model"), str) else None, - is_stale=evaluation.completed_rows > snapshot_int, - ) - - -def _sample_rationales_per_metric( - eval_rows: List[CallImportEvaluationRow], - *, - per_metric_cap: int = 3, - rationale_char_cap: int = 600, -) -> Dict[str, List[str]]: - """Collect up to ``per_metric_cap`` distinct rationales per metric. - - Distinctness is case- and whitespace-insensitive. We truncate each - rationale to ``rationale_char_cap`` so a few unusually verbose rows - can't dominate the prompt budget. Empty / non-string rationales are - skipped. - """ - out: Dict[str, List[str]] = {} - seen: Dict[str, set[str]] = {} - for row in eval_rows: - scores = row.metric_scores if isinstance(row.metric_scores, dict) else {} - for metric_id, entry in scores.items(): - if not isinstance(entry, dict): - continue - rationale = entry.get("rationale") - if not isinstance(rationale, str): - continue - text = rationale.strip() - if not text: - continue - bucket = out.setdefault(metric_id, []) - if len(bucket) >= per_metric_cap: - continue - key = " ".join(text.lower().split()) - seen_set = seen.setdefault(metric_id, set()) - if key in seen_set: - continue - seen_set.add(key) - bucket.append(text[:rationale_char_cap]) - return out - - -def _build_insights_messages( - evaluation: CallImportEvaluation, - aggregate: List[CallImportMetricAggregate], - rationale_samples: Dict[str, List[str]], - metric_meta: Dict[str, Metric], -) -> List[Dict[str, str]]: - """Render the user prompt fed to the LLM. - - The shape is plain markdown-ish text instead of JSON so the LLM can - skim it without us spending tokens on verbose schema delimiters. - Parent metrics surface their child metrics nested underneath so the - model sees the hierarchy and can talk about "X often co-occurred - with Y" rather than treating sub-labels as standalone metrics. - """ - name = evaluation.name or f"Run {str(evaluation.id)[:8]}" - lines: List[str] = [ - f"Evaluation: {name}", - ( - f"Rows: total={evaluation.total_rows} " - f"completed={evaluation.completed_rows} " - f"failed={evaluation.failed_rows}" - ), - "", - "## Per-metric aggregate", - ] - - # Group metrics by parent so the prompt mirrors the hierarchy. Any - # aggregate row whose ``metric_id`` is missing from ``metric_meta`` - # is rendered as a leaf at the top-level list (handles renamed / - # deleted parents). - children_by_parent: Dict[str, List[CallImportMetricAggregate]] = {} - top_level: List[CallImportMetricAggregate] = [] - for agg in aggregate: - meta = metric_meta.get(agg.metric_id) - parent_id = ( - str(meta.parent_metric_id) - if meta is not None and getattr(meta, "parent_metric_id", None) - else None - ) - if parent_id: - children_by_parent.setdefault(parent_id, []).append(agg) - else: - top_level.append(agg) - - def _format_metric_block(agg: CallImportMetricAggregate, indent: int) -> List[str]: - prefix = " " * indent + "- " - bits: List[str] = [f"{prefix}{agg.metric_name} [id={agg.metric_id}] (n={agg.count}"] - if agg.skipped_count: - bits.append(f", skipped={agg.skipped_count}") - if agg.error_count: - bits.append(f", errors={agg.error_count}") - bits.append(")") - meta = metric_meta.get(agg.metric_id) - description = (meta.description or "").strip() if meta else "" - if description: - bits.append(f" | definition={description[:500]}") - if agg.mean is not None: - mean_s = f"{agg.mean:.2f}" - stddev_s = f"{agg.stddev:.2f}" if agg.stddev is not None else "-" - bits.append(f" | mean={mean_s} stddev={stddev_s}") - if agg.min is not None and agg.max is not None: - bits.append(f" range=[{agg.min:.2f}, {agg.max:.2f}]") - if agg.value_counts: - total = sum(v.count for v in agg.value_counts) or 1 - top = agg.value_counts[:3] - shares = ", ".join( - f'"{v.label}"={v.count}/{total}' for v in top - ) - bits.append(f" | top={shares}") - result = ["".join(bits)] - rationales = rationale_samples.get(agg.metric_id, []) - for r in rationales: - result.append(" " * (indent + 1) + f"- rationale: {r}") - return result - - for agg in top_level: - lines.extend(_format_metric_block(agg, indent=0)) - meta = metric_meta.get(agg.metric_id) - children = children_by_parent.get(str(meta.id), []) if meta else [] - for child in children: - lines.extend(_format_metric_block(child, indent=1)) - - lines.append("") - top_level_ids = [agg.metric_id for agg in top_level] - if top_level_ids: - lines.append( - "metric_insights keys must exactly use these top-level metric ids: " - + ", ".join(top_level_ids) - ) - lines.append("") - lines.append( - "Write the JSON object as instructed. Do not include " - "preamble, code fences, or trailing commentary." - ) - - return [ - {"role": "system", "content": _INSIGHTS_SYSTEM_PROMPT}, - {"role": "user", "content": "\n".join(lines)}, - ] - - -def _parse_insights_response(text: str) -> EvaluationTldrSummary: - """Coerce the LLM response into ``narrative`` + ``patterns``. - - Matches the JSON-with-fallback pattern used by - ``app.api.v1.routes.metrics._parse_metric_generation_response``: try - ``json.loads`` first, then fall back to regex extraction of the - first ``{...}`` block. Raises ``HTTPException`` with a 502 when the - response can't be parsed at all. - """ - cleaned = (text or "").strip() - if not cleaned: - raise HTTPException( - status_code=502, detail="LLM returned an empty insights response" - ) - try: - parsed = json.loads(cleaned) - except json.JSONDecodeError: - import re - - match = re.search(r"\{.*\}", cleaned, re.DOTALL) - if not match: - raise HTTPException( - status_code=502, - detail="Could not parse LLM insights response as JSON", - ) - try: - parsed = json.loads(match.group(0)) - except json.JSONDecodeError as e: - raise HTTPException( - status_code=502, - detail=f"Could not parse LLM insights response: {e}", - ) - - if not isinstance(parsed, dict): - raise HTTPException( - status_code=502, detail="LLM insights JSON was not an object" - ) - - narrative = parsed.get("narrative") - if not isinstance(narrative, str) or not narrative.strip(): - raise HTTPException( - status_code=502, - detail="LLM insights JSON missing 'narrative' string", - ) - - patterns_raw = parsed.get("patterns") - if patterns_raw is None: - patterns: List[str] = [] - elif isinstance(patterns_raw, list): - patterns = [ - str(p).strip() - for p in patterns_raw - if isinstance(p, str) and p.strip() - ] - else: - raise HTTPException( - status_code=502, - detail="LLM insights JSON 'patterns' must be a list of strings", - ) - metric_insights_raw = parsed.get("metric_insights") - if metric_insights_raw is None: - metric_insights: Dict[str, str] = {} - elif isinstance(metric_insights_raw, dict): - metric_insights = { - str(metric_id): str(insight).strip() - for metric_id, insight in metric_insights_raw.items() - if str(metric_id).strip() - and isinstance(insight, str) - and insight.strip() - } - else: - raise HTTPException( - status_code=502, - detail="LLM insights JSON 'metric_insights' must be an object", - ) - - return EvaluationTldrSummary( - narrative=_clamp_prose_to_sentences(narrative.strip()), - patterns=patterns, - metric_insights=metric_insights, - generated_at=datetime.now(timezone.utc), - generated_at_completed_rows=0, # filled in by caller - is_stale=False, - ) - - -def _generate_and_persist_tldr_summary( - db: Session, - evaluation: CallImportEvaluation, - *, - organization_id: UUID, - provider: Optional[str] = None, - model: Optional[str] = None, -) -> EvaluationTldrSummary: - """LLM TLDR generation used by the imports-queue Celery worker.""" - eval_id = evaluation.id - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - - pairs = load_evaluation_row_pairs(db, eval_id) - eval_rows = [eval_row for eval_row, _ in pairs] - aggregate = _compute_metric_aggregates(db, evaluation, eval_rows) - if not aggregate: - raise HTTPException( - status_code=400, - detail=( - "No metric data yet. Wait for at least one row to " - "finish scoring before generating a summary." - ), - ) - - metric_ids: List[UUID] = [] - for agg in aggregate: - try: - metric_ids.append(UUID(agg.metric_id)) - except (TypeError, ValueError): - continue - metrics = _metrics_for_ids(db, organization_id, metric_ids) - metric_meta: Dict[str, Metric] = {str(m.id): m for m in metrics} - - rationale_samples = _sample_rationales_per_metric(eval_rows) - messages = _build_insights_messages( - evaluation, aggregate, rationale_samples, metric_meta - ) - - from app.services.ai.llm_resolver import get_llm_provider_and_model - from app.services.ai.llm_service import llm_service - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, provider, model - ) - - try: - llm_result = llm_service.generate_response( - messages=messages, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - temperature=0.4, - max_tokens=1400, - ) - except Exception as e: - logger.error(f"[CallImportInsights] LLM call failed: {e}") - raise HTTPException( - status_code=502, detail=f"LLM call failed: {e}" - ) from e - - summary = _parse_insights_response(llm_result.get("text", "")) - total = int(evaluation.total_rows or 0) - ui_completed = min(int(evaluation.completed_rows or 0), total) if total else int( - evaluation.completed_rows or 0 - ) - summary.generated_at_completed_rows = ui_completed - summary.provider = provider_enum.value - summary.model = model_str - summary.is_stale = False - - evaluation.tldr_summary = { - "narrative": summary.narrative, - "patterns": summary.patterns, - "metric_insights": summary.metric_insights, - "generated_at": summary.generated_at.isoformat(), - "generated_at_completed_rows": summary.generated_at_completed_rows, - "provider": summary.provider, - "model": summary.model, - } - flag_modified(evaluation, "tldr_summary") - db.commit() - db.refresh(evaluation) - return summary - - -@router.get( - "/{eval_id}/insights", - response_model=Optional[EvaluationTldrSummary], - operation_id="getCallImportEvaluationInsights", -) -async def get_call_import_evaluation_insights( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Optional[EvaluationTldrSummary]: - """Return the cached TLDR (or ``null``) without contacting the LLM. - - Used by the Visualizations tab on first paint so the empty-state - CTA can show up before the user opts into generation. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - return _tldr_summary_payload(evaluation) - - -@router.post( - "/{eval_id}/insights", - response_model=EvaluationTldrSummary, - operation_id="generateCallImportEvaluationInsights", -) -async def generate_call_import_evaluation_insights( - call_import_id: UUID, - eval_id: UUID, - body: EvaluationInsightsRequest = Body(default_factory=EvaluationInsightsRequest), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> EvaluationTldrSummary: - """Generate (or return-cached) the LLM TLDR for an evaluation run. - - Behavior: - - * ``body.regenerate=False`` and a cached summary at the current - ``completed_rows`` watermark exists -> return it as-is. - * ``body.regenerate=False`` and a stale cached summary exists - (``generated_at_completed_rows < completed_rows``) -> return it - with ``is_stale=True``; the UI prompts the user to regenerate. - * Otherwise -> resolve provider+model (auto-detect when omitted), - call the LLM, persist the new summary, return it. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - if not body.regenerate: - cached = _tldr_summary_payload(evaluation) - if cached is not None: - return cached - - # Run the TLDR LLM on the imports worker (not the default worker or API). - from app.workers.tasks.generate_evaluation_tldr_insights import ( - generate_evaluation_tldr_insights_task, - ) - - try: - task_result = generate_evaluation_tldr_insights_task.apply_async( - kwargs={ - "evaluation_id": str(eval_id), - "call_import_id": str(call_import_id), - "organization_id": str(organization_id), - "provider": body.provider, - "model": body.model, - }, - ).get(timeout=25 * 60) - except Exception as exc: - logger.error( - "[CallImportInsights] TLDR task failed for evaluation {}: {}", - eval_id, - exc, - ) - raise HTTPException( - status_code=502, - detail=f"Summary generation failed: {exc}", - ) from exc - - if isinstance(task_result, dict) and task_result.get("error"): - status_code = int(task_result.get("status_code") or 502) - raise HTTPException( - status_code=status_code, - detail=str(task_result["error"]), - ) - - summary = EvaluationTldrSummary.model_validate(task_result) - db.refresh(evaluation) - stamp_evaluation_actor(evaluation, principal) - db.commit() - db.refresh(evaluation) - - from app.services.ai.llm_resolver import get_llm_provider_and_model - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, body.provider, body.model, body.credential_id - ) - - _enqueue_user_insights_job( - evaluation, - provider=summary.provider or provider_enum.value, - model=summary.model or model_str, - force=body.regenerate, - max_llm_calls=body.max_llm_calls, - db=db, - principal=principal, - ) - - return summary - - -def _user_insights_payload( - evaluation: CallImportEvaluation, -) -> Optional[EvaluationUserInsightsState]: - raw = getattr(evaluation, "user_insights", None) - if raw is None: - return None - return user_insights_state_from_raw( - raw, - completed_rows=evaluation.completed_rows, - ) - - -def _selected_generated_user_insights( - state: Optional[EvaluationUserInsightsState], - report_config: dict[str, Any], -) -> list[dict[str, Any]]: - """Filter and order generated insights for PDF section 03.""" - if state is None or state.status != "completed" or not state.insights: - return [] - - selected_ids = report_config.get("user_insight_ids") - if isinstance(selected_ids, list) and selected_ids: - allowed = {str(item) for item in selected_ids if item} - items = [item for item in state.insights if item.id in allowed] - else: - items = list(state.insights) - - order_raw = report_config.get("order") - order_ids: list[str] = [] - if isinstance(order_raw, dict): - user_order = order_raw.get("user_insights") - if isinstance(user_order, list): - order_ids = [str(item) for item in user_order if item] - - if order_ids: - by_id = {item.id: item for item in items} - ordered = [by_id[iid] for iid in order_ids if iid in by_id] - seen = set(order_ids) - ordered.extend(item for item in items if item.id not in seen) - items = ordered - - return [item.model_dump(mode="json") for item in items] - - -def _enqueue_user_insights_job( - evaluation: CallImportEvaluation, - *, - provider: Optional[str] = None, - model: Optional[str] = None, - force: bool = False, - max_llm_calls: Optional[int] = None, - db: Optional[Session] = None, - principal: Optional[Principal] = None, -) -> None: - """Enqueue background user-insights generation unless already running.""" - current = _user_insights_payload(evaluation) - if current is not None and current.status == "running" and not force: - return - - llm_budget = normalize_max_llm_calls(max_llm_calls) - - completed_count = ( - _count_completed_eval_rows(db, evaluation.id) - if db is not None - else evaluation.completed_rows - ) - total_calls = total_llm_calls_for_rows(completed_count, max_llm_calls=llm_budget) - evaluation.user_insights = { - "status": "running", - "insights": ( - (evaluation.user_insights or {}).get("insights", []) - if isinstance(evaluation.user_insights, dict) - else [] - ), - "generated_at": datetime.now(timezone.utc).isoformat(), - "generated_at_completed_rows": evaluation.completed_rows, - "progress": {"completed_llm_calls": 0, "total_llm_calls": total_calls}, - "provider": provider, - "model": model, - "max_llm_calls": llm_budget, - "llm_calls_used": 0, - "error_message": None, - } - if db is not None: - if principal is not None: - stamp_evaluation_actor(evaluation, principal) - flag_modified(evaluation, "user_insights") - db.commit() - - from app.workers.tasks.generate_evaluation_user_insights import ( - generate_evaluation_user_insights_task, - ) - - generate_evaluation_user_insights_task.delay( - str(evaluation.id), - provider=provider, - model=model, - max_llm_calls=llm_budget, - ) - - -@router.get( - "/{eval_id}/user-insights", - response_model=Optional[EvaluationUserInsightsState], - operation_id="getCallImportEvaluationUserInsights", -) -async def get_call_import_evaluation_user_insights( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Optional[EvaluationUserInsightsState]: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - return _user_insights_payload(evaluation) - - -@router.post( - "/{eval_id}/user-insights", - response_model=EvaluationUserInsightsState, - operation_id="generateCallImportEvaluationUserInsights", -) -async def generate_call_import_evaluation_user_insights( - call_import_id: UUID, - eval_id: UUID, - body: EvaluationUserInsightsRequest = Body( - default_factory=EvaluationUserInsightsRequest - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> EvaluationUserInsightsState: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - if not body.regenerate and not body.force: - cached = _user_insights_payload(evaluation) - if cached is not None and cached.status in {"running", "completed"}: - return cached - - eval_rows = _load_eval_rows(db, eval_id) - if not any(row.status == "completed" for row in eval_rows): - raise HTTPException( - status_code=400, - detail=( - "No completed rows yet. Wait for at least one row to " - "finish scoring before generating user insights." - ), - ) - - from app.services.ai.llm_resolver import get_llm_provider_and_model - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, body.provider, body.model, body.credential_id - ) - - _enqueue_user_insights_job( - evaluation, - provider=provider_enum.value, - model=model_str, - force=body.force or body.regenerate, - max_llm_calls=body.max_llm_calls, - db=db, - principal=principal, - ) - - db.refresh(evaluation) - return _user_insights_payload(evaluation) or EvaluationUserInsightsState( - status="running" - ) - - -def _metric_clusters_payload( - evaluation: CallImportEvaluation, -) -> Optional[EvaluationMetricClustersState]: - raw = getattr(evaluation, "metric_clusters", None) - if raw is None: - return None - return metric_clusters_state_from_raw( - raw, - completed_rows=evaluation.completed_rows, - ) - - -def _selected_metric_clusters_for_pdf( - state: Optional[EvaluationMetricClustersState], - report_config: dict[str, Any], -) -> dict[str, Any]: - if state is None or state.status != "completed": - return {} - sections = report_config.get("sections") - if isinstance(sections, dict) and sections.get("failure_diagnostics") is False: - return {} - payload: dict[str, Any] = { - "groups": [g.model_dump(mode="json") for g in state.groups], - "discovered_problems": [ - d.model_dump(mode="json") for d in state.discovered_problems - ], - } - if state.rca_summary is not None: - payload["rca_summary"] = state.rca_summary.model_dump(mode="json") - return payload - - -def _prompt_improvements_payload( - evaluation: CallImportEvaluation, -) -> Optional[EvaluationPromptImprovementsState]: - from app.services.call_import_prompt_improvements import ( - prompt_improvements_state_from_raw, - ) - - raw = getattr(evaluation, "prompt_improvements", None) - if raw is None: - return None - return prompt_improvements_state_from_raw( - raw, - completed_rows=evaluation.completed_rows, - ) - - -def _selected_prompt_improvements_for_pdf( - state: Optional[EvaluationPromptImprovementsState], - report_config: dict[str, Any], -) -> dict[str, Any]: - if state is None or state.status != "completed": - return {} - sections = report_config.get("sections") - if isinstance(sections, dict) and sections.get("prompt_improvements") is False: - return {} - return { - "imported_agent_id": state.imported_agent_id, - "imported_agent_name": state.imported_agent_name, - "overview": state.overview, - "suggestions": [s.model_dump(mode="json") for s in state.suggestions], - } - - -def _enqueue_prompt_improvements_job( - evaluation: CallImportEvaluation, - *, - imported_agent_id: UUID, - imported_agent_name: str, - provider: Optional[str] = None, - model: Optional[str] = None, - credential_id: Optional[UUID] = None, - force: bool = False, - db: Optional[Session] = None, - principal: Optional[Principal] = None, -) -> None: - current = _prompt_improvements_payload(evaluation) - if current is not None and current.status == "running" and not force: - return - - evaluation.prompt_improvements = { - "status": "running", - "imported_agent_id": str(imported_agent_id), - "imported_agent_name": imported_agent_name, - "suggestions": [], - "generated_at": datetime.now(timezone.utc).isoformat(), - "generated_at_completed_rows": evaluation.completed_rows, - "provider": provider, - "model": model, - "error_message": None, - } - if db is not None: - if principal is not None: - stamp_evaluation_actor(evaluation, principal) - flag_modified(evaluation, "prompt_improvements") - db.commit() - - from app.workers.tasks.generate_evaluation_prompt_improvements import ( - generate_evaluation_prompt_improvements_task, - ) - - async_result = generate_evaluation_prompt_improvements_task.apply_async( - kwargs={ - "evaluation_id": str(evaluation.id), - "imported_agent_id": str(imported_agent_id), - "provider": provider, - "model": model, - "credential_id": str(credential_id) if credential_id else None, - }, - queue="imports", - ) - if db is not None and isinstance(evaluation.prompt_improvements, dict): - evaluation.prompt_improvements["celery_task_id"] = async_result.id - flag_modified(evaluation, "prompt_improvements") - db.commit() - - -def _load_eval_rows(db: Session, evaluation_id: UUID) -> List[CallImportEvaluationRow]: - from app.db_sharding.eval_rows import load_evaluation_rows_for_run - - return load_evaluation_rows_for_run(db, evaluation_id) - - -def _count_completed_eval_rows(db: Session, evaluation_id: UUID) -> int: - from app.db_sharding.eval_rows import count_evaluation_rows_for_run - - return count_evaluation_rows_for_run( - db, evaluation_id, statuses=["completed"] - ) - - -def _completed_row_pairs_for_evaluation( - db: Session, - evaluation_id: UUID, -) -> List[Tuple[CallImportEvaluationRow, CallImportRow]]: - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - - row_pairs = load_evaluation_row_pairs(db, evaluation_id) - return [ - (eval_row, source_row) - for eval_row, source_row in row_pairs - if eval_row.status == "completed" - ] - - -def _resolve_metric_cluster_row_selection( - db: Session, - evaluation: CallImportEvaluation, - eval_rows: List[CallImportEvaluationRow], - evaluation_row_ids: Optional[List[UUID]], - *, - row_limit: Optional[int] = None, - policies: Optional[Dict[str, MetricFailurePolicy]] = None, -) -> Tuple[List[Tuple[CallImportEvaluationRow, CallImportRow]], List[str]]: - """Return filtered completed row pairs and the selected row id strings.""" - completed_pairs = _completed_row_pairs_for_evaluation(db, evaluation.id) - metrics = _metrics_for_clustering(db, evaluation, eval_rows) - if policies is None: - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - parent_ids = [ - m.id - for m in metrics - if getattr(m, "selection_mode", None) - and not getattr(m, "parent_metric_id", None) - ] - child_names_by_parent = _child_names_by_parent( - db, evaluation.organization_id, parent_ids - ) - policies, _ = effective_policies( - evaluation, - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - ) - eligible = list_eligible_cluster_rows( - evaluation, completed_pairs, metrics, policies - ) - eligible_ordered_ids = [str(item["evaluation_row_id"]) for item in eligible] - eligible_id_set = set(eligible_ordered_ids) - - if evaluation_row_ids is None and row_limit is not None: - selected_ids = eligible_ordered_ids[:row_limit] - filtered = filter_completed_row_pairs( - completed_pairs, - [UUID(rid) for rid in selected_ids], - ) - return filtered, selected_ids - - if evaluation_row_ids is None: - selected_ids = eligible_ordered_ids - filtered = filter_completed_row_pairs( - completed_pairs, - [UUID(rid) for rid in selected_ids], - ) - return filtered, selected_ids - - requested = {str(rid) for rid in evaluation_row_ids} - completed_id_set = {str(eval_row.id) for eval_row, _ in completed_pairs} - unknown = sorted(requested - completed_id_set) - if unknown: - raise HTTPException( - status_code=400, - detail=( - "One or more evaluation_row_ids are missing or not completed: " - + ", ".join(unknown[:5]) - + ("…" if len(unknown) > 5 else "") - ), - ) - not_eligible = sorted(requested - eligible_id_set) - if not_eligible: - raise HTTPException( - status_code=400, - detail=( - "Each selected row must have at least one flagged quality metric. " - "Ineligible row(s): " - + ", ".join(not_eligible[:5]) - + ("…" if len(not_eligible) > 5 else "") - ), - ) - selected_ids = sorted(requested) - filtered = filter_completed_row_pairs(completed_pairs, evaluation_row_ids) - return filtered, selected_ids - - -def _enqueue_metric_clusters_job( - evaluation: CallImportEvaluation, - *, - provider: Optional[str] = None, - model: Optional[str] = None, - credential_id: Optional[UUID] = None, - force: bool = False, - max_llm_calls: Optional[int] = None, - evaluation_row_ids: Optional[List[UUID]] = None, - selected_evaluation_row_ids: Optional[List[str]] = None, - failure_policies: Optional[Dict[str, MetricFailurePolicy]] = None, - db: Optional[Session] = None, - principal: Optional[Principal] = None, -) -> None: - current = _metric_clusters_payload(evaluation) - if current is not None and current.status == "running" and not force: - return - - llm_budget = normalize_max_llm_calls(max_llm_calls) - total_calls = 1 - row_ids_for_task: Optional[List[str]] = None - if db is not None: - eval_rows = _load_eval_rows(db, evaluation.id) - if selected_evaluation_row_ids is None: - _, selected_evaluation_row_ids = _resolve_metric_cluster_row_selection( - db, - evaluation, - eval_rows, - evaluation_row_ids, - ) - completed_pairs = filter_completed_row_pairs( - _completed_row_pairs_for_evaluation(db, evaluation.id), - [UUID(rid) for rid in selected_evaluation_row_ids], - ) - metrics = _metrics_for_clustering(db, evaluation, eval_rows) - policies_for_estimate = failure_policies - if policies_for_estimate is None: - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - parent_ids = [ - m.id - for m in metrics - if getattr(m, "selection_mode", None) - and not getattr(m, "parent_metric_id", None) - ] - child_names_by_parent = _child_names_by_parent( - db, evaluation.organization_id, parent_ids - ) - policies_for_estimate, _ = effective_policies( - evaluation, - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - ) - _, total_calls = estimate_metric_clusters_llm_calls( - evaluation, - metrics, - completed_pairs, - policies_for_estimate, - max_llm_calls=llm_budget, - ) - row_ids_for_task = list(selected_evaluation_row_ids) - - prior_raw = ( - evaluation.metric_clusters - if isinstance(evaluation.metric_clusters, dict) - else {} - ) - policy_blob: Dict[str, Any] = {} - if failure_policies: - policy_blob = failure_policies_to_db(failure_policies, source="user") - - evaluation.metric_clusters = { - "status": "running", - "groups": prior_raw.get("groups", []) if isinstance(prior_raw, dict) else [], - "discovered_problems": ( - prior_raw.get("discovered_problems", []) - if isinstance(prior_raw, dict) - else [] - ), - "generated_at": datetime.now(timezone.utc).isoformat(), - "generated_at_completed_rows": evaluation.completed_rows, - "progress": {"completed_llm_calls": 0, "total_llm_calls": total_calls}, - "provider": provider, - "model": model, - "max_llm_calls": llm_budget, - "llm_calls_used": 0, - "error_message": None, - "selected_evaluation_row_ids": selected_evaluation_row_ids or [], - **policy_blob, - } - if db is not None: - if principal is not None: - stamp_evaluation_actor(evaluation, principal) - flag_modified(evaluation, "metric_clusters") - db.commit() - - from app.workers.tasks.generate_evaluation_metric_clusters import ( - generate_evaluation_metric_clusters_task, - ) - - async_result = generate_evaluation_metric_clusters_task.apply_async( - kwargs={ - "evaluation_id": str(evaluation.id), - "provider": provider, - "model": model, - "credential_id": str(credential_id) if credential_id else None, - "max_llm_calls": llm_budget, - "evaluation_row_ids": row_ids_for_task, - }, - queue="imports", - ) - if db is not None and isinstance(evaluation.metric_clusters, dict): - evaluation.metric_clusters["celery_task_id"] = async_result.id - flag_modified(evaluation, "metric_clusters") - if principal is not None: - stamp_evaluation_actor(evaluation, principal) - db.commit() - - -def _revoke_metric_clusters_task(evaluation: CallImportEvaluation) -> None: - """Best-effort SIGTERM revoke of the in-flight clustering Celery task.""" - raw = evaluation.metric_clusters - if not isinstance(raw, dict): - return - task_id = str(raw.get("celery_task_id") or "").strip() - if not task_id: - return - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke(task_id, terminate=True, signal="SIGTERM") - logger.info( - "Revoked metric-clusters task {} for evaluation {}", - task_id, - evaluation.id, - ) - except Exception as exc: # noqa: BLE001 - logger.warning( - "Failed to revoke metric-clusters task {} for evaluation {}: {}", - task_id, - evaluation.id, - exc, - ) - - -def _apply_metric_clusters_cancel(evaluation: CallImportEvaluation) -> bool: - """Mark clustering as cancelled and revoke the worker task. - - Returns True if a running job was cancelled, False if already terminal. - """ - raw = evaluation.metric_clusters - if not isinstance(raw, dict): - return False - if (raw.get("status") or "").lower() != "running": - return False - - _revoke_metric_clusters_task(evaluation) - progress = raw.get("progress") if isinstance(raw.get("progress"), dict) else {} - evaluation.metric_clusters = { - **raw, - "status": "cancelled", - "error_message": METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, - "progress": progress, - "celery_task_id": None, - } - return True - - -@router.get( - "/{eval_id}/metric-clusters/failure-policies", - response_model=MetricFailurePoliciesResponse, - operation_id="getCallImportEvaluationMetricClusterFailurePolicies", -) -async def get_call_import_evaluation_metric_cluster_failure_policies( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> MetricFailurePoliciesResponse: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - eval_rows = _load_eval_rows(db, eval_id) - metrics, aggregates, policies, source, child_names_by_parent = _clustering_context( - db, evaluation, eval_rows - ) - previews = build_failure_policy_previews( - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - effective=policies, - ) - updated_at = None - raw_mc = evaluation.metric_clusters - if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): - try: - updated_at = datetime.fromisoformat( - str(raw_mc["failure_policies_updated_at"]) - ) - except ValueError: - updated_at = None - return MetricFailurePoliciesResponse( - previews=previews, - policies=policies, - source=source, - updated_at=updated_at, - ) - - -@router.put( - "/{eval_id}/metric-clusters/failure-policies", - response_model=MetricFailurePoliciesResponse, - operation_id="saveCallImportEvaluationMetricClusterFailurePolicies", -) -async def save_call_import_evaluation_metric_cluster_failure_policies( - call_import_id: UUID, - eval_id: UUID, - body: MetricFailurePoliciesSaveRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> MetricFailurePoliciesResponse: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - eval_rows = _load_eval_rows(db, eval_id) - metrics, aggregates, _existing, _source, child_names_by_parent = _clustering_context( - db, evaluation, eval_rows - ) - try: - validate_failure_policies_for_metrics(body.policies, metrics) - except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc - - prior = ( - evaluation.metric_clusters - if isinstance(evaluation.metric_clusters, dict) - else {} - ) - evaluation.metric_clusters = merge_failure_policies_into_raw( - prior, - body.policies, - source="user", - ) - flag_modified(evaluation, "metric_clusters") - stamp_evaluation_actor(evaluation, principal) - db.commit() - db.refresh(evaluation) - - policies, source = policies_from_evaluation_raw(evaluation.metric_clusters) - if source != "user": - source = "user" - previews = build_failure_policy_previews( - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - effective=policies, - ) - updated_at = None - raw_mc = evaluation.metric_clusters - if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): - try: - updated_at = datetime.fromisoformat( - str(raw_mc["failure_policies_updated_at"]) - ) - except ValueError: - updated_at = None - return MetricFailurePoliciesResponse( - previews=previews, - policies=policies, - source="user", - updated_at=updated_at, - ) - - -@router.get( - "/{eval_id}/metric-clusters/eligible-rows", - response_model=MetricClusterEligibleRowsResponse, - operation_id="listCallImportEvaluationMetricClusterEligibleRows", -) -async def list_call_import_evaluation_metric_cluster_eligible_rows( - call_import_id: UUID, - eval_id: UUID, - limit: Optional[int] = Query(default=None, ge=1), - count_only: bool = Query(default=False), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> MetricClusterEligibleRowsResponse: - """Completed rows that have at least one flagged quality metric.""" - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - eval_rows = _load_eval_rows(db, eval_id) - completed_pairs = _completed_row_pairs_for_evaluation(db, eval_id) - metrics, _aggregates, policies, _source, _child_map = _clustering_context( - db, evaluation, eval_rows - ) - all_eligible = list_eligible_cluster_rows( - evaluation, completed_pairs, metrics, policies - ) - total = len(all_eligible) - if count_only: - return MetricClusterEligibleRowsResponse(items=[], total=total) - raw_items = all_eligible if limit is None else all_eligible[:limit] - items = [MetricClusterEligibleRow.model_validate(item) for item in raw_items] - return MetricClusterEligibleRowsResponse(items=items, total=total) - - -@router.get( - "/{eval_id}/metric-clusters", - response_model=Optional[EvaluationMetricClustersState], - operation_id="getCallImportEvaluationMetricClusters", -) -async def get_call_import_evaluation_metric_clusters( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Optional[EvaluationMetricClustersState]: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - return _metric_clusters_payload(evaluation) - - -@router.post( - "/{eval_id}/metric-clusters", - response_model=EvaluationMetricClustersState, - operation_id="generateCallImportEvaluationMetricClusters", -) -async def generate_call_import_evaluation_metric_clusters( - call_import_id: UUID, - eval_id: UUID, - body: EvaluationMetricClustersRequest = Body( - default_factory=EvaluationMetricClustersRequest - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> EvaluationMetricClustersState: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - if not body.regenerate and not body.force: - cached = _metric_clusters_payload(evaluation) - if cached is not None and cached.status in {"running", "completed"}: - return cached - - eval_rows = _load_eval_rows(db, eval_id) - if not any(row.status == "completed" for row in eval_rows): - raise HTTPException( - status_code=400, - detail=( - "No completed rows yet. Wait for at least one row to " - "finish scoring before generating metric clusters." - ), - ) - - if body.evaluation_row_ids and body.row_limit is not None: - raise HTTPException( - status_code=400, - detail="Specify either evaluation_row_ids or row_limit, not both.", - ) - - if body.evaluation_row_ids: - completed_pairs = _completed_row_pairs_for_evaluation(db, evaluation.id) - completed_id_set = {str(eval_row.id) for eval_row, _ in completed_pairs} - requested = {str(rid) for rid in body.evaluation_row_ids} - unknown = sorted(requested - completed_id_set) - if unknown: - raise HTTPException( - status_code=400, - detail=( - "One or more evaluation_row_ids are missing or not completed: " - + ", ".join(unknown[:5]) - + ("…" if len(unknown) > 5 else "") - ), - ) - - from app.services.ai.llm_resolver import get_llm_provider_and_model - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, body.provider, body.model, body.credential_id - ) - - metrics, aggregates, _inferred, _source, child_names_by_parent = _clustering_context( - db, evaluation, eval_rows - ) - merged_policies = merge_clustering_policies( - body.failure_policies, - evaluation, - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - ) - try: - validate_failure_policies_for_metrics( - body.failure_policies or merged_policies, metrics - ) - except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc - - if not has_clusterable_metrics(metrics, merged_policies, eval_rows): - raise HTTPException( - status_code=400, - detail=( - "No calls match any failure policy. Select failure values on " - "metrics that have matching rows, or leave metrics with no " - "failures unchecked — they are skipped automatically." - ), - ) - - filtered_pairs, selected_row_ids = _resolve_metric_cluster_row_selection( - db, - evaluation, - eval_rows, - body.evaluation_row_ids, - row_limit=body.row_limit, - policies=merged_policies, - ) - if not selected_row_ids: - raise HTTPException( - status_code=400, - detail=( - "No eligible rows to cluster. Select completed calls that match " - "at least one configured failure policy." - ), - ) - if not filtered_pairs: - raise HTTPException( - status_code=400, - detail="No completed rows match the selected evaluation_row_ids.", - ) - - _enqueue_metric_clusters_job( - evaluation, - provider=provider_enum.value, - model=model_str, - credential_id=body.credential_id, - force=body.force or body.regenerate, - max_llm_calls=body.max_llm_calls, - evaluation_row_ids=body.evaluation_row_ids, - selected_evaluation_row_ids=selected_row_ids, - failure_policies=merged_policies, - db=db, - principal=principal, - ) - - db.refresh(evaluation) - return _metric_clusters_payload(evaluation) or EvaluationMetricClustersState( - status="running" - ) - - -@router.post( - "/{eval_id}/metric-clusters/cancel", - response_model=EvaluationMetricClustersState, - operation_id="cancelCallImportEvaluationMetricClusters", -) -async def cancel_call_import_evaluation_metric_clusters( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> EvaluationMetricClustersState: - """Abort in-flight failure-diagnostics clustering. - - Idempotent: if clustering is not ``running``, returns the current state - unchanged. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - _apply_metric_clusters_cancel(evaluation) - flag_modified(evaluation, "metric_clusters") - stamp_evaluation_actor(evaluation, principal) - db.commit() - db.refresh(evaluation) - - return _metric_clusters_payload(evaluation) or EvaluationMetricClustersState( - status="idle" - ) - - -@router.get( - "/{eval_id}/prompt-improvements", - response_model=Optional[EvaluationPromptImprovementsState], - operation_id="getCallImportEvaluationPromptImprovements", -) -async def get_call_import_evaluation_prompt_improvements( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Optional[EvaluationPromptImprovementsState]: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - return _prompt_improvements_payload(evaluation) - - -@router.post( - "/{eval_id}/prompt-improvements", - response_model=EvaluationPromptImprovementsState, - operation_id="generateCallImportEvaluationPromptImprovements", -) -async def generate_call_import_evaluation_prompt_improvements( - call_import_id: UUID, - eval_id: UUID, - body: EvaluationPromptImprovementsRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> EvaluationPromptImprovementsState: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - clusters = _metric_clusters_payload(evaluation) - if clusters is None or clusters.status != "completed": - raise HTTPException( - status_code=400, - detail=( - "Metric clusters must be completed before generating prompt " - "improvements. Run failure diagnostics first." - ), - ) - - from app.services.call_import_prompt_improvements import is_imported_agent - from app.services.ai.llm_resolver import get_llm_provider_and_model - - imported_agent = ( - db.query(PromptPartial) - .filter( - PromptPartial.id == body.imported_agent_id, - PromptPartial.organization_id == organization_id, - PromptPartial.workspace_id == workspace_id, - ) - .first() - ) - if imported_agent is None or not is_imported_agent(imported_agent): - raise HTTPException( - status_code=404, - detail="Imported agent not found in the active workspace", - ) - - if not body.regenerate and not body.force: - cached = _prompt_improvements_payload(evaluation) - if ( - cached is not None - and cached.status in {"running", "completed"} - and cached.imported_agent_id == str(body.imported_agent_id) - ): - return cached - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, body.provider, body.model, body.credential_id - ) - - _enqueue_prompt_improvements_job( - evaluation, - imported_agent_id=body.imported_agent_id, - imported_agent_name=imported_agent.name, - provider=provider_enum.value, - model=model_str, - credential_id=body.credential_id, - force=body.force or body.regenerate, - db=db, - principal=principal, - ) - - db.refresh(evaluation) - return _prompt_improvements_payload(evaluation) or EvaluationPromptImprovementsState( - status="running", - imported_agent_id=str(body.imported_agent_id), - imported_agent_name=imported_agent.name, - ) - - -# --------------------------------------------------------------------------- -# Flow chart: turns per-row LLM-inferred ``sequence`` arrays into a -# directed graph of (label -> label) transitions across the whole run. -# Powers the aggregate Sankey-style React Flow chart on the evaluation -# overview; per-call flow charts are built client-side from the same -# ``sequence`` field on a single row's metric_scores entry. -# --------------------------------------------------------------------------- - - -_FLOW_TERMINAL_THRESHOLD = 0.2 # Mark as terminal when >=20% of sequences end here. -_FLOW_START_NODE_ID = "__START__" -_DISCOVERED_NODE_PREFIX = "disc:" - - -def _slug_label(value: Any) -> str: - """Lowercase + whitespace-collapse + underscore-join. - - Used everywhere we need a stable key for a metric/label name — - matching the same convention the worker uses when emitting - ``sequence`` entries and discovered keys. - """ - if value is None: - return "" - return "_".join(str(value).strip().lower().split()) - - -def _resolve_alias(alias_map: Dict[str, str], key: str) -> str: - """Walk the alias map until we hit a slug that doesn't redirect. - - The merge endpoint stores ``from_slug -> to_slug`` pairs. The delete - endpoint stores ``from_slug -> ""`` (empty string sentinel) to mark - a slug as tombstoned. Chains can accumulate when the user merges - A→B and later merges B→C; this helper collapses them so callers - always land on the final canonical slug. - - Returns: - * the canonical slug if it still resolves to a real label, - * an empty string if the slug has been tombstoned (callers MUST - treat an empty result as "drop this entry entirely"), - * the input ``key`` if it isn't aliased. - - Cycles are guarded by a hard step limit since the alias map is - user-driven. - """ - if not key: - return "" - if not alias_map: - return key - current = key - seen: set[str] = set() - for _ in range(16): - if current in seen: - return current - seen.add(current) - if current not in alias_map: - return current - nxt = alias_map[current] - if nxt == current: - return current - if nxt == "": - # Deletion sentinel — the user has explicitly retired this - # slug. Propagate the empty string up so callers drop it. - return "" - current = nxt - return current - - -# Reserved JSON key under which the worker stores top-level metric -# discoveries on each row's ``metric_scores`` dict. Mirrors the constant -# in ``app/workers/tasks/helpers/llm_evaluation.py`` — kept local here to -# avoid a worker import cycle from the routes module. -DISCOVERED_METRICS_KEY = "__discovered_metrics__" - -# Allowed values for an LLM-suggested top-level metric type. Kept in -# sync with ``DiscoveredMetricSuggestedType`` in -# ``app/models/schemas.py``. -_DISCOVERED_METRIC_TYPES = ("boolean", "rating", "category") - - -def normalize_scores_with_aliases( - metric_scores: Dict[str, Any], - evaluation: CallImportEvaluation, - db: Session, - organization_id: UUID, -) -> Dict[str, Any]: - """Rewrite per-row ``metric_scores`` to honor merges + promotions. - - Called by the worker right after ``evaluate_with_llm`` returns so - every row that finishes AFTER a user has merged or promoted a - discovered label persists data already reflecting that decision. - Without this hook, a worker holding a stale prompt could re-emit a - ``from_key`` slug long after the user merged it away. - - For every parent entry (``selection_mode != null`` and a - ``discovered_labels`` / ``sequence`` field) we: - - * resolve discovered slugs through the evaluation's - ``discovered_label_aliases`` map (transitively), - * drop any discovered_labels entry whose canonical slug now - matches a real promoted child of the parent (merging them out - of the panel for free), and - * collapse adjacent duplicate sequence entries that result. - - Returns ``metric_scores`` (mutated in place) for chaining. - """ - if not isinstance(metric_scores, dict): - return metric_scores - - aliases_top = ( - evaluation.discovered_label_aliases - if isinstance(evaluation.discovered_label_aliases, dict) - else {} - ) - - # Identify the parent entries inside metric_scores. They're the - # dicts that carry a ``selection_mode`` key (set by the LLM - # hierarchy parser) and either a ``sequence`` or a - # ``discovered_labels`` list. - for key, entry in list(metric_scores.items()): - if not isinstance(entry, dict): - continue - if entry.get("type") != "category" and not entry.get("selection_mode"): - continue - try: - parent_uuid = UUID(str(key)) - except (TypeError, ValueError): - continue - - alias_map = {} - sub = aliases_top.get(str(parent_uuid)) - if isinstance(sub, dict): - alias_map = { - str(k): str(v) - for k, v in sub.items() - if isinstance(k, str) and isinstance(v, str) - } - promoted = _promoted_child_slugs(db, parent_uuid, organization_id) - - # Rewrite discovered_labels: alias-resolve keys, drop duplicates - # post-resolution, and drop entries that have been promoted. - discovered = entry.get("discovered_labels") - if isinstance(discovered, list): - kept_disc: List[Dict[str, Any]] = [] - seen: set[str] = set() - for d in discovered: - if not isinstance(d, dict): - continue - slug = _slug_label(d.get("key") or d.get("name")) - slug = _resolve_alias(alias_map, slug) - if not slug or slug in promoted or slug in seen: - continue - seen.add(slug) - new_entry = dict(d) - new_entry["key"] = slug - kept_disc.append(new_entry) - entry["discovered_labels"] = kept_disc - - # Rewrite sequence: alias-resolve every entry; collapse adjacent - # duplicates that result. We DON'T drop slugs that match - # promoted children — the promoted child slug is still a valid - # sequence entry; the flow chart will resolve it to the real - # child node. - seq = entry.get("sequence") - if isinstance(seq, list): - new_seq: List[str] = [] - last: Optional[str] = None - for item in seq: - if not isinstance(item, str): - continue - slug = _resolve_alias(alias_map, _slug_label(item)) - if not slug or slug == last: - continue - new_seq.append(slug) - last = slug - entry["sequence"] = new_seq - - # Top-level metric discoveries live alongside the parent entries - # under the reserved ``DISCOVERED_METRICS_KEY`` slot. Apply the - # flat evaluation-level alias/tombstone map + suppress slugs that - # already correspond to a real top-level Metric so workers that - # finish AFTER the user has merged / deleted / promoted can't - # resurrect a retired candidate. - discovered_metrics_payload = metric_scores.get(DISCOVERED_METRICS_KEY) - if isinstance(discovered_metrics_payload, list): - flat_alias_map = ( - evaluation.discovered_metric_aliases - if isinstance(evaluation.discovered_metric_aliases, dict) - else {} - ) - promoted_metric_slugs = _promoted_top_level_metric_slugs( - db, organization_id - ) - kept_metrics: List[Dict[str, Any]] = [] - seen_metrics: set[str] = set() - for d in discovered_metrics_payload: - if not isinstance(d, dict): - continue - slug = _slug_label(d.get("key") or d.get("name")) - slug = _resolve_alias(flat_alias_map, slug) - if ( - not slug - or slug in promoted_metric_slugs - or slug in seen_metrics - ): - continue - seen_metrics.add(slug) - new_entry = dict(d) - new_entry["key"] = slug - kept_metrics.append(new_entry) - if kept_metrics: - metric_scores[DISCOVERED_METRICS_KEY] = kept_metrics - else: - # No survivors — drop the empty array so empty-discovery rows - # keep their pre-feature payload shape. - metric_scores.pop(DISCOVERED_METRICS_KEY, None) - - return metric_scores - - -def _alias_map_for_parent( - evaluation: CallImportEvaluation, parent_metric_id: UUID -) -> Dict[str, str]: - """Pull ``{from_slug: to_slug}`` for one parent out of the eval's blob. - - Stored shape on the evaluation row is - ``{parent_id_str: {from_slug: to_slug, ...}}``. Returns an empty - dict for parents that have never had a merge applied. - """ - raw = getattr(evaluation, "discovered_label_aliases", None) - if not isinstance(raw, dict): - return {} - submap = raw.get(str(parent_metric_id)) - if not isinstance(submap, dict): - return {} - return { - str(k): str(v) - for k, v in submap.items() - if isinstance(k, str) and isinstance(v, str) - } - - -def _promoted_child_slugs( - db: Session, parent_metric_id: UUID, organization_id: UUID -) -> set[str]: - """Slugs of every real child currently sitting under the parent. - - The Discovered Labels panel hides any candidate whose slug already - matches a real child — that covers both freshly-promoted candidates - and legacy children the LLM happened to re-discover. We pull from - the live ``metrics`` table rather than the eval's - ``selected_metric_groups`` snapshot so newly-promoted children take - effect immediately, even on evaluations that ran before the - promotion. - """ - children = ( - db.query(Metric.name) - .filter( - Metric.parent_metric_id == parent_metric_id, - Metric.organization_id == organization_id, - ) - .all() - ) - out: set[str] = set() - for (name,) in children: - slug = _slug_label(name) - if slug: - out.add(slug) - return out - - -def _promoted_top_level_metric_slugs( - db: Session, organization_id: UUID -) -> set[str]: - """Slugs of every top-level (non-child) Metric in the organization. - - Used to suppress discovered-metric candidates whose slug already - matches a real standalone metric. We intentionally include both - standalone metrics AND parent category metrics — a top-level - discovery that collides with either name is a duplicate by - definition. - """ - rows = ( - db.query(Metric.name) - .filter( - Metric.organization_id == organization_id, - Metric.parent_metric_id.is_(None), - ) - .all() - ) - out: set[str] = set() - for (name,) in rows: - slug = _slug_label(name) - if slug: - out.add(slug) - return out - - -def _get_running_discovered_labels( - db: Session, - eval_id: UUID, - parent_metric_id: UUID, - organization_id: Optional[UUID] = None, - alias_map: Optional[Dict[str, str]] = None, -) -> List[Dict[str, Any]]: - """Slug-deduped view of every discovered label seen in this eval so far. - - Walks each ``call_import_evaluation_rows`` row's - ``metric_scores[parent_id]["discovered_labels"]`` and folds entries - that share the same slug. Returns a list ordered by descending - count and stable on label key, shaped like:: - - [{"key": "customer_on_hold", "name": "Customer put on hold", - "description": "...", "sample_rationale": "...", "count": 12}] - - Powers two callers: - * The worker prompt builder ("REUSE the existing key if it fits") - — invoked just before each row's LLM call to feed the model the - running list of previously-discovered labels in this evaluation. - * The ``/discovered-labels`` API surface used by the frontend - Discovered Labels panel to render candidates with counts + - sample rationales. - - Non-completed rows are skipped: an in-flight row's discoveries are - not yet reliable (the row could fail and never produce final - metric_scores). We accept the tradeoff that rows running - concurrently won't see each other's labels — slug-collision dedup - catches identical re-inventions, and near-paraphrases surface in - the UI panel where the user can manually merge. - """ - - parent_id_str = str(parent_metric_id) - from app.db_sharding.eval_rows import load_evaluation_rows_for_run - - eval_rows = load_evaluation_rows_for_run(db, eval_id) - rows = [ - (row.metric_scores,) - for row in eval_rows - if row.status == CallImportRowStatus.COMPLETED.value - ] - - # Suppress slugs that have either: - # * been promoted to a real child of the parent (so the panel doesn't - # keep nagging the user about a candidate they've already - # accepted), or - # * been merged INTO another slug (the "from" side of a merge) — - # those occurrences fold into the canonical target instead. - promoted_slugs: set[str] = set() - if organization_id is not None: - promoted_slugs = _promoted_child_slugs( - db, parent_metric_id, organization_id - ) - aliases = alias_map or {} - - by_key: Dict[str, Dict[str, Any]] = {} - for (scores,) in rows: - if not isinstance(scores, dict): - continue - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - continue - discovered = parent_entry.get("discovered_labels") - if not isinstance(discovered, list): - continue - for entry in discovered: - if not isinstance(entry, dict): - continue - raw_key = entry.get("key") or entry.get("name") - key = _slug_label(raw_key) - if not key: - continue - # Apply user merges + deletions first, THEN drop anything - # that ended up on a real child slug. Order matters: a - # candidate that was merged into a slug which has since - # been promoted should disappear, not show up at the - # canonical slug. An empty resolved key means the slug was - # tombstoned via the delete endpoint. - key = _resolve_alias(aliases, key) - if not key or key in promoted_slugs: - continue - name = (entry.get("name") or "").strip() or key.replace("_", " ") - description = (entry.get("description") or "").strip() or None - sample = (entry.get("rationale") or "").strip() or None - - existing = by_key.get(key) - if existing is None: - # Track up to N=3 distinct rationales per candidate so - # the Promote-to-child flow can pre-fill the new - # sub-metric's rubric with concrete LLM examples - # without the user copy-pasting from the row table. - # ``sample_rationale`` is preserved for back-compat - # with older clients; ``examples`` is the new field. - examples = [sample] if sample else [] - by_key[key] = { - "key": key, - "name": name, - "description": description, - "sample_rationale": sample, - "examples": examples, - "count": 1, - } - continue - - existing["count"] += 1 - if not existing["description"] and description: - existing["description"] = description - if not existing["sample_rationale"] and sample: - existing["sample_rationale"] = sample - # Append distinct rationales (case-insensitive trim) up - # to a small cap. Headroom is intentionally one above - # what the UI surfaces (2) so we have a backup when the - # first rationale is unhelpful. - if sample: - ex_list: List[str] = existing.setdefault("examples", []) - if len(ex_list) < 3 and not any( - s.strip().lower() == sample.strip().lower() for s in ex_list - ): - ex_list.append(sample) - - return sorted( - by_key.values(), - key=lambda item: (-item["count"], item["key"]), - ) - - -def _get_running_discovered_metrics( - db: Session, - eval_id: UUID, - organization_id: Optional[UUID] = None, - alias_map: Optional[Dict[str, str]] = None, -) -> List[Dict[str, Any]]: - """Slug-deduped view of every discovered top-level metric in this eval. - - Mirrors :func:`_get_running_discovered_labels` but is keyed at the - evaluation level (no ``parent_metric_id``). Walks each completed - row's ``metric_scores[DISCOVERED_METRICS_KEY]`` list, folds entries - that share the same slug (post-alias resolution), and suppresses - slugs that already correspond to a real top-level :class:`Metric` - in the organization. - - Each returned entry is shaped:: - - {"key": "customer_satisfaction", - "name": "Customer Satisfaction", - "description": "...", - "suggested_type": "boolean" | "rating" | "category", - "sample_rationale": "...", - "examples": ["..."], - "count": 12} - """ - - from app.db_sharding.eval_rows import load_evaluation_rows_for_run - - eval_rows = load_evaluation_rows_for_run(db, eval_id) - rows = [ - (row.metric_scores,) - for row in eval_rows - if row.status == CallImportRowStatus.COMPLETED.value - ] - - promoted_slugs: set[str] = set() - if organization_id is not None: - promoted_slugs = _promoted_top_level_metric_slugs( - db, organization_id - ) - aliases = alias_map or {} - - by_key: Dict[str, Dict[str, Any]] = {} - for (scores,) in rows: - if not isinstance(scores, dict): - continue - discovered = scores.get(DISCOVERED_METRICS_KEY) - if not isinstance(discovered, list): - continue - for entry in discovered: - if not isinstance(entry, dict): - continue - raw_key = entry.get("key") or entry.get("name") - key = _slug_label(raw_key) - if not key: - continue - # Apply user merges + deletions first, THEN drop anything - # that ended up on an already-existing top-level metric - # slug. Empty resolved key = tombstoned. - key = _resolve_alias(aliases, key) - if not key or key in promoted_slugs: - continue - name = (entry.get("name") or "").strip() or key.replace( - "_", " " - ) - description = (entry.get("description") or "").strip() or None - sample = (entry.get("rationale") or "").strip() or None - raw_type = str(entry.get("suggested_type") or "").strip().lower() - if raw_type not in _DISCOVERED_METRIC_TYPES: - raw_type = "boolean" - - existing = by_key.get(key) - if existing is None: - examples = [sample] if sample else [] - by_key[key] = { - "key": key, - "name": name, - "description": description, - "suggested_type": raw_type, - "sample_rationale": sample, - "examples": examples, - "count": 1, - } - continue - - existing["count"] += 1 - if not existing["description"] and description: - existing["description"] = description - if not existing["sample_rationale"] and sample: - existing["sample_rationale"] = sample - # Keep the most-frequently-suggested type. We don't track - # per-type frequency yet; defer to the first non-default - # type encountered when the existing entry has the default. - if existing.get("suggested_type") == "boolean" and raw_type != "boolean": - existing["suggested_type"] = raw_type - if sample: - ex_list: List[str] = existing.setdefault("examples", []) - if len(ex_list) < 3 and not any( - s.strip().lower() == sample.strip().lower() for s in ex_list - ): - ex_list.append(sample) - - return sorted( - by_key.values(), - key=lambda item: (-item["count"], item["key"]), - ) - - -def _build_flow_graph( - eval_rows: List[CallImportEvaluationRow], - parent_metric: Metric, - children: List[Metric], - alias_map: Optional[Dict[str, str]] = None, - extra_children: Optional[List[Metric]] = None, -) -> MetricFlowResponse: - """Walk per-row ``sequence`` arrays and produce aggregate nodes/edges. - - A synthetic ``START`` node is prepended to every sequence so the - diagram has a single origin. Children that never appear in any - sequence are still emitted as nodes (count=0) so the UI can render - them in the legend. - - ``alias_map`` lets callers fold merged-out discovered slugs into - their canonical target before building the graph; ``extra_children`` - are children of the parent that aren't in the legend list (e.g. - children promoted *after* the evaluation was created and therefore - missing from ``selected_metric_groups``) but should still resolve in - sequences so the slug doesn't get redrawn as a discovered candidate. - """ - parent_id_str = str(parent_metric.id) - aliases = alias_map or {} - # Build a fast lookup keyed by both the lower_snake child key (what the - # LLM emits in ``sequence``) and the child UUID (what some clients may - # store) so legacy / drifted payloads still resolve. - child_lookup: Dict[str, Metric] = {} - for child in children: - slug = _slug_label(child.name) - child_lookup[slug] = child - child_lookup[str(child.id)] = child - # ``extra_children`` are resolved-only — they shouldn't add legend - # nodes (those come from the explicit ``children`` argument), but - # they need to be in ``child_lookup`` so a sequence step that - # matches a freshly-promoted child resolves to the real child UUID - # instead of falling through to ``discovered_lookup`` and rendering - # as a "discovered" node. - if extra_children: - for child in extra_children: - slug = _slug_label(child.name) - if slug and slug not in child_lookup: - child_lookup[slug] = child - cid = str(child.id) - child_lookup.setdefault(cid, child) - - # Discovered labels: walk every row's discovered_labels first so we - # know which discovered slugs are valid before resolving sequences. - # Discovered nodes get a ``disc:`` prefixed id so they can't collide - # with real child UUIDs in the node/edge graph. We apply - # ``alias_map`` first so merged-out source slugs fold into their - # canonical target — preserving the user's "merge" intent on still- - # in-flight rows whose JSON wasn't rewritten by the merge endpoint. - discovered_lookup: Dict[str, Dict[str, Any]] = {} - for row in eval_rows: - scores = ( - row.metric_scores if isinstance(row.metric_scores, dict) else {} - ) - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - continue - raw_discovered = parent_entry.get("discovered_labels") - if not isinstance(raw_discovered, list): - continue - for entry in raw_discovered: - if not isinstance(entry, dict): - continue - slug = _slug_label(entry.get("key") or entry.get("name")) - slug = _resolve_alias(aliases, slug) - if not slug or slug in child_lookup: - continue - name = (entry.get("name") or "").strip() or slug.replace("_", " ") - existing = discovered_lookup.get(slug) - if existing is None: - discovered_lookup[slug] = { - "id": f"{_DISCOVERED_NODE_PREFIX}{slug}", - "name": name, - } - - node_counts: Dict[str, int] = {} - edge_counts: Dict[tuple[str, str], int] = {} - terminal_counts: Dict[str, int] = {} - - total_rows = len(eval_rows) - rows_with_sequence = 0 - - for row in eval_rows: - scores = ( - row.metric_scores if isinstance(row.metric_scores, dict) else {} - ) - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - continue - raw_sequence = parent_entry.get("sequence") - if not isinstance(raw_sequence, list): - continue - - resolved_ids: List[str] = [] - last_resolved: Optional[str] = None - for item in raw_sequence: - if not isinstance(item, str): - continue - normalized = _resolve_alias(aliases, _slug_label(item)) - child = child_lookup.get(normalized) or child_lookup.get(item) - if child is not None: - cid = str(child.id) - # Adjacent dedupe AFTER alias resolution so two - # different raw slugs that fold to the same target - # don't draw a self-edge through the chart. - if cid == last_resolved: - continue - resolved_ids.append(cid) - last_resolved = cid - continue - disc = discovered_lookup.get(normalized) - if disc is not None: - if disc["id"] == last_resolved: - continue - resolved_ids.append(disc["id"]) - last_resolved = disc["id"] - - if not resolved_ids: - continue - - rows_with_sequence += 1 - for nid in resolved_ids: - node_counts[nid] = node_counts.get(nid, 0) + 1 - - edge_counts[(_FLOW_START_NODE_ID, resolved_ids[0])] = ( - edge_counts.get((_FLOW_START_NODE_ID, resolved_ids[0]), 0) + 1 - ) - for src, tgt in zip(resolved_ids, resolved_ids[1:]): - if src == tgt: - continue - edge_counts[(src, tgt)] = edge_counts.get((src, tgt), 0) + 1 - - terminal_id = resolved_ids[-1] - terminal_counts[terminal_id] = terminal_counts.get(terminal_id, 0) + 1 - - nodes: List[MetricFlowNode] = [] - # Always include a START node so the UI has a stable entry point. - nodes.append( - MetricFlowNode( - id=_FLOW_START_NODE_ID, - label="Start", - count=rows_with_sequence, - is_terminal=False, - ) - ) - - def _emit_child_node(child: Metric) -> None: - cid = str(child.id) - count = node_counts.get(cid, 0) - terminal_count = terminal_counts.get(cid, 0) - is_terminal = False - if rows_with_sequence > 0: - is_terminal = ( - terminal_count / rows_with_sequence - ) >= _FLOW_TERMINAL_THRESHOLD - nodes.append( - MetricFlowNode( - id=cid, - label=child.name, - count=count, - is_terminal=is_terminal, - ) - ) - - emitted_child_ids: set[str] = set() - for child in children: - cid = str(child.id) - if cid in emitted_child_ids: - continue - emitted_child_ids.add(cid) - _emit_child_node(child) - # Extra children (promoted after the eval was created) only get - # legend nodes if they actually appear in the data — otherwise we'd - # pollute the diagram with every standalone promotion the user has - # ever made under this parent. - if extra_children: - for child in extra_children: - cid = str(child.id) - if cid in emitted_child_ids: - continue - if node_counts.get(cid, 0) == 0: - continue - emitted_child_ids.add(cid) - _emit_child_node(child) - # Append discovered nodes after the real children so legend ordering - # keeps user-defined labels first. - for slug, info in discovered_lookup.items(): - nid = info["id"] - count = node_counts.get(nid, 0) - terminal_count = terminal_counts.get(nid, 0) - is_terminal = False - if rows_with_sequence > 0: - is_terminal = ( - terminal_count / rows_with_sequence - ) >= _FLOW_TERMINAL_THRESHOLD - nodes.append( - MetricFlowNode( - id=nid, - label=info["name"], - count=count, - is_terminal=is_terminal, - is_discovered=True, - ) - ) - - edges: List[MetricFlowEdge] = [ - MetricFlowEdge(source=src, target=tgt, count=count) - for (src, tgt), count in sorted( - edge_counts.items(), key=lambda kv: kv[1], reverse=True - ) - ] - - return MetricFlowResponse( - parent_metric_id=parent_id_str, - parent_metric_name=parent_metric.name, - selection_mode=parent_metric.selection_mode, - nodes=nodes, - edges=edges, - total_rows=total_rows, - rows_with_sequence=rows_with_sequence, - ) - - -@router.get( - "/{eval_id}/flow", - response_model=MetricFlowResponse, - operation_id="getCallImportEvaluationFlow", -) -async def get_call_import_evaluation_flow( - call_import_id: UUID, - eval_id: UUID, - parent_metric_id: UUID = Query( - ..., - description=( - "Parent (category) metric whose children's sequences should be " - "aggregated into a flow graph." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> MetricFlowResponse: - """Aggregate the LLM-inferred per-row sequences into one flow graph. - - Returns ``nodes`` (one per child of the parent metric, plus a - synthetic ``START`` node) and ``edges`` (counts of consecutive - label transitions across every row that produced a sequence). The - frontend feeds this directly into a React Flow / xyflow canvas; - edge thickness should scale with ``count / total_rows`` and - ``is_terminal`` nodes should be styled as outcomes. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - parent = ( - db.query(Metric) - .filter( - Metric.id == parent_metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if not parent: - raise HTTPException( - status_code=404, - detail="Parent metric not found in this organization.", - ) - if not parent.selection_mode: - raise HTTPException( - status_code=400, - detail=( - "Flow charts are only meaningful for parent metrics " - "(selection_mode set). This metric is standalone." - ), - ) - - # Children are taken from selected_metric_groups when present so the - # flow chart reflects exactly the subset that ran in this - # evaluation; otherwise fall back to every enabled child of the - # parent. - groups_raw = ( - evaluation.selected_metric_groups - if isinstance(evaluation.selected_metric_groups, dict) - else {} - ) - parent_id_str = str(parent.id) - children: List[Metric] = [] - if parent_id_str in groups_raw and isinstance( - groups_raw[parent_id_str], list - ): - child_ids: List[UUID] = [] - for c in groups_raw[parent_id_str]: - try: - child_ids.append(UUID(str(c))) - except (TypeError, ValueError): - continue - if child_ids: - children = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.id.in_(child_ids), - ) - .order_by(Metric.created_at.asc()) - .all() - ) - if not children: - children = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.parent_metric_id == parent.id, - ) - .order_by(Metric.created_at.asc()) - .all() - ) - - # Children promoted AFTER this evaluation was created aren't in - # ``selected_metric_groups`` but their slugs still appear in already- - # scored rows' sequences. Pass them as ``extra_children`` so those - # sequence entries resolve against the real (now promoted) child - # instead of being redrawn as discovered candidates. - extra_children: List[Metric] = [] - if children: - existing_ids = {child.id for child in children} - all_children = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.parent_metric_id == parent.id, - ) - .all() - ) - extra_children = [c for c in all_children if c.id not in existing_ids] - - eval_rows = _load_eval_rows(db, eval_id) - - alias_map = _alias_map_for_parent(evaluation, parent.id) - return _build_flow_graph( - eval_rows, - parent, - children, - alias_map=alias_map, - extra_children=extra_children, - ) - - -@router.get( - "/{eval_id}/discovered-labels", - response_model=DiscoveredLabelsResponse, - operation_id="getCallImportEvaluationDiscoveredLabels", -) -async def get_call_import_evaluation_discovered_labels( - call_import_id: UUID, - eval_id: UUID, - parent_metric_id: UUID = Query( - ..., - description=( - "Parent (category) metric whose LLM-discovered candidate " - "sub-labels should be aggregated across rows." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> DiscoveredLabelsResponse: - """Aggregate candidate sub-labels the LLM discovered during this eval. - - Only meaningful for parents with ``allow_discovery=true``; for other - parents we just return an empty ``items`` list rather than 400-ing - so the frontend can call the endpoint unconditionally for every - parent on the Flow tab without branching. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - parent = ( - db.query(Metric) - .filter( - Metric.id == parent_metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if not parent: - raise HTTPException( - status_code=404, - detail="Parent metric not found in this organization.", - ) - - alias_map = _alias_map_for_parent(evaluation, parent_metric_id) - items_raw = _get_running_discovered_labels( - db, - eval_id, - parent_metric_id, - organization_id=organization_id, - alias_map=alias_map, - ) - items = [DiscoveredLabelItem(**item) for item in items_raw] - return DiscoveredLabelsResponse( - parent_metric_id=str(parent.id), items=items - ) - - -@router.post( - "/{eval_id}/discovered-labels/merge", - response_model=DiscoveredLabelsResponse, - operation_id="mergeCallImportEvaluationDiscoveredLabels", -) -async def merge_call_import_evaluation_discovered_labels( - call_import_id: UUID, - eval_id: UUID, - body: DiscoveredLabelMergeRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> DiscoveredLabelsResponse: - """Rewrite every row's ``discovered_labels`` entry from from_key -> to_key. - - Idempotent — re-merging the same pair is a no-op. Discovered slugs - inside per-row ``sequence`` arrays are also rewritten so the flow - chart stays consistent with the panel. When a row already has - ``to_key`` and we're merging ``from_key`` into it, we drop the - ``from_key`` entry instead of producing two entries with the same - slug. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - parent = ( - db.query(Metric) - .filter( - Metric.id == body.parent_metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if not parent: - raise HTTPException( - status_code=404, - detail="Parent metric not found in this organization.", - ) - - from_key = _slug_label(body.from_key) - to_key = _slug_label(body.to_key) - if not from_key or not to_key: - raise HTTPException( - status_code=400, - detail="from_key and to_key must be non-empty slugs.", - ) - if from_key == to_key: - # No-op; just return the current aggregate so the client can - # refresh its view. - alias_map_existing = _alias_map_for_parent(evaluation, parent.id) - items_raw = _get_running_discovered_labels( - db, - eval_id, - body.parent_metric_id, - organization_id=organization_id, - alias_map=alias_map_existing, - ) - return DiscoveredLabelsResponse( - parent_metric_id=str(parent.id), - items=[DiscoveredLabelItem(**item) for item in items_raw], - ) - - parent_id_str = str(parent.id) - from app.db_sharding.eval_rows import foreach_evaluation_row_mutating - - def _merge_discovered_label_row(row: CallImportEvaluationRow) -> bool: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else None - ) - if not scores: - return False - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - return False - - mutated = False - discovered = parent_entry.get("discovered_labels") - if isinstance(discovered, list): - kept: List[Dict[str, Any]] = [] - existing_to = next( - ( - e - for e in discovered - if isinstance(e, dict) - and _slug_label(e.get("key") or e.get("name")) == to_key - ), - None, - ) - for entry in discovered: - if not isinstance(entry, dict): - kept.append(entry) - continue - key = _slug_label(entry.get("key") or entry.get("name")) - if key == from_key: - if existing_to is not None: - mutated = True - continue - new_entry = dict(entry) - new_entry["key"] = to_key - kept.append(new_entry) - mutated = True - else: - kept.append(entry) - if mutated: - parent_entry["discovered_labels"] = kept - - seq = parent_entry.get("sequence") - if isinstance(seq, list): - new_seq: List[str] = [] - seq_changed = False - last_added: Optional[str] = None - for item in seq: - if isinstance(item, str) and _slug_label(item) == from_key: - seq_changed = True - if last_added == to_key: - continue - new_seq.append(to_key) - last_added = to_key - else: - new_seq.append(item) - last_added = ( - _slug_label(item) if isinstance(item, str) else None - ) - if seq_changed: - parent_entry["sequence"] = new_seq - mutated = True - - if mutated: - row.metric_scores = dict(scores) - return mutated - - foreach_evaluation_row_mutating(db, eval_id, _merge_discovered_label_row) - - # Persist the merge at the evaluation level too. This is what makes - # the merge survive future scoring: rows that finish AFTER this - # call (e.g. retries, in-flight workers) will go through the - # alias map in the API surface even if the per-row JSON they - # write still mentions ``from_key``. We chain through any existing - # alias so merging A→B and then B→C resolves A→C in the panel. - raw_aliases = ( - evaluation.discovered_label_aliases - if isinstance(evaluation.discovered_label_aliases, dict) - else {} - ) - aliases_top = dict(raw_aliases) - parent_aliases = dict(aliases_top.get(parent_id_str) or {}) - # Resolve transitively: if to_key itself was previously merged into - # something else, point from_key at the canonical end-of-chain. - canonical_to = _resolve_alias(parent_aliases, to_key) - parent_aliases[from_key] = canonical_to - # Re-target any earlier aliases that pointed AT from_key — without - # this, A→B and then B→C would leave A still pointing to B (now a - # broken pointer because B is gone). Rewriting them keeps the - # alias map self-consistent. - for k, v in list(parent_aliases.items()): - if v == from_key: - parent_aliases[k] = canonical_to - aliases_top[parent_id_str] = parent_aliases - evaluation.discovered_label_aliases = aliases_top - - stamp_evaluation_actor(evaluation, principal) - db.commit() - - alias_map_after = _alias_map_for_parent(evaluation, parent.id) - items_raw = _get_running_discovered_labels( - db, - eval_id, - body.parent_metric_id, - organization_id=organization_id, - alias_map=alias_map_after, - ) - return DiscoveredLabelsResponse( - parent_metric_id=str(parent.id), - items=[DiscoveredLabelItem(**item) for item in items_raw], - ) - - -@router.post( - "/{eval_id}/discovered-labels/delete", - response_model=DiscoveredLabelsResponse, - operation_id="deleteCallImportEvaluationDiscoveredLabel", -) -async def delete_call_import_evaluation_discovered_label( - call_import_id: UUID, - eval_id: UUID, - body: DiscoveredLabelDeleteRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> DiscoveredLabelsResponse: - """Tombstone a single LLM-discovered candidate for this evaluation. - - Symmetric with the merge endpoint, but instead of redirecting the - slug at another candidate we mark it as deleted. After this call: - - * the slug is stripped from every row's - ``metric_scores[parent].discovered_labels`` list, and from - every row's ``sequence`` array (so the flow chart no longer - draws a node for it); - * the slug is recorded in - ``evaluation.discovered_label_aliases[parent][slug] = ""`` - so any worker that finishes a row AFTER this call (e.g. a row - still in flight when the user clicked Delete) silently drops - the slug instead of resurrecting it. - - Idempotent: deleting an already-deleted slug is a no-op. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - parent = ( - db.query(Metric) - .filter( - Metric.id == body.parent_metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if not parent: - raise HTTPException( - status_code=404, - detail="Parent metric not found in this organization.", - ) - - target_key = _slug_label(body.key) - if not target_key: - raise HTTPException( - status_code=400, - detail="key must be a non-empty slug.", - ) - - parent_id_str = str(parent.id) - from app.db_sharding.eval_rows import foreach_evaluation_row_mutating - - def _delete_discovered_label_row(row: CallImportEvaluationRow) -> bool: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else None - ) - if not scores: - return False - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - return False - - mutated = False - discovered = parent_entry.get("discovered_labels") - if isinstance(discovered, list): - kept = [ - e - for e in discovered - if not ( - isinstance(e, dict) - and _slug_label(e.get("key") or e.get("name")) - == target_key - ) - ] - if len(kept) != len(discovered): - parent_entry["discovered_labels"] = kept - mutated = True - - seq = parent_entry.get("sequence") - if isinstance(seq, list): - new_seq: List[str] = [] - seq_changed = False - last_added: Optional[str] = None - for item in seq: - if isinstance(item, str) and _slug_label(item) == target_key: - seq_changed = True - continue - if isinstance(item, str): - norm = _slug_label(item) - if norm == last_added: - seq_changed = True - continue - last_added = norm - new_seq.append(item) - if seq_changed: - parent_entry["sequence"] = new_seq - mutated = True - - if mutated: - row.metric_scores = dict(scores) - return mutated - - foreach_evaluation_row_mutating(db, eval_id, _delete_discovered_label_row) - - # 3. Persist the tombstone on the evaluation so workers that finish - # later don't re-surface the deleted slug. We also retarget any - # existing aliases whose ``to_key`` was the deleted slug — without - # this, a previous merge that pointed at this slug would leave a - # dangling pointer. - raw_aliases = ( - evaluation.discovered_label_aliases - if isinstance(evaluation.discovered_label_aliases, dict) - else {} - ) - aliases_top = dict(raw_aliases) - parent_aliases = dict(aliases_top.get(parent_id_str) or {}) - parent_aliases[target_key] = "" # deletion sentinel - for k, v in list(parent_aliases.items()): - if v == target_key: - parent_aliases[k] = "" - aliases_top[parent_id_str] = parent_aliases - evaluation.discovered_label_aliases = aliases_top - - stamp_evaluation_actor(evaluation, principal) - db.commit() - - alias_map_after = _alias_map_for_parent(evaluation, parent.id) - items_raw = _get_running_discovered_labels( - db, - eval_id, - body.parent_metric_id, - organization_id=organization_id, - alias_map=alias_map_after, - ) - return DiscoveredLabelsResponse( - parent_metric_id=str(parent.id), - items=[DiscoveredLabelItem(**item) for item in items_raw], - ) - - -# --------------------------------------------------------------------------- -# Discovered TOP-LEVEL METRICS (per-evaluation discovery) -# -# These endpoints are the parallel of the discovered-labels trio above but -# scoped to the evaluation as a whole instead of to a parent metric. They -# all live under ``/{eval_id}/discovered-metrics`` and operate on the -# reserved ``DISCOVERED_METRICS_KEY`` slot of each per-row -# ``metric_scores`` plus the flat ``CallImportEvaluation.discovered_metric_aliases`` -# map (no parent-id nesting). -# --------------------------------------------------------------------------- - - -def _flat_metric_aliases( - evaluation: CallImportEvaluation, -) -> Dict[str, str]: - """Pull the flat ``{from_slug: to_slug}`` map for an evaluation.""" - raw = getattr(evaluation, "discovered_metric_aliases", None) - if not isinstance(raw, dict): - return {} - return { - str(k): str(v) - for k, v in raw.items() - if isinstance(k, str) and isinstance(v, str) - } - - -@router.get( - "/{eval_id}/discovered-metrics", - response_model=DiscoveredMetricsResponse, - operation_id="getCallImportEvaluationDiscoveredMetrics", -) -async def get_call_import_evaluation_discovered_metrics( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> DiscoveredMetricsResponse: - """Aggregate top-level metric candidates the LLM discovered during this eval. - - Returns an empty ``items`` list when the evaluation did not opt - into top-level metric discovery; this keeps the frontend able to - call the endpoint unconditionally without branching on the - evaluation's ``discover_new_metrics`` flag. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - if not bool(getattr(evaluation, "discover_new_metrics", False)): - return DiscoveredMetricsResponse(evaluation_id=evaluation.id, items=[]) - - items_raw = _get_running_discovered_metrics( - db, - eval_id, - organization_id=organization_id, - alias_map=_flat_metric_aliases(evaluation), - ) - return DiscoveredMetricsResponse( - evaluation_id=evaluation.id, - items=[DiscoveredMetricItem(**item) for item in items_raw], - ) - - -@router.post( - "/{eval_id}/discovered-metrics/merge", - response_model=DiscoveredMetricsResponse, - operation_id="mergeCallImportEvaluationDiscoveredMetrics", -) -async def merge_call_import_evaluation_discovered_metrics( - call_import_id: UUID, - eval_id: UUID, - body: DiscoveredMetricMergeRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> DiscoveredMetricsResponse: - """Rewrite every row's ``__discovered_metrics__`` entry from→to. - - Mirrors the discovered-labels merge endpoint but operates on the - flat top-level metric list. Idempotent — re-merging is a no-op. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - from_key = _slug_label(body.from_key) - to_key = _slug_label(body.to_key) - if not from_key or not to_key: - raise HTTPException( - status_code=400, - detail="from_key and to_key must be non-empty slugs.", - ) - if from_key == to_key: - items_raw = _get_running_discovered_metrics( - db, - eval_id, - organization_id=organization_id, - alias_map=_flat_metric_aliases(evaluation), - ) - return DiscoveredMetricsResponse( - evaluation_id=evaluation.id, - items=[DiscoveredMetricItem(**item) for item in items_raw], - ) - - from app.db_sharding.eval_rows import foreach_evaluation_row_mutating - - def _merge_discovered_metric_row(row: CallImportEvaluationRow) -> bool: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else None - ) - if not scores: - return False - discovered = scores.get(DISCOVERED_METRICS_KEY) - if not isinstance(discovered, list): - return False - - kept: List[Dict[str, Any]] = [] - mutated = False - existing_to = next( - ( - e - for e in discovered - if isinstance(e, dict) - and _slug_label(e.get("key") or e.get("name")) == to_key - ), - None, - ) - for entry in discovered: - if not isinstance(entry, dict): - kept.append(entry) - continue - key = _slug_label(entry.get("key") or entry.get("name")) - if key == from_key: - if existing_to is not None: - mutated = True - continue - new_entry = dict(entry) - new_entry["key"] = to_key - kept.append(new_entry) - mutated = True - else: - kept.append(entry) - if mutated: - scores[DISCOVERED_METRICS_KEY] = kept - row.metric_scores = dict(scores) - return mutated - - foreach_evaluation_row_mutating(db, eval_id, _merge_discovered_metric_row) - - raw_aliases = ( - evaluation.discovered_metric_aliases - if isinstance(evaluation.discovered_metric_aliases, dict) - else {} - ) - aliases = dict(raw_aliases) - canonical_to = _resolve_alias(aliases, to_key) - aliases[from_key] = canonical_to - for k, v in list(aliases.items()): - if v == from_key: - aliases[k] = canonical_to - evaluation.discovered_metric_aliases = aliases - - stamp_evaluation_actor(evaluation, principal) - db.commit() - - items_raw = _get_running_discovered_metrics( - db, - eval_id, - organization_id=organization_id, - alias_map=_flat_metric_aliases(evaluation), - ) - return DiscoveredMetricsResponse( - evaluation_id=evaluation.id, - items=[DiscoveredMetricItem(**item) for item in items_raw], - ) - - -@router.post( - "/{eval_id}/discovered-metrics/delete", - response_model=DiscoveredMetricsResponse, - operation_id="deleteCallImportEvaluationDiscoveredMetric", -) -async def delete_call_import_evaluation_discovered_metric( - call_import_id: UUID, - eval_id: UUID, - body: DiscoveredMetricDeleteRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> DiscoveredMetricsResponse: - """Tombstone a single LLM-discovered top-level metric candidate.""" - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - target_key = _slug_label(body.key) - if not target_key: - raise HTTPException( - status_code=400, - detail="key must be a non-empty slug.", - ) - - from app.db_sharding.eval_rows import foreach_evaluation_row_mutating - - def _delete_discovered_metric_row(row: CallImportEvaluationRow) -> bool: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else None - ) - if not scores: - return False - discovered = scores.get(DISCOVERED_METRICS_KEY) - if not isinstance(discovered, list): - return False - kept = [ - e - for e in discovered - if not ( - isinstance(e, dict) - and _slug_label(e.get("key") or e.get("name")) - == target_key - ) - ] - if len(kept) == len(discovered): - return False - if kept: - scores[DISCOVERED_METRICS_KEY] = kept - else: - scores.pop(DISCOVERED_METRICS_KEY, None) - row.metric_scores = dict(scores) - return True - - foreach_evaluation_row_mutating(db, eval_id, _delete_discovered_metric_row) - - raw_aliases = ( - evaluation.discovered_metric_aliases - if isinstance(evaluation.discovered_metric_aliases, dict) - else {} - ) - aliases = dict(raw_aliases) - aliases[target_key] = "" # tombstone - for k, v in list(aliases.items()): - if v == target_key: - aliases[k] = "" - evaluation.discovered_metric_aliases = aliases - - stamp_evaluation_actor(evaluation, principal) - db.commit() - - items_raw = _get_running_discovered_metrics( - db, - eval_id, - organization_id=organization_id, - alias_map=_flat_metric_aliases(evaluation), - ) - return DiscoveredMetricsResponse( - evaluation_id=evaluation.id, - items=[DiscoveredMetricItem(**item) for item in items_raw], - ) - - -@router.delete( - "/{eval_id}/rows/{eval_row_id}", - status_code=status.HTTP_204_NO_CONTENT, - operation_id="deleteCallImportEvaluationRow", -) -async def delete_call_import_evaluation_row( - call_import_id: UUID, - eval_id: UUID, - eval_row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> Response: - """Delete a single per-row scoring entry within an evaluation run. - - Useful when the user wants to drop a noisy row before re-exporting - the CSV — e.g. a row whose audio was corrupt and skewed the - aggregate. Counters on the parent are recomputed so the rolled-up - status stays accurate. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - from app.db_sharding.eval_rows import delete_evaluation_row_on_shards - - if not delete_evaluation_row_on_shards(eval_row_id, eval_id): - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) - _rollup_evaluation_status(evaluation, db) - stamp_evaluation_actor(evaluation, principal) - db.commit() - return Response(status_code=status.HTTP_204_NO_CONTENT) - - eval_row = ( - db.query(CallImportEvaluationRow) - .filter( - CallImportEvaluationRow.id == eval_row_id, - CallImportEvaluationRow.evaluation_id == eval_id, - ) - .first() - ) - if not eval_row: - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) - - # If the row was still in flight, best-effort revoke the worker task - # so it doesn't try to write into a deleted DB row mid-execution. - if eval_row.celery_task_id and eval_row.status in {"pending", "running"}: - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke(eval_row.celery_task_id, terminate=False) - except Exception: - pass - - db.delete(eval_row) - db.flush() - _rollup_evaluation_status(evaluation, db) - stamp_evaluation_actor(evaluation, principal) - db.commit() - return Response(status_code=status.HTTP_204_NO_CONTENT) - - -# --------------------------------------------------------------------------- -# Retry endpoints -# --------------------------------------------------------------------------- -# -# The create endpoint enqueues every row of a fresh run; these endpoints -# let the user re-enqueue a *subset* of rows in an existing run — most -# commonly the ones that failed. We keep the worker contract identical -# (``evaluate_call_import_row_task(eval_row_id)``), so the retry path -# only has to reset row state and re-fan-out. When a row is missing its -# diarised transcript and the run was configured for diarised -# transcripts, we chain through ``transcribe_call_import_row_task`` the -# same way the create endpoint does — that's what makes "retry" feel -# like "just fix it" instead of "fail again immediately". - - -def _prepare_source_row_for_retry( - source_row: CallImportRow, - *, - transcribe_overwrite: bool, -) -> None: - """Clear stale diarisation markers so retry dispatch can re-run the pipeline.""" - source_row.celery_task_id = None - - # Re-fetch recordings when a prior import failed or stalled without S3 audio. - # Mirrors retry_failed_call_import_rows so eval retry can re-enqueue imports. - if ( - source_row.status - in (CallImportRowStatus.FAILED, CallImportRowStatus.PROCESSING) - and not (source_row.recording_s3_key or "").strip() - ): - source_row.status = CallImportRowStatus.PENDING - source_row.error_message = None - - if transcribe_overwrite and (source_row.diarised_transcript or "").strip(): - source_row.diarised_transcript = None - - has_dia = bool((source_row.diarised_transcript or "").strip()) - dia_status = (source_row.diarised_transcript_status or "").strip().lower() - - if has_dia and not transcribe_overwrite: - source_row.diarised_transcript_status = "completed" - source_row.diarised_transcript_error = None - return - - if dia_status in {"failed", "pending", "running", "idle"}: - source_row.diarised_transcript_status = "idle" - source_row.diarised_transcript_error = None - - -def _reset_eval_row_for_retry( - eval_row: CallImportEvaluationRow, - *, - metric_ids: Optional[List[UUID]] = None, - skip_revoke: bool = False, -) -> None: - """Wipe per-row state so the worker can re-run it cleanly. - - Mirrors the initial state used by ``create_call_import_evaluation`` - when it first inserts a row, with the addition of revoking any - lingering Celery task id. - - When ``metric_ids`` is provided, this is a **metric-subset retry**: - only the scores for those metrics are removed from - ``metric_scores`` (other metrics' previously-computed values are - preserved so the worker's partial-merge write keeps them intact). - Otherwise the entire ``metric_scores`` dict is reset, matching the - legacy behaviour. - """ - if ( - not skip_revoke - and eval_row.celery_task_id - and eval_row.status in {"pending", "running"} - ): - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke(eval_row.celery_task_id, terminate=False) - except Exception: # noqa: BLE001 — revoke is best-effort - pass - eval_row.status = "pending" - eval_row.error_message = None - if metric_ids: - # Strip ONLY the targeted metric keys. Both string and UUID - # forms can appear in ``metric_scores`` depending on which - # code path wrote the dict, so we normalise to lower-case - # strings for the comparison. - existing = ( - eval_row.metric_scores if isinstance(eval_row.metric_scores, dict) else {} - ) - target_keys = {str(mid).lower() for mid in metric_ids} - eval_row.metric_scores = { - key: value - for key, value in existing.items() - if str(key).lower() not in target_keys - } - else: - eval_row.metric_scores = {} - eval_row.started_at = None - eval_row.finished_at = None - eval_row.celery_task_id = None - - -def _enqueue_eval_rows_with_optional_transcribe( - db: Session, - evaluation: CallImportEvaluation, - eval_rows_with_source: List[ - Tuple[CallImportEvaluationRow, CallImportRow] - ], - *, - transcribe_overwrite: bool = False, - restricted_metric_ids: Optional[List[UUID]] = None, -) -> Tuple[int, int]: - """Schedule throttled evaluation dispatch for pending eval rows. - - Returns ``(evaluate_only_count, transcribe_then_evaluate_count)`` for - logging/UI compatibility. Actual Celery fan-out is handled by - :func:`dispatch_evaluation_rows_task` under Redis fair-share limits. - """ - from app.workers.concurrency.eval_dispatch import _needs_transcribe_for_eval - from app.workers.concurrency.fair_dispatch import ( - schedule_fair_dispatch, - store_evaluation_transcribe_overwrite, - store_row_restricted_metrics, - ) - - eval_only_count = 0 - transcribe_count = 0 - if eval_rows_with_source: - for eval_row, source_row in eval_rows_with_source: - if _needs_transcribe_for_eval( - evaluation, - source_row, - transcribe_overwrite=transcribe_overwrite, - ): - transcribe_count += 1 - else: - eval_only_count += 1 - - restricted_metric_ids_str: Optional[List[str]] = ( - [str(mid) for mid in restricted_metric_ids] - if restricted_metric_ids - else None - ) - if restricted_metric_ids_str: - for eval_row, _ in eval_rows_with_source: - store_row_restricted_metrics(eval_row.id, restricted_metric_ids_str) - else: - restricted_metric_ids_str = ( - [str(mid) for mid in restricted_metric_ids] if restricted_metric_ids else None - ) - store_evaluation_transcribe_overwrite( - evaluation.id, - overwrite=transcribe_overwrite, - ) - schedule_fair_dispatch(max_workspace_turns=999) - return eval_only_count, transcribe_count - - -def _apply_telephony_retry_overrides( - db: Session, - *, - call_import: CallImport, - organization_id: UUID, - payload: CallImportEvaluationRetryRequest, -) -> None: - """Pin or clear telephony credentials on the batch for this retry pass.""" - fields_set = payload.model_fields_set - if ( - "provider" not in fields_set - and "telephony_integration_id" not in fields_set - ): - return - - from app.api.v1.routes.call_imports import _resolve_telephony_integration - - if payload.telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, - organization_id, - payload.telephony_integration_id, - payload.provider or "", - ) - call_import.provider = integration.provider - call_import.telephony_integration_id = integration.id - else: - call_import.provider = None - call_import.telephony_integration_id = None - db.flush() - - -def _apply_retry_overrides( - db: Session, - evaluation: CallImportEvaluation, - organization_id: UUID, - payload: CallImportEvaluationRetryRequest, -) -> None: - """Validate + persist the LLM/STT override fields on the run. - - Mirrors the validation in ``create_call_import_evaluation`` but - only touches the fields the caller actually sent — leaving any - field ``None`` preserves the run's existing value. Raises - ``HTTPException(400)`` on bad input so the route handler can let - FastAPI turn it into a clean 400 response. - """ - # --- LLM provider + model (must be sent together) --- - if payload.llm_provider is not None or payload.llm_model is not None: - if not (payload.llm_provider and payload.llm_model): - raise HTTPException( - status_code=400, - detail=( - "Both llm_provider and llm_model are required when " - "overriding the run LLM on retry." - ), - ) - try: - evaluation.llm_provider = ModelProvider( - payload.llm_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Unknown LLM provider '{payload.llm_provider}'. " - "Valid keys are documented in ModelProvider." - ), - ) - new_model = payload.llm_model.strip() or None - if not new_model: - raise HTTPException( - status_code=400, detail="llm_model cannot be empty." - ) - evaluation.llm_model = new_model - - # --- LLM credential pin --- - if payload.llm_credential_id is not None: - cred = ( - db.query(AIProvider) - .filter( - AIProvider.id == payload.llm_credential_id, - AIProvider.organization_id == organization_id, - ) - .first() - ) - if not cred: - raise HTTPException( - status_code=400, - detail=( - "The provided llm_credential_id does not exist in " - "this organization." - ), - ) - evaluation.llm_credential_id = payload.llm_credential_id - - if payload.llm_config is not None: - evaluation.llm_config = payload.llm_config - - # --- Per-metric LLM overrides --- - # We accept the same dict shape as the create endpoint but - # constrain keys to leaf metrics that are actually in this run. - # Passing an empty dict explicitly clears existing overrides. - if payload.metric_llm_overrides is not None: - valid_leaf_ids = { - str(mid) for mid in (evaluation.selected_metric_ids or []) - } - overrides_payload: Dict[str, Dict[str, Any]] = {} - for metric_id, override in payload.metric_llm_overrides.items(): - if metric_id not in valid_leaf_ids: - raise HTTPException( - status_code=400, - detail=( - "metric_llm_overrides references metric " - f"{metric_id} which is not a leaf metric in " - "this run." - ), - ) - override_dict: Dict[str, Any] = {} - if override.provider is not None: - if not override.model: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} has a " - "provider but no model." - ), - ) - try: - override_dict["provider"] = ModelProvider( - override.provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} uses " - f"unknown provider '{override.provider}'." - ), - ) - override_dict["model"] = override.model.strip() - elif override.model: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} has a model " - "but no provider." - ), - ) - if override.credential_id is not None: - override_dict["credential_id"] = str(override.credential_id) - if override.llm_config is not None: - override_dict["llm_config"] = override.llm_config - if override_dict: - overrides_payload[metric_id] = override_dict - evaluation.metric_llm_overrides = overrides_payload or None - - # --- STT provider + model (must be sent together) --- - if payload.stt_provider is not None or payload.stt_model is not None: - if not (payload.stt_provider and payload.stt_model): - raise HTTPException( - status_code=400, - detail=( - "Both stt_provider and stt_model are required " - "when overriding the run STT on retry." - ), - ) - try: - evaluation.stt_provider = ModelProvider( - payload.stt_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=f"Unknown STT provider '{payload.stt_provider}'.", - ) - new_stt_model = payload.stt_model.strip() or None - if not new_stt_model: - raise HTTPException( - status_code=400, detail="stt_model cannot be empty." - ) - evaluation.stt_model = new_stt_model - - # --- STT credential pin --- - if payload.stt_credential_id is not None: - evaluation.stt_credential_id = payload.stt_credential_id - - # --- LLM diariser provider + model (must be sent together) --- - if ( - payload.diarization_llm_provider is not None - or payload.diarization_llm_model is not None - ): - if not ( - payload.diarization_llm_provider - and payload.diarization_llm_model - ): - raise HTTPException( - status_code=400, - detail=( - "Both diarization_llm_provider and " - "diarization_llm_model are required when overriding " - "the run diariser on retry." - ), - ) - try: - evaluation.diarisation_llm_provider = ModelProvider( - payload.diarization_llm_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - "Unknown diarisation LLM provider " - f"'{payload.diarization_llm_provider}'." - ), - ) - new_diariser_model = ( - payload.diarization_llm_model.strip() or None - ) - if not new_diariser_model: - raise HTTPException( - status_code=400, - detail="diarization_llm_model cannot be empty.", - ) - evaluation.diarisation_llm_model = new_diariser_model - - if payload.diarization_llm_credential_id is not None: - evaluation.diarisation_llm_credential_id = ( - payload.diarization_llm_credential_id - ) - - # ``diarization_prompt`` semantics: None = leave untouched; - # empty string = clear (fall back to the canonical default at - # worker time); anything else = persist verbatim. - if payload.diarization_prompt is not None: - cleaned = payload.diarization_prompt.strip() - evaluation.diarisation_prompt = cleaned or None - - if payload.transcribe_mode is not None: - mode = payload.transcribe_mode.strip().lower() - if mode not in {"stt_llm", "llm_only"}: - raise HTTPException( - status_code=400, - detail=( - f"Unknown transcribe_mode '{payload.transcribe_mode}'. " - "Valid values are 'stt_llm' and 'llm_only'." - ), - ) - evaluation.transcribe_mode = mode - - -def _gather_retry_targets( - db: Session, - evaluation: CallImportEvaluation, - requested_ids: Optional[List[UUID]], - *, - include_completed: bool = False, -) -> Tuple[ - List[Tuple[CallImportEvaluationRow, CallImportRow]], - List[CallImportEvaluationRetrySkippedItem], -]: - """Resolve which rows to retry + reasons for any we refuse. - - When ``requested_ids`` is None we retry every row whose status is - ``failed`` (or every row when ``include_completed`` is also set — - used by the metric-subset retry path which legitimately wants to - recompute a metric on already-successful rows). When the caller - passes ids explicitly we still filter out rows that are currently - in flight; ``include_completed`` controls whether previously- - successful rows are eligible. - """ - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - from app.db_sharding.eval_rows import gather_retry_targets_sharded - - return gather_retry_targets_sharded( - db, - evaluation, - requested_ids, - include_completed=include_completed, - ) - - eval_rows_query = db.query(CallImportEvaluationRow).filter( - CallImportEvaluationRow.evaluation_id == evaluation.id - ) - - targets: List[Tuple[CallImportEvaluationRow, CallImportRow]] = [] - skipped: List[CallImportEvaluationRetrySkippedItem] = [] - - if requested_ids is None: - if include_completed: - # "Retry everything" path used by the metric-subset re-run - # UI. Still skip in-flight rows below so we don't trample - # work the worker is actively doing. - candidate_rows = eval_rows_query.filter( - CallImportEvaluationRow.status.in_(["failed", "completed"]) - ).all() - else: - candidate_rows = eval_rows_query.filter( - CallImportEvaluationRow.status == "failed" - ).all() - else: - requested_set = set(requested_ids) - candidate_rows = eval_rows_query.filter( - CallImportEvaluationRow.id.in_(requested_set) - ).all() - found_ids = {row.id for row in candidate_rows} - for missing in requested_set - found_ids: - skipped.append( - CallImportEvaluationRetrySkippedItem( - eval_row_id=missing, - reason="unknown", - ) - ) - - if not candidate_rows: - return targets, skipped - - source_row_ids = [row.call_import_row_id for row in candidate_rows] - source_rows = ( - db.query(CallImportRow) - .filter(CallImportRow.id.in_(source_row_ids)) - .all() - ) - source_by_id = {row.id: row for row in source_rows} - - for eval_row in candidate_rows: - if eval_row.status in {"pending", "running"}: - skipped.append( - CallImportEvaluationRetrySkippedItem( - eval_row_id=eval_row.id, - reason="in_progress", - ) - ) - continue - if eval_row.status == "completed" and not include_completed: - skipped.append( - CallImportEvaluationRetrySkippedItem( - eval_row_id=eval_row.id, - reason="completed", - ) - ) - continue - source_row = source_by_id.get(eval_row.call_import_row_id) - if source_row is None: - skipped.append( - CallImportEvaluationRetrySkippedItem( - eval_row_id=eval_row.id, - reason="source_row_missing", - ) - ) - continue - targets.append((eval_row, source_row)) - - return targets, skipped - - -@router.post( - "/{eval_id}/retry", - response_model=CallImportEvaluationRetryResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="retryCallImportEvaluation", -) -async def retry_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - payload: Optional[CallImportEvaluationRetryRequest] = Body(default=None), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportEvaluationRetryResponse: - """Re-enqueue failed rows in an evaluation run. - - Default behavior (no body) is "retry every row that failed". Pass - ``eval_row_ids`` to scope the retry to a specific subset (e.g. the - single row a user clicked in the UI). Rows that are still - in-flight or already completed are returned in ``skipped`` rather - than re-enqueued, so this endpoint is always safe to call. - - When ``metric_ids`` is set in the payload, this is a **metric- - subset retry**: only the listed metrics are recomputed (and merged - into the row's existing ``metric_scores`` — other metrics' values - are preserved). The route auto-flips ``include_completed=True`` in - that case so previously-successful rows are eligible for re- - scoring; without it the call would no-op because every row would - be skipped as ``completed``. - - The worker contract is the same as the create endpoint: - ``evaluate_call_import_row_task(eval_row_id, [restricted_metric_ids])``. - When the run is configured for diarised transcripts and the row's - diarised transcript is missing, we chain through - ``transcribe_call_import_row_task`` first — matching the - auto-transcribe behavior of POST ``/evaluations``. - """ - del api_key - call_import = _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - requested_ids = payload.eval_row_ids if payload else None - # Metric-subset retry: validate that every metric is something this - # run actually scored. Empty list is rejected too — callers that - # want a full re-run should omit the field entirely. - # - # ``selected_metric_ids`` holds the LEAVES only (children for - # hierarchical / category metrics, standalone metrics otherwise) — - # see ``leaf_metric_ids`` in :func:`create_call_import_evaluation`. - # Parent IDs for hierarchical metrics live separately in - # ``selected_metric_groups`` (``{parent_id: [child_ids]}``) so the - # UI can reconstruct the tree without round-tripping through the - # metric table. - # - # The Re-run-metrics modal surfaces PARENTS for hierarchical - # metrics (it suppresses individual children via - # ``childrenInGroups`` in ``CallImportEvaluationDetail.tsx``), so a - # naive ``metric_ids ⊆ selected_metric_ids`` check rejects every - # parent-ID request with a misleading "unknown ids" 400. We accept - # both shapes here and then EXPAND any parent IDs into - # ``{parent_id, *child_ids}`` so the downstream helpers see the - # full set of keys that need clearing + the full set of leaves - # that need re-scoring. - metric_ids: Optional[List[UUID]] = ( - payload.metric_ids if payload else None - ) - if metric_ids is not None: - if not metric_ids: - raise HTTPException( - status_code=400, - detail=( - "metric_ids must be a non-empty list. Omit the " - "field to re-run all metrics." - ), - ) - - leaf_set: Set[str] = { - str(item).lower() - for item in (evaluation.selected_metric_ids or []) - } - # ``selected_metric_groups`` is a dict ``{parent_id_str: - # [child_id_str, ...]}`` (see line ~487 in - # ``create_call_import_evaluation``). We tolerate stale data - # (string / UUID / non-dict) without crashing the retry path — - # if it's malformed we just treat it as "no parents" and fall - # back to the leaf-only check. - groups_raw = ( - evaluation.selected_metric_groups - if isinstance(evaluation.selected_metric_groups, dict) - else {} - ) - parent_to_children_str: Dict[str, List[str]] = {} - for parent_key, children_raw in groups_raw.items(): - if not isinstance(children_raw, (list, tuple)): - continue - children_norm = [ - str(c).lower() for c in children_raw if c is not None - ] - parent_to_children_str[str(parent_key).lower()] = children_norm - parent_set = set(parent_to_children_str.keys()) - - unknown = [ - mid for mid in metric_ids - if str(mid).lower() not in leaf_set - and str(mid).lower() not in parent_set - ] - if unknown: - raise HTTPException( - status_code=400, - detail=( - "metric_ids must be a subset of this evaluation's " - f"selected metrics; unknown ids: {[str(u) for u in unknown]}." - ), - ) - - # Expand parent IDs into ``{parent, *children}`` so: - # * ``_reset_eval_row_for_retry`` strips BOTH the parent - # entry (with ``chosen_child_id`` / rationale) AND every - # per-child boolean entry that the LLM evaluator wrote - # under each child's ID (see - # ``app/workers/tasks/helpers/llm_evaluation.py`` lines - # 1584 and 1649). - # * ``_enqueue_eval_rows_with_optional_transcribe`` → - # ``evaluate_call_import_row_task`` filters the work-list - # off ``selected_metric_ids`` (leaves), so we MUST hand it - # the child IDs for the parent to actually get re-scored. - # Leaves pass through unchanged. - expanded: List[UUID] = [] - seen: Set[str] = set() - for mid in metric_ids: - mid_norm = str(mid).lower() - children_str = parent_to_children_str.get(mid_norm) - if children_str is not None: - # Parent: include the parent ID itself (so the parent - # entry in ``metric_scores`` is also cleared) and all - # of its children. - candidates = [mid_norm, *children_str] - else: - candidates = [mid_norm] - for candidate in candidates: - if candidate in seen: - continue - try: - expanded.append(UUID(candidate)) - except (TypeError, ValueError): - # Defensive: skip junk values rather than 500. - continue - seen.add(candidate) - metric_ids = expanded - - # ``include_completed`` is auto-enabled when the caller asked for a - # metric subset (otherwise the metric-subset retry would always - # no-op on a green run, which is the whole reason this feature - # exists). The explicit payload flag wins for full-row retries. - include_completed = bool( - (payload.include_completed if payload else False) - or (metric_ids is not None) - ) - - transcribe_overwrite = bool( - payload.transcribe_overwrite if payload else False - ) - - skipped: List[CallImportEvaluationRetrySkippedItem] = [] - if requested_ids is None: - from app.db_sharding.eval_rows import count_evaluation_rows_for_run - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - statuses = ( - ["failed", "completed"] if include_completed else ["failed"] - ) - target_count = count_evaluation_rows_for_run( - db, eval_id, statuses=statuses - ) - else: - from sqlalchemy import func - - count_query = db.query(func.count(CallImportEvaluationRow.id)).filter( - CallImportEvaluationRow.evaluation_id == eval_id - ) - if include_completed: - count_query = count_query.filter( - CallImportEvaluationRow.status.in_(["failed", "completed"]) - ) - else: - count_query = count_query.filter( - CallImportEvaluationRow.status == "failed" - ) - target_count = int(count_query.scalar() or 0) - if target_count == 0: - return CallImportEvaluationRetryResponse( - requeued=0, - transcribe_requeued=0, - skipped=skipped, - ) - else: - targets, skipped = _gather_retry_targets( - db, - evaluation, - requested_ids, - include_completed=include_completed, - ) - if not targets: - return CallImportEvaluationRetryResponse( - requeued=0, - transcribe_requeued=0, - skipped=skipped, - ) - target_count = len(targets) - - # Apply LLM / STT overrides BEFORE enqueueing so the persisted run - # config is correct by the time the worker reads it. - if payload is not None: - _apply_retry_overrides(db, evaluation, organization_id, payload) - _apply_telephony_retry_overrides( - db, - call_import=call_import, - organization_id=organization_id, - payload=payload, - ) - - evaluation.error_message = None - evaluation.finished_at = None - evaluation.status = "running" - if not evaluation.started_at: - from datetime import datetime, timezone - - evaluation.started_at = datetime.now(timezone.utc) - - _claim_evaluation_bulk_operation(eval_id, "retry") - stamp_evaluation_actor(evaluation, principal) - db.commit() - - from app.workers.tasks.call_import_bulk_ops import ( - retry_call_import_evaluation_task, - ) - - retry_call_import_evaluation_task.delay( - str(eval_id), - { - "eval_row_ids": [str(rid) for rid in requested_ids] - if requested_ids - else None, - "metric_ids": [str(mid) for mid in metric_ids] if metric_ids else None, - "include_completed": include_completed, - "transcribe_overwrite": transcribe_overwrite, - }, - ) - - return CallImportEvaluationRetryResponse( - requeued=target_count, - transcribe_requeued=0, - skipped=skipped, - ) - - -@router.post( - "/{eval_id}/rows/{eval_row_id}/retry", - response_model=CallImportEvaluationRowResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="retryCallImportEvaluationRow", -) -async def retry_call_import_evaluation_row( - call_import_id: UUID, - eval_id: UUID, - eval_row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportEvaluationRowResponse: - """Re-enqueue a single failed evaluation row. - - Convenience wrapper around ``retry_call_import_evaluation`` for the - "Retry this row" affordance in the row table. Returns the - refreshed row so the UI can update its badge immediately, without - waiting for the next polling tick. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - _require_no_evaluation_bulk_operation(eval_id) - - from app.db_sharding.eval_rows import ( - evaluation_row_session, - find_evaluation_row_in_run, - ) - from app.db_sharding.sessions import is_sharding_enabled - - eval_row, _source_stub = find_evaluation_row_in_run(db, eval_id, eval_row_id) - if eval_row is None: - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) - - if eval_row.status in {"pending", "running"}: - raise HTTPException( - status_code=409, - detail=( - "This row is still in progress — wait for it to finish " - "before retrying." - ), - ) - - targets, _ = _gather_retry_targets(db, evaluation, [eval_row.id]) - if not targets: - raise HTTPException( - status_code=409, - detail=( - "This row cannot be retried in its current state " - f"(status={eval_row.status})." - ), - ) - - if is_sharding_enabled(): - with evaluation_row_session(eval_row_id) as ( - row_db, - _catalog_db, - eval_row, - source_row, - _shard_id, - ): - _prepare_source_row_for_retry(source_row, transcribe_overwrite=False) - _reset_eval_row_for_retry(eval_row) - row_db.commit() - targets = [(eval_row, source_row)] - else: - for er, source_row in targets: - _prepare_source_row_for_retry(source_row, transcribe_overwrite=False) - _reset_eval_row_for_retry(er) - - evaluation.error_message = None - evaluation.finished_at = None - evaluation.status = "running" - if not evaluation.started_at: - from datetime import datetime, timezone - - evaluation.started_at = datetime.now(timezone.utc) - db.flush() - _rollup_evaluation_status(evaluation, db) - stamp_evaluation_actor(evaluation, principal) - db.commit() - - try: - _enqueue_eval_rows_with_optional_transcribe(db, evaluation, targets) - except Exception as exc: # noqa: BLE001 - logger.exception( - "Failed to re-enqueue retry for evaluation row {}", eval_row_id - ) - if is_sharding_enabled(): - with evaluation_row_session(eval_row_id) as ( - row_db, - _catalog_db, - eval_row, - _source_row, - _shard_id, - ): - eval_row.status = "failed" - eval_row.error_message = f"Failed to re-enqueue retry: {exc}" - row_db.commit() - else: - eval_row.status = "failed" - eval_row.error_message = f"Failed to re-enqueue retry: {exc}" - _rollup_evaluation_status(evaluation, db) - db.commit() - raise HTTPException( - status_code=500, - detail=f"Failed to re-enqueue retry: {exc}", - ) - - if is_sharding_enabled(): - with evaluation_row_session(eval_row_id) as ( - _row_db, - _catalog_db, - eval_row, - source_row, - _shard_id, - ): - return _to_evaluation_row_response(eval_row, source_row, evaluation) - - db.refresh(eval_row) - source_row = targets[0][1] - return _to_evaluation_row_response(eval_row, source_row, evaluation) - - -from app.core.auth.capabilities import EVALS_RUN, EVALS_VIEW, REPORTS_GENERATE -from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities - -apply_workspace_route_capabilities( - router, - view_capability=EVALS_VIEW, - manage_capability=EVALS_RUN, - run_capability=EVALS_RUN, - report_capability=REPORTS_GENERATE, -) +"""Evaluation routes scoped to a Call Import batch.""" + +from __future__ import annotations + +import asyncio +import csv +import base64 +import io +import json +import math +import re +import statistics +from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple +from uuid import UUID, uuid4 + +from datetime import date, datetime, timedelta, timezone + +from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status +from fastapi.responses import StreamingResponse +from loguru import logger +from pydantic import BaseModel, Field, field_validator +from sqlalchemy import desc, func, or_, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.core.auth import Principal, get_principal +from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message +from app.database import get_db +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.call_imports.audit import ( + actor_emails_for_evaluation, + emails_for_user_ids, + stamp_call_import_actor, + stamp_evaluation_actor, + user_ids_from_evaluations, +) +from app.services.workspace_rbac import resolve_workspace_capabilities +from app.models.database import ( + AIProvider, + CallImport, + CallImportEvaluation, + CallImportEvaluationReportSnapshot, + CallImportEvaluationPdfReport, + CallImportEvaluationRow, + CallImportRow, + Metric, + PromptPartial, + Workspace, +) +from app.models.enums import CallImportRowStatus, ModelProvider +from app.models.schemas import ( + CallImportEvaluationAggregateResponse, + CallImportEvaluationBulkDelete, + CallImportEvaluationBulkActionResponse, + CallImportEvaluationCreate, + CallImportEvaluationListResponse, + CallImportEvaluationResponse, + CallImportEvaluationRetryRequest, + CallImportEvaluationRetryResponse, + CallImportEvaluationRetrySkippedItem, + CallImportEvaluationRowListResponse, + CallImportEvaluationRowResponse, + CallImportEvaluationUpdate, + CallImportMetricAggregate, + CallImportMetricHistogramBucket, + CallImportMetricLabelPair, + CallImportMetricSummary, + CallImportMetricValueCount, + DiscoveredLabelDeleteRequest, + DiscoveredLabelItem, + DiscoveredLabelMergeRequest, + DiscoveredLabelsResponse, + DiscoveredMetricDeleteRequest, + DiscoveredMetricItem, + DiscoveredMetricMergeRequest, + DiscoveredMetricsResponse, + EvaluationInsightsRequest, + EvaluationTldrSummary, + EvaluationMetricClustersRequest, + EvaluationMetricClustersState, + EvaluationPromptImprovementsRequest, + EvaluationPromptImprovementsState, + MetricFailurePoliciesResponse, + MetricFailurePoliciesSaveRequest, + MetricFailurePolicy, + MetricClusterEligibleRow, + MetricClusterEligibleRowsResponse, + EvaluationUserInsightsRequest, + EvaluationUserInsightsState, + MetricFlowEdge, + MetricPeriodDelta, + MetricFlowNode, + MetricFlowResponse, +) +from app.services.reporting.call_import_evaluation_pdf_report import ( + call_import_evaluation_pdf_report_service, +) +from app.services.reporting.call_import_pdf_report_storage import ( + build_pdf_report_s3_key, + compute_pdf_report_cache_fingerprint, + compute_pdf_report_config_fingerprint, + compute_pdf_report_content_fingerprint, + config_summary_from_report_config, + find_cached_pdf_report, + presigned_urls_for_pdf_report, +) +from app.services.call_import_metric_clusters import ( + METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + estimate_metric_clusters_llm_calls, + filter_completed_row_pairs, + list_eligible_cluster_rows, + metric_clusters_raw_is_cancelled, + metric_clusters_state_from_raw, + metric_clusters_state_to_db, +) +from app.services.metric_failure_policy import ( + aggregate_primary_percent, + build_failure_policy_previews, + effective_policies, + failure_rate_percent_from_rows, + failure_policies_to_db, + has_clusterable_metrics, + merge_clustering_policies, + merge_failure_policies_into_raw, + policies_from_evaluation_raw, + validate_failure_policies_for_metrics, +) +from app.services.call_import_user_insights import ( + normalize_max_llm_calls, + total_llm_calls_for_rows, + user_insights_state_from_raw, +) + +router = APIRouter( + prefix="/call-imports/{call_import_id}/evaluations", + tags=["Call Import Evaluations"], + dependencies=[Depends(require_enterprise_feature("call_imports"))], +) + + +class CallImportEvaluationPdfReportRequest(BaseModel): + vendor_name: str = Field(..., min_length=1, max_length=120) + report_type: Literal["external", "internal"] = "external" + include_weekly_delta: bool = False + include_period_delta: bool = False + baseline_evaluation_id: Optional[str] = None + period_label: Optional[str] = Field(default=None, max_length=64) + use_case: Optional[str] = Field(default=None, max_length=120) + internal_brand_image_id: Optional[str] = None + external_brand_image_id: Optional[str] = None + report_config: Dict[str, Any] = Field(default_factory=dict) + platform_base_url: Optional[str] = Field( + default=None, + max_length=512, + description="Frontend origin for deep links to example calls in internal PDFs.", + ) + + @field_validator("vendor_name") + @classmethod + def _clean_vendor_name(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("Vendor name is required.") + return cleaned + + +class CallImportEvaluationPdfReportResponse(BaseModel): + id: str + filename: str + preview_url: Optional[str] = None + download_url: Optional[str] = None + created_at: datetime + created_by: Optional[str] = None + report_type: str + vendor_name: str + config_summary: Optional[str] = None + storage_available: bool = True + cache_hit: bool = False + + +class CallImportEvaluationPdfReportListItem(BaseModel): + id: str + filename: Optional[str] = None + vendor_name: str + report_type: str + created_by: Optional[str] = None + created_at: datetime + config_summary: Optional[str] = None + cache_fingerprint: Optional[str] = None + + +class CallImportEvaluationPdfReportListResponse(BaseModel): + items: List[CallImportEvaluationPdfReportListItem] + + +class CallImportEvaluationBaselineCandidate(BaseModel): + evaluation_id: str + name: str + dataset: str + period_label: Optional[str] = None + period_start: Optional[date] = None + period_end: Optional[date] = None + period_display: str + completed_rows: int + created_at: datetime + is_default: bool = False + + +class CallImportEvaluationBaselineCandidatesResponse(BaseModel): + items: List[CallImportEvaluationBaselineCandidate] + default_evaluation_id: Optional[str] = None + + +def _require_import( + db: Session, + call_import_id: UUID, + organization_id: UUID, +) -> CallImport: + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException(status_code=404, detail="Call import not found") + return call_import + + +def require_call_import_capability(capability: str): + """Ensure the caller has *capability* in the call import's workspace (not just the header).""" + + def _dep( + call_import_id: UUID, + principal: Principal = Depends(get_principal), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), + ) -> CallImport: + call_import = _require_import(db, call_import_id, organization_id) + caps, _, role = resolve_workspace_capabilities( + db, + principal=principal, + workspace_id=call_import.workspace_id, + organization_id=organization_id, + ) + if capability not in caps: + raise HTTPException( + status_code=403, + detail=capability_denied_message( + capability, + role_name=role.name if role else None, + workspace_label="the active workspace", + ), + ) + return call_import + + return _dep + + +def _flatten_transcript(text: Optional[str]) -> str: + """Collapse a multi-line transcript onto a single line for spreadsheet export. + + The diarised transcript is stored as ``: `` lines joined + by ``\\n`` because the in-app ``TranscriptView`` parses those line + breaks to render chat bubbles. In Excel / Google Sheets that same + newline-per-turn formatting causes each cell to balloon vertically, + which the user reads as "lots of empty space on top of the cell". + Flattening at export time keeps the DB shape intact while giving the + spreadsheet a single-line cell per row. + """ + if not text: + return "" + parts = [ + segment.strip() + for segment in text.replace("\r\n", "\n").replace("\r", "\n").split("\n") + ] + return " ".join(p for p in parts if p) + + +def _evaluated_transcript_source_label( + evaluation: CallImportEvaluation, + source_row: CallImportRow, +) -> str: + """Label which transcript source this row was scored against.""" + source = (evaluation.transcript_source or "diarised").strip().lower() + if source == "production": + if not (source_row.transcript or "").strip(): + return "" + return "Production" + if not (source_row.diarised_transcript or "").strip(): + return "" + return "Diarised" + + +def _pick_evaluation_row_transcript( + source_row: Optional[CallImportRow], + evaluation: Optional[CallImportEvaluation] = None, +) -> Optional[str]: + """Transcript shown in evaluation row detail for the run's source.""" + if source_row is None: + return None + source = ( + (evaluation.transcript_source or "diarised").strip().lower() + if evaluation is not None + else "diarised" + ) + if source == "production": + raw = (source_row.transcript or "").strip() + return raw or None + diarised = (source_row.diarised_transcript or "").strip() + if diarised: + return diarised + raw = (source_row.transcript or "").strip() + return raw or None + + +def _to_evaluation_row_response( + eval_row_obj: CallImportEvaluationRow, + source_row: Optional[CallImportRow], + evaluation: Optional[CallImportEvaluation] = None, +) -> CallImportEvaluationRowResponse: + """Serialize one evaluation row plus joined source-row metadata.""" + return CallImportEvaluationRowResponse( + id=eval_row_obj.id, + evaluation_id=eval_row_obj.evaluation_id, + call_import_row_id=eval_row_obj.call_import_row_id, + row_index=source_row.row_index if source_row else None, + conversation_id=source_row.conversation_id if source_row else None, + transcript=_pick_evaluation_row_transcript(source_row, evaluation), + raw_columns=source_row.raw_columns if source_row else None, + recording_url=source_row.recording_url if source_row else None, + recording_date=source_row.recording_date if source_row else None, + recording_s3_key=source_row.recording_s3_key if source_row else None, + diarised_transcript_status=( + source_row.diarised_transcript_status if source_row else None + ), + diarised_transcript_error=( + source_row.diarised_transcript_error if source_row else None + ), + status=eval_row_obj.status, + metric_scores=eval_row_obj.metric_scores or {}, + error_message=eval_row_obj.error_message, + started_at=eval_row_obj.started_at, + finished_at=eval_row_obj.finished_at, + created_at=eval_row_obj.created_at, + updated_at=eval_row_obj.updated_at, + ) + + +def _serialize_selected_metric_ids(value) -> List[UUID]: + result: List[UUID] = [] + if not isinstance(value, list): + return result + for item in value: + try: + result.append(UUID(str(item))) + except (TypeError, ValueError): + continue + return result + + +def _metrics_for_ids(db: Session, org_id: UUID, ids: List[UUID]) -> List[Metric]: + if not ids: + return [] + rows = ( + db.query(Metric) + .filter( + Metric.organization_id == org_id, + Metric.id.in_(ids), + ) + .all() + ) + by_id = {row.id: row for row in rows} + return [by_id[mid] for mid in ids if mid in by_id] + + +def _expand_metric_selection( + db: Session, + org_id: UUID, + selected_ids: List[UUID], +) -> Tuple[List[Metric], Dict[UUID, List[Metric]]]: + """Resolve user-supplied metric ids into actual leaves + parent grouping. + + Rules: + * If a parent id is in ``selected_ids`` and no specific children of + that parent are also listed, include EVERY enabled child of that + parent. + * If a parent id AND some of its children are listed, include only + the listed children (treat the parent selection as the + "container" so users can deselect labels). + * Standalone metrics (no parent, no children) pass through + unchanged. + * Disabled metrics are filtered out at this layer so the caller + doesn't have to repeat the check. + + Returns: + (effective_metrics, parent_to_children) + + ``effective_metrics`` is the deduplicated list of metrics the + worker will actually score (children + standalone). Order is + preserved from ``selected_ids`` for display stability. + + ``parent_to_children`` maps each parent metric id (UUID) to the + list of its selected children. Useful for grouping in the LLM + prompt builder. + """ + if not selected_ids: + return [], {} + + requested = list(selected_ids) + initial_rows = ( + db.query(Metric) + .filter( + Metric.organization_id == org_id, + Metric.id.in_(requested), + ) + .all() + ) + initial_by_id = {row.id: row for row in initial_rows} + + parent_ids_requested = { + m.id for m in initial_rows if m.selection_mode and not m.parent_metric_id + } + # Map parent id -> children explicitly requested by the user. + explicit_children_by_parent: Dict[UUID, List[Metric]] = {} + for m in initial_rows: + if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: + explicit_children_by_parent.setdefault( + m.parent_metric_id, [] + ).append(m) + + # For parents without explicit children, hydrate every enabled child. + parents_needing_full_expansion = [ + pid + for pid in parent_ids_requested + if pid not in explicit_children_by_parent + ] + auto_expanded_children: Dict[UUID, List[Metric]] = {} + if parents_needing_full_expansion: + for pid in parents_needing_full_expansion: + child_rows = ( + db.query(Metric) + .filter( + Metric.organization_id == org_id, + Metric.parent_metric_id == pid, + Metric.enabled.is_(True), + ) + .order_by(Metric.created_at.asc()) + .all() + ) + auto_expanded_children[pid] = child_rows + + parent_to_children: Dict[UUID, List[Metric]] = {} + for pid in parent_ids_requested: + children = explicit_children_by_parent.get( + pid + ) or auto_expanded_children.get(pid, []) + # Drop disabled children so the worker doesn't waste a slot on + # them. Empty parents (no enabled children) are still tracked + # because the UI may want to show "0 of 0" rather than swallow + # them silently. + parent_to_children[pid] = [c for c in children if c.enabled] + + effective: List[Metric] = [] + seen: set[UUID] = set() + for mid in requested: + m = initial_by_id.get(mid) + if m is None: + continue + if m.selection_mode and not m.parent_metric_id: + # Parent row itself is not scored — only its children. + for child in parent_to_children.get(m.id, []): + if child.id in seen or not child.enabled: + continue + seen.add(child.id) + effective.append(child) + continue + if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: + # Already accounted for via the parent expansion above. + continue + if not m.enabled: + continue + if m.id in seen: + continue + seen.add(m.id) + effective.append(m) + + return effective, parent_to_children + + +def _evaluation_bulk_operation_for_response( + evaluation_id: UUID, +) -> Optional[str]: + from app.services.call_imports.evaluation_bulk_op import ( + get_evaluation_bulk_operation, + ) + + return get_evaluation_bulk_operation(evaluation_id) + + +def _serialize_eval( + db: Session, + row: CallImportEvaluation, + *, + sibling_evaluation_ids: Optional[List[UUID]] = None, + user_emails: Optional[Dict[UUID, str]] = None, +) -> CallImportEvaluationResponse: + selected_ids = _serialize_selected_metric_ids(row.selected_metric_ids) + + # Pull every metric referenced anywhere in the run's grouping (leaves, + # standalone, AND parents from selected_metric_groups) so the UI can + # render parent labels even when only children were materialized into + # selected_metric_ids. + groups_raw: Dict[str, List[str]] = {} + if isinstance(row.selected_metric_groups, dict): + for parent_str, children in row.selected_metric_groups.items(): + if not isinstance(children, list): + continue + cleaned: List[str] = [] + for c in children: + try: + UUID(str(c)) + cleaned.append(str(c)) + except (TypeError, ValueError): + continue + try: + UUID(parent_str) + groups_raw[parent_str] = cleaned + except (TypeError, ValueError): + continue + + metric_ids_for_lookup: List[UUID] = list(selected_ids) + for parent_str in groups_raw.keys(): + try: + pid = UUID(parent_str) + if pid not in metric_ids_for_lookup: + metric_ids_for_lookup.append(pid) + except (TypeError, ValueError): + continue + + metrics = _metrics_for_ids( + db, row.organization_id, metric_ids_for_lookup + ) + + from app.services.call_imports.progress_counters import merge_eval_counters_for_ui + + ui_completed_raw, ui_failed_raw = merge_eval_counters_for_ui(row) + total = int(row.total_rows or 0) + ui_completed = ( + min(ui_completed_raw, total) if total else ui_completed_raw + ) + ui_failed = min(ui_failed_raw, total) if total else ui_failed_raw + + if user_emails is None: + user_emails = emails_for_user_ids(db, user_ids_from_evaluations([row])) + created_email, updated_email = actor_emails_for_evaluation(row, user_emails) + + return CallImportEvaluationResponse( + id=row.id, + call_import_id=row.call_import_id, + organization_id=row.organization_id, + name=row.name, + selected_metric_ids=selected_ids, + selected_metric_groups=groups_raw or None, + metrics=[ + CallImportMetricSummary( + id=metric.id, + name=metric.name, + metric_type=metric.metric_type, + description=metric.description, + parent_metric_id=metric.parent_metric_id, + selection_mode=metric.selection_mode, + # Required by the Flow tab to know whether a parent + # opted into discovery; without it the + # DiscoveredLabelsPanel stays hidden even when the + # worker is actively producing discovered_labels. + allow_discovery=bool( + getattr(metric, "allow_discovery", False) + ), + ) + for metric in metrics + ], + status=row.status, + total_rows=row.total_rows, + completed_rows=ui_completed, + failed_rows=ui_failed, + error_message=row.error_message, + llm_provider=row.llm_provider, + llm_model=row.llm_model, + llm_credential_id=row.llm_credential_id, + llm_config=( + row.llm_config if isinstance(getattr(row, "llm_config", None), dict) else None + ), + metric_llm_overrides=( + row.metric_llm_overrides + if isinstance(row.metric_llm_overrides, dict) + else None + ), + stt_provider=row.stt_provider, + stt_model=row.stt_model, + stt_credential_id=row.stt_credential_id, + diarisation_llm_provider=getattr(row, "diarisation_llm_provider", None), + diarisation_llm_model=getattr(row, "diarisation_llm_model", None), + diarisation_llm_credential_id=getattr( + row, "diarisation_llm_credential_id", None + ), + diarisation_prompt=getattr(row, "diarisation_prompt", None), + transcribe_mode=( + (getattr(row, "transcribe_mode", None) or "stt_llm") + ), + transcript_source=(row.transcript_source or "diarised"), + sibling_evaluation_ids=list(sibling_evaluation_ids or []), + started_at=row.started_at, + finished_at=row.finished_at, + created_at=row.created_at, + updated_at=row.updated_at, + created_by_email=created_email, + last_updated_by_email=updated_email, + tldr_summary=_tldr_summary_payload(row), + user_insights=_user_insights_payload(row), + metric_clusters=_metric_clusters_payload(row), + discover_new_metrics=bool( + getattr(row, "discover_new_metrics", False) + ), + bulk_operation=_evaluation_bulk_operation_for_response(row.id), + ) + + +def _normalize_name(value: Optional[str]) -> Optional[str]: + """Trim user-supplied name; empty string becomes ``NULL``.""" + if value is None: + return None + trimmed = value.strip() + return trimmed or None + + +def _rollup_evaluation_status(evaluation: CallImportEvaluation, db: Session) -> None: + """Recompute counters + terminal status after rows are added/removed. + + Uses a single aggregate query instead of loading every row status. + """ + from app.workers.tasks.evaluate_call_import_row_core import ( + _apply_parent_status_from_counters, + reconcile_evaluation_counters, + ) + + reconcile_evaluation_counters(db, evaluation) + _apply_parent_status_from_counters(evaluation) + db.flush() + + if evaluation.status in {"completed", "failed", "partial"}: + from app.models.database import CallImport + from app.services.call_imports.bulk_ops import rollup_call_import_batch_status + + call_import = ( + db.query(CallImport) + .filter(CallImport.id == evaluation.call_import_id) + .first() + ) + if call_import is not None: + rollup_call_import_batch_status(db, call_import) + + +@router.post( + "", + response_model=CallImportEvaluationResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="createCallImportEvaluation", +) +async def create_call_import_evaluation( + call_import_id: UUID, + payload: CallImportEvaluationCreate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationResponse: + del api_key + call_import = _require_import(db, call_import_id, organization_id) + + metric_ids = payload.metric_ids + if not metric_ids: + raise HTTPException( + status_code=400, + detail="Select at least one metric to run the evaluation against.", + ) + + org_metrics = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.id.in_(metric_ids), + ) + .all() + ) + by_id = {metric.id: metric for metric in org_metrics} + unknown_ids = [mid for mid in metric_ids if mid not in by_id] + if unknown_ids: + raise HTTPException( + status_code=400, + detail=( + "These metric ids do not exist in your organization: " + f"{', '.join(str(mid) for mid in unknown_ids)}. " + "Refresh the metrics list and try again." + ), + ) + draft_metrics = [ + metric + for metric in org_metrics + if (getattr(metric, "lifecycle", None) or "active") == "draft" + ] + if draft_metrics: + names = ", ".join(metric.name for metric in draft_metrics) + raise HTTPException( + status_code=400, + detail=( + f"Draft metrics cannot be used in call import evaluations: {names}. " + "Promote them in Metrics Studio first." + ), + ) + # Parents themselves are containers, not scored rows, so a disabled + # parent shouldn't block the run as long as it has enabled children. + # We only reject disabled rows that the worker will actually try to + # evaluate (children + standalone leaves). + disabled_leaves = [ + metric + for metric in org_metrics + if not metric.enabled + and not (metric.selection_mode and not metric.parent_metric_id) + ] + if disabled_leaves: + names = ", ".join(metric.name for metric in disabled_leaves) + raise HTTPException( + status_code=400, + detail=( + f"These metrics are disabled and cannot be evaluated: {names}. " + "Enable them on the Metrics page (or pick different ones) and " + "try again." + ), + ) + + # Expand hierarchical selection: parents auto-include their enabled + # children, mixed parent+child selections respect the user's subset. + effective_metrics, parent_to_children = _expand_metric_selection( + db, organization_id, metric_ids + ) + if not effective_metrics: + raise HTTPException( + status_code=400, + detail=( + "None of the selected metrics yielded an enabled leaf to " + "evaluate. Check that parent categories have enabled " + "children, then try again." + ), + ) + + # The effective list (children + standalone leaves) is what gets + # persisted to ``selected_metric_ids`` and scored by the worker. + # The original parents are preserved in ``selected_metric_groups`` + # so the UI can rebuild the tree later. + leaf_metric_ids: List[UUID] = [m.id for m in effective_metrics] + selected_metric_groups: Dict[str, List[str]] = { + str(pid): [str(c.id) for c in children] + for pid, children in parent_to_children.items() + } + metric_rows = effective_metrics + valid_metric_id_strs = {str(m.id) for m in metric_rows} + + # ----- Validate run-level + per-metric LLM config ----- + llm_provider_norm: Optional[str] = None + llm_model_norm: Optional[str] = None + if payload.llm_provider or payload.llm_model: + if not (payload.llm_provider and payload.llm_model): + raise HTTPException( + status_code=400, + detail="Both llm_provider and llm_model are required when overriding the run LLM.", + ) + try: + llm_provider_norm = ModelProvider( + payload.llm_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Unknown LLM provider '{payload.llm_provider}'. " + "Valid keys are documented in ModelProvider." + ), + ) + llm_model_norm = payload.llm_model.strip() or None + if not llm_model_norm: + raise HTTPException( + status_code=400, detail="llm_model cannot be empty." + ) + + if payload.llm_credential_id is not None: + cred = ( + db.query(AIProvider) + .filter( + AIProvider.id == payload.llm_credential_id, + AIProvider.organization_id == organization_id, + ) + .first() + ) + if not cred: + raise HTTPException( + status_code=400, + detail=( + "The provided llm_credential_id does not exist in this " + "organization." + ), + ) + + # Per-metric overrides: keys can be either a leaf metric id (applies + # to that metric only) or a parent metric id (applies to every + # child of that parent). Parent keys are expanded to their + # children so the worker only sees concrete leaf ids. + metric_overrides_payload: Optional[Dict[str, Dict[str, Any]]] = None + if payload.metric_llm_overrides: + metric_overrides_payload = {} + for metric_id, override in payload.metric_llm_overrides.items(): + target_leaf_ids: List[str] = [] + if metric_id in valid_metric_id_strs: + target_leaf_ids = [metric_id] + else: + # Maybe it's a parent id — expand to the children that + # are part of THIS run. + try: + parent_uuid = UUID(metric_id) + except (TypeError, ValueError): + raise HTTPException( + status_code=400, + detail=( + "metric_llm_overrides references metric " + f"{metric_id} which is not a valid UUID." + ), + ) + children_for_parent = parent_to_children.get(parent_uuid) + if not children_for_parent: + raise HTTPException( + status_code=400, + detail=( + "metric_llm_overrides references metric " + f"{metric_id} which is not in metric_ids." + ), + ) + target_leaf_ids = [str(c.id) for c in children_for_parent] + + override_dict: Dict[str, Any] = {} + if override.provider is not None: + if not override.model: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} has a provider " + "but no model." + ), + ) + try: + override_dict["provider"] = ModelProvider( + override.provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} uses unknown " + f"provider '{override.provider}'." + ), + ) + override_dict["model"] = override.model.strip() + elif override.model: + # Model without provider doesn't make sense — treat as 400 + # so the UI can fix it instead of silently falling back. + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} has a model but " + "no provider." + ), + ) + if override.credential_id is not None: + override_dict["credential_id"] = str(override.credential_id) + if override.llm_config is not None: + override_dict["llm_config"] = override.llm_config + if override_dict: + for leaf_id in target_leaf_ids: + metric_overrides_payload[leaf_id] = override_dict + + # ----- Validate auto-transcribe settings ----- + # Diarised runs auto-diarise rows missing a diarised transcript and + # require STT + diariser LLM config. Production runs score the CSV + # transcript directly and skip diarisation entirely. + use_diarised = payload.transcript_sources[0] == "diarised" + auto_transcribe = use_diarised + + transcribe_mode_norm: Optional[str] = None + stt_provider_norm: Optional[str] = None + stt_model_norm: Optional[str] = None + diarisation_llm_provider_norm: Optional[str] = None + diarisation_llm_model_norm: Optional[str] = None + diarisation_prompt_norm: Optional[str] = None + + if use_diarised: + transcribe_mode_norm = (payload.transcribe_mode or "stt_llm").strip().lower() + if transcribe_mode_norm not in {"stt_llm", "llm_only"}: + raise HTTPException( + status_code=400, + detail=( + f"Unknown transcribe_mode '{payload.transcribe_mode}'. " + "Expected 'stt_llm' or 'llm_only'." + ), + ) + + if transcribe_mode_norm == "stt_llm": + if not payload.stt_provider: + raise HTTPException( + status_code=400, + detail=( + "stt_provider is required when " + "transcribe_mode='stt_llm': every evaluation run " + "auto-diarises rows that are missing a diarised " + "transcript." + ), + ) + if not payload.stt_model: + raise HTTPException( + status_code=400, + detail=( + "stt_model is required when transcribe_mode='stt_llm'." + ), + ) + try: + stt_provider_norm = ModelProvider( + payload.stt_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Unknown STT provider '{payload.stt_provider}'.", + ) + stt_model_norm = payload.stt_model.strip() or None + if not stt_model_norm: + raise HTTPException( + status_code=400, detail="stt_model cannot be empty." + ) + else: + # llm_only — explicitly reject lingering STT inputs so the + # contract is unambiguous (the worker would ignore them but + # silent acceptance hides accidental misconfiguration). + if (payload.stt_provider or "").strip() or ( + payload.stt_model or "" + ).strip(): + raise HTTPException( + status_code=400, + detail=( + "stt_provider / stt_model must be omitted when " + "transcribe_mode='llm_only'; the LLM consumes the " + "audio directly." + ), + ) + + # --- Validate LLM diariser settings ----- + if not payload.diarization_llm_provider: + raise HTTPException( + status_code=400, + detail=( + "diarization_llm_provider is required: every evaluation " + "run diarises STT output with an LLM." + ), + ) + if not payload.diarization_llm_model: + raise HTTPException( + status_code=400, + detail=( + "diarization_llm_model is required: every evaluation " + "run diarises STT output with an LLM." + ), + ) + try: + diarisation_llm_provider_norm = ModelProvider( + payload.diarization_llm_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Unknown diarisation LLM provider " + f"'{payload.diarization_llm_provider}'." + ), + ) + diarisation_llm_model_norm = ( + payload.diarization_llm_model.strip() or None + ) + if not diarisation_llm_model_norm: + raise HTTPException( + status_code=400, + detail="diarization_llm_model cannot be empty.", + ) + diarisation_prompt_norm = ( + payload.diarization_prompt.strip() + if isinstance(payload.diarization_prompt, str) + else None + ) or None + + from app.models.enums import CallImportParameterType, CallImportStatus + from app.services.call_imports.bulk_ops import ( + count_all_source_rows, + count_completed_source_rows, + count_source_rows_with_production_transcript, + ) + + if use_diarised: + from app.api.v1.routes.call_imports import _is_manual_audio_call_import + + if not _is_manual_audio_call_import(call_import): + if call_import.schema_id: + from app.api.v1.routes.call_imports import ( + _resolve_schema, + _validate_diarised_eval_recording_ready, + ) + + diarised_schema = _resolve_schema( + db, organization_id, call_import.workspace_id, call_import.schema_id + ) + _validate_diarised_eval_recording_ready( + list(diarised_schema.parameters), + dict(call_import.parameter_mapping or {}), + ) + else: + legacy_recording_column = ( + (call_import.column_mapping or {}).get("recording_url") or "" + ).strip() + if not legacy_recording_column: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "Diarize then evaluate requires a recording URL column " + "to be mapped." + ), + ) + + starting_from_mapped = False + if call_import.status == CallImportStatus.MAPPED: + if not call_import.source_s3_key or not call_import.source_format: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This batch has no staged source file. Upload and map " + "a CSV/Excel file before running evaluation." + ), + ) + if not call_import.schema_id: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Cannot run evaluation without a mapped schema.", + ) + from app.api.v1.routes.call_imports import ( + _ensure_blob_storage_enabled, + _resolve_schema, + _resolve_telephony_integration, + _validate_direct_url_import_ready, + _validate_telephony_credentials_live, + ) + + workspace_id = call_import.workspace_id + schema = _resolve_schema( + db, organization_id, workspace_id, call_import.schema_id + ) + parameters = list(schema.parameters) + if not use_diarised: + transcript_mapped = any( + param.type == CallImportParameterType.TRANSCRIPT + and (call_import.parameter_mapping or {}).get(param.name) + for param in parameters + ) + if not transcript_mapped: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "No transcript column is mapped in this batch. " + "Map a schema transcript parameter to a CSV column, " + "or choose 'Diarize then evaluate'." + ), + ) + if payload.telephony_integration_id is not None: + integration = _resolve_telephony_integration( + db, + organization_id, + payload.telephony_integration_id, + payload.provider or "", + ) + _validate_telephony_credentials_live(db, organization_id, integration) + else: + _validate_direct_url_import_ready( + parameters, dict(call_import.parameter_mapping or {}) + ) + integration = None + + _ensure_blob_storage_enabled() + + if integration is not None: + call_import.provider = integration.provider + call_import.telephony_integration_id = integration.id + else: + call_import.provider = None + call_import.telephony_integration_id = None + + call_import.total_rows = 0 + call_import.completed_rows = 0 + call_import.failed_rows = 0 + call_import.error_message = None + call_import.status = CallImportStatus.PROCESSING + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + starting_from_mapped = True + + if use_diarised: + total_row_count = count_completed_source_rows(db, call_import.id) + else: + # Production runs score CSV text — rows need not wait for + # recording fetch to finish before they are evaluable. + total_row_count = count_source_rows_with_production_transcript( + db, call_import.id + ) + + requested_sources: List[str] = list(payload.transcript_sources) + + if ( + not use_diarised + and not starting_from_mapped + and count_all_source_rows(db, call_import.id) > 0 + and total_row_count == 0 + ): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "No rows have a production transcript. " + "Choose 'Diarize then evaluate' or import rows with " + "a transcript column." + ), + ) + + base_name = _normalize_name(payload.name) + + def _name_for_source(source: str) -> Optional[str]: + # Single-source runs preserve the user's chosen name verbatim. + del source + return base_name + + created_evaluations: List[CallImportEvaluation] = [] + + for source in requested_sources: + evaluation = CallImportEvaluation( + call_import_id=call_import.id, + organization_id=organization_id, + # Mirror the parent CallImport's workspace so listings can + # filter on workspace_id directly without joining. + workspace_id=call_import.workspace_id, + name=_name_for_source(source), + selected_metric_ids=[ + str(metric_id) for metric_id in leaf_metric_ids + ], + selected_metric_groups=selected_metric_groups or None, + status="pending", + total_rows=total_row_count, + completed_rows=0, + failed_rows=0, + llm_provider=llm_provider_norm, + llm_model=llm_model_norm, + llm_credential_id=payload.llm_credential_id, + llm_config=payload.llm_config, + metric_llm_overrides=metric_overrides_payload, + stt_provider=stt_provider_norm, + stt_model=stt_model_norm, + stt_credential_id=( + payload.stt_credential_id if auto_transcribe else None + ), + diarisation_llm_provider=diarisation_llm_provider_norm, + diarisation_llm_model=diarisation_llm_model_norm, + diarisation_llm_credential_id=( + payload.diarization_llm_credential_id if auto_transcribe else None + ), + diarisation_prompt=diarisation_prompt_norm, + transcribe_mode=transcribe_mode_norm, + transcript_source=source, + discover_new_metrics=bool( + getattr(payload, "discover_new_metrics", False) + ), + ) + stamp_evaluation_actor(evaluation, principal, creating=True) + db.add(evaluation) + db.flush() + created_evaluations.append(evaluation) + + db.commit() + for evaluation in created_evaluations: + db.refresh(evaluation) + + primary_evaluation = created_evaluations[0] + sibling_ids = [e.id for e in created_evaluations[1:]] + + if not total_row_count and not starting_from_mapped: + for evaluation in created_evaluations: + evaluation.status = "completed" + db.commit() + for evaluation in created_evaluations: + db.refresh(evaluation) + return _serialize_eval( + db, primary_evaluation, sibling_evaluation_ids=sibling_ids + ) + + if starting_from_mapped: + from app.workers.tasks.call_import_bulk_ops import ( + materialize_mapped_call_import_evaluation_task, + ) + + for evaluation in created_evaluations: + materialize_mapped_call_import_evaluation_task.delay( + str(call_import.id), + str(organization_id), + str(call_import.workspace_id), + str(evaluation.id), + transcribe_overwrite=payload.transcribe_overwrite, + ) + else: + from app.workers.tasks.call_import_bulk_ops import ( + materialize_call_import_evaluation_task, + ) + + for evaluation in created_evaluations: + materialize_call_import_evaluation_task.delay( + str(evaluation.id), + transcribe_overwrite=payload.transcribe_overwrite, + ) + + for evaluation in created_evaluations: + db.refresh(evaluation) + + return _serialize_eval( + db, primary_evaluation, sibling_evaluation_ids=sibling_ids + ) + + +@router.get( + "", + response_model=CallImportEvaluationListResponse, + operation_id="listCallImportEvaluations", +) +async def list_call_import_evaluations( + call_import_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationListResponse: + del api_key + _require_import(db, call_import_id, organization_id) + rows = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .order_by(desc(CallImportEvaluation.created_at)) + .all() + ) + email_map = emails_for_user_ids(db, user_ids_from_evaluations(rows)) + return CallImportEvaluationListResponse( + items=[_serialize_eval(db, row, user_emails=email_map) for row in rows], + total=len(rows), + ) + + +@router.get( + "/{eval_id}", + response_model=CallImportEvaluationResponse, + operation_id="getCallImportEvaluation", +) +async def get_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationResponse: + del api_key + _require_import(db, call_import_id, organization_id) + row = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + return _serialize_eval(db, row) + + +@router.get( + "/{eval_id}/rows", + response_model=CallImportEvaluationRowListResponse, + operation_id="listCallImportEvaluationRows", +) +async def list_call_import_evaluation_rows( + call_import_id: UUID, + eval_id: UUID, + page: int = Query(1, ge=1), + page_size: int = Query(100, ge=1, le=500), + q: Optional[str] = Query( + None, + description=( + "Free-text search across conversation_id and transcript " + "(case-insensitive substring match)." + ), + ), + metric_id: Optional[UUID] = Query( + None, + description=( + "If set, only return rows whose ``metric_scores[metric_id].value`` " + "exactly matches ``metric_value`` (string-compared). " + "Use together with ``metric_value``." + ), + ), + metric_value: Optional[str] = Query( + None, + description="Value to match against metric_id (string compare).", + ), + status_filter: Optional[str] = Query( + None, + alias="status", + description="Restrict to rows with this evaluation row status.", + ), + flow_parent_id: Optional[UUID] = Query( + None, + description=( + "Parent (category) metric whose ``sequence`` array should be " + "checked against ``flow_node`` and ``flow_edge_target``. Used " + "to drill into the calls behind a flow-chart node or edge." + ), + ), + flow_node: Optional[str] = Query( + None, + description=( + "If set together with ``flow_parent_id``, only return rows " + "whose sequence under that parent contains this step. Accepts " + "either a child metric UUID (resolved to slug(name)), a " + "``disc:`` discovered-label id, or a raw slug." + ), + ), + flow_edge_target: Optional[str] = Query( + None, + description=( + "Optional companion to ``flow_node``: when set, restrict to " + "rows whose sequence contains the directed transition " + "``flow_node -> flow_edge_target`` (immediately adjacent). " + "Same id format as ``flow_node``." + ), + ), + discovered_parent_id: Optional[UUID] = Query( + None, + description=( + "Parent (category) metric that defines the discovery scope " + "for ``discovered_label_key`` / ``has_discovered``." + ), + ), + discovered_label_key: Optional[str] = Query( + None, + description=( + "If set together with ``discovered_parent_id``, only return " + "rows whose ``metric_scores[parent].discovered_labels`` " + "list contains an entry with this slug (after applying " + "evaluation-level merge aliases)." + ), + ), + has_discovered: Optional[bool] = Query( + None, + description=( + "If true together with ``discovered_parent_id``, only return " + "rows that have at least one LLM-discovered label for the " + "parent. Useful to triage which calls produced novel labels." + ), + ), + sort_by: Optional[str] = Query( + None, + description=( + "Column to sort by. Accepted values: ``row_index`` (default " + "when omitted), ``conversation_id``, ``status`` (the " + "evaluation-row status), or ``metric:`` to sort " + "by ``metric_scores[].value``. Metric sorts compare " + "the extracted JSON text — adequate for booleans, enum " + "labels, and 0-1 ratings; large integer values may sort " + "lexicographically (10 before 2)." + ), + ), + sort_dir: Optional[str] = Query( + "asc", + description="Sort direction: ``asc`` (default) or ``desc``.", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationRowListResponse: + del api_key + _require_import(db, call_import_id, organization_id) + + eval_row = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not eval_row: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + query = ( + db.query(CallImportEvaluationRow, CallImportRow) + .join(CallImportRow, CallImportRow.id == CallImportEvaluationRow.call_import_row_id) + .filter(CallImportEvaluationRow.evaluation_id == eval_id) + ) + + # --- Filters ---------------------------------------------------------- + if q and q.strip(): + needle = f"%{q.strip()}%" + # Search across both transcript columns so a hit in either the + # production or the diarised version surfaces the row, + # independent of which source the evaluation actually scored. + query = query.filter( + or_( + CallImportRow.conversation_id.ilike(needle), + CallImportRow.transcript.ilike(needle), + CallImportRow.diarised_transcript.ilike(needle), + ) + ) + + if status_filter: + # The CallImportEvaluationRow.status column is a string in PG so a + # plain == filter works; we lowercase to match the stored values. + query = query.filter( + CallImportEvaluationRow.status == status_filter.strip().lower() + ) + + if metric_id is not None and metric_value is not None: + # ``metric_scores`` is a JSONB column shaped like + # ``{"": {"value": , "type": "boolean", ...}}``. We + # extract the nested ``value`` as text and compare to the user + # input as a string — that handles bool/int/enum without needing + # per-type casts. ``metric_value`` is matched case-insensitively + # so chart clicks on labels like "True" survive any casing drift + # between worker output and the chart label. + path_value = func.json_extract_path_text( + CallImportEvaluationRow.metric_scores, + str(metric_id), + "value", + ) + query = query.filter(func.lower(path_value) == metric_value.strip().lower()) + + # --- Flow chart drilldown filter ------------------------------------- + # Translates a clicked node (or edge) on the flow chart into a + # SQL filter against ``metric_scores[].sequence``. The + # frontend sends either a child UUID, a ``disc:`` discovered + # node id, or a raw slug — we normalize all three to the slug that + # actually appears in stored ``sequence`` arrays. + if flow_parent_id is not None and flow_node and flow_node.strip(): + parent_id_str_local = str(flow_parent_id) + alias_map_flow = _alias_map_for_parent(eval_row, flow_parent_id) + + def _flow_node_to_slug(raw: str) -> Optional[str]: + raw_clean = raw.strip() + if not raw_clean: + return None + if raw_clean == _FLOW_START_NODE_ID: + # The synthetic START node isn't a real sequence entry; + # filtering on it is meaningless so we skip silently. + return None + if raw_clean.startswith(_DISCOVERED_NODE_PREFIX): + return _resolve_alias( + alias_map_flow, + _slug_label(raw_clean[len(_DISCOVERED_NODE_PREFIX) :]), + ) + # Try to interpret as a child metric UUID first; fall back + # to treating it as a slug. + try: + child_uuid = UUID(raw_clean) + except (TypeError, ValueError): + return _resolve_alias(alias_map_flow, _slug_label(raw_clean)) + child = ( + db.query(Metric.name) + .filter( + Metric.id == child_uuid, + Metric.organization_id == organization_id, + ) + .first() + ) + if child and child[0]: + return _resolve_alias(alias_map_flow, _slug_label(child[0])) + return _resolve_alias(alias_map_flow, _slug_label(raw_clean)) + + from_slug = _flow_node_to_slug(flow_node) + target_slug: Optional[str] = None + if flow_edge_target and flow_edge_target.strip(): + target_slug = _flow_node_to_slug(flow_edge_target) + + if from_slug: + # The ``metric_scores`` column is declared as ``Column(JSON)`` + # in the model so on databases where the table was created + # from the model (rather than the migration) the physical + # type is ``json``, not ``jsonb``. The JSONB-only operators + # below (``jsonb_exists``, ``jsonb_array_elements_text``, + # ``@>``) require a JSONB input — we cast once up front so + # the same SQL works regardless of which path created the + # table. + scores_jsonb = ( + "(call_import_evaluation_rows.metric_scores)::jsonb" + ) + if target_slug: + # Edge filter: rows whose sequence under this parent + # contains ``from_slug`` immediately followed by + # ``target_slug``. Implemented as a correlated EXISTS + # over ``jsonb_array_elements_text`` with ORDINALITY, + # which is the portable way to express "next array + # index" against a JSONB array in Postgres. + edge_filter_sql = text( + f""" + EXISTS ( + SELECT 1 + FROM jsonb_array_elements_text( + COALESCE( + {scores_jsonb} -> :p_id -> 'sequence', + '[]'::jsonb + ) + ) WITH ORDINALITY AS s1(elem, ord) + JOIN jsonb_array_elements_text( + COALESCE( + {scores_jsonb} -> :p_id -> 'sequence', + '[]'::jsonb + ) + ) WITH ORDINALITY AS s2(elem, ord) + ON s2.ord = s1.ord + 1 + WHERE s1.elem = :from_slug + AND s2.elem = :to_slug + ) + """ + ).bindparams( + p_id=parent_id_str_local, + from_slug=from_slug, + to_slug=target_slug, + ) + query = query.filter(edge_filter_sql) + else: + # Node filter: rows whose ``metric_scores -> parent -> + # 'sequence'`` array contains ``from_slug``. We use the + # function form ``jsonb_exists`` rather than the ``?`` + # operator to avoid psycopg2 mistaking the question + # mark for a parameter placeholder. + node_filter_sql = text( + f""" + jsonb_exists( + COALESCE( + {scores_jsonb} -> :p_id -> 'sequence', + '[]'::jsonb + ), + :slug + ) + """ + ).bindparams(p_id=parent_id_str_local, slug=from_slug) + query = query.filter(node_filter_sql) + + # --- Discovered label filters --------------------------------------- + # Surfaces "which calls produced THIS LLM-discovered label" and the + # broader "which calls produced ANY LLM-discovered label". Both + # operate on ``metric_scores[].discovered_labels`` (a list + # of dicts) plus the same ``sequence`` array — covering both legacy + # rows where the slug only made it into ``sequence`` and newer + # rows where it landed in both. + if discovered_parent_id is not None and ( + discovered_label_key or has_discovered + ): + d_parent_str = str(discovered_parent_id) + alias_map_disc = _alias_map_for_parent(eval_row, discovered_parent_id) + # See note above: cast once so the JSONB operators don't reject + # the column when it's typed as ``json`` in the database. + scores_jsonb = "(call_import_evaluation_rows.metric_scores)::jsonb" + if discovered_label_key and discovered_label_key.strip(): + target = _resolve_alias( + alias_map_disc, _slug_label(discovered_label_key) + ) + if target: + # Match rows whose discovered_labels list has an entry + # ``{"key": }`` OR whose sequence array still + # contains the slug. The latter covers older rows that + # were rewritten by a merge in the discovered_labels + # blob but whose sequence may have lagged. + contains_json = json.dumps( + {d_parent_str: {"discovered_labels": [{"key": target}]}} + ) + disc_filter_sql = text( + f""" + ( + {scores_jsonb} @> CAST(:contains AS JSONB) + OR + jsonb_exists( + COALESCE( + {scores_jsonb} -> :p_id -> 'sequence', + '[]'::jsonb + ), + :slug + ) + ) + """ + ).bindparams( + contains=contains_json, + p_id=d_parent_str, + slug=target, + ) + query = query.filter(disc_filter_sql) + elif has_discovered: + # No specific slug — just rows that surfaced any candidate + # under this parent. We coalesce missing paths to ``[]`` so + # ``jsonb_array_length`` always sees an array (it raises on + # non-array inputs, but our shape guarantees a list when + # the key is present). + has_disc_sql = text( + f""" + jsonb_array_length( + COALESCE( + {scores_jsonb} -> :p_id -> 'discovered_labels', + '[]'::jsonb + ) + ) > 0 + """ + ).bindparams(p_id=d_parent_str) + query = query.filter(has_disc_sql) + + # --- Sorting ---------------------------------------------------------- + # Column-click sorting from the UI. Falls back to ``row_index`` so + # paging stays stable when the user clears the sort. We always add a + # secondary ``row_index`` tiebreaker so duplicate sort keys (e.g. + # many rows with ``status = 'completed'``) keep a deterministic + # order across page boundaries — without this, pagination can + # double-show or skip rows when Postgres picks a different physical + # order on each query. + direction_desc = (sort_dir or "asc").strip().lower() == "desc" + + def _apply_direction(column_expr): + return column_expr.desc() if direction_desc else column_expr.asc() + + # Whether the caller's ``sort_by`` resolved to a known column. We + # use this flag to decide whether ``sort_dir`` is honoured on the + # fallback path: unrecognized columns (typos, stale UI state) fall + # back to the implicit ``row_index ASC`` default and intentionally + # ignore ``sort_dir`` so users don't get a surprise reverse order + # from a typo'd column name. + sort_recognized = False + sort_by_clean = (sort_by or "").strip() + primary_sort = None + metric_uuid: Optional[UUID] = None + if sort_by_clean == "row_index": + sort_recognized = True + # Falls through to the default ``order_by`` below with + # ``primary_sort`` still None — but ``sort_recognized=True`` + # tells the fallback branch to apply the requested direction. + elif sort_by_clean == "conversation_id": + sort_recognized = True + primary_sort = _apply_direction(CallImportRow.conversation_id) + elif sort_by_clean == "status": + sort_recognized = True + primary_sort = _apply_direction(CallImportEvaluationRow.status) + elif sort_by_clean.startswith("metric:"): + raw_metric_id = sort_by_clean.split(":", 1)[1].strip() + try: + metric_uuid = UUID(raw_metric_id) + except (TypeError, ValueError): + metric_uuid = None + if metric_uuid is not None: + sort_recognized = True + # ``metric_scores`` is JSON-typed but the helper functions + # for path extraction differ between Postgres (production) + # and SQLite (default test backend). Branch on the active + # dialect so we can use the right primitive: + # * Postgres → ``json_extract_path_text(col, key, "value")`` + # which returns the value as TEXT for both ``json`` and + # ``jsonb`` columns. + # * SQLite → ``json_extract(col, '$."".value')`` + # using JSONPath syntax. ``metric_uuid`` is already + # validated above (``UUID(raw_metric_id)``), so the + # interpolated path is safe from injection. + # NULL values (rows where the metric wasn't scored) sort + # to the END regardless of direction so un-scored rows + # don't crowd the top of an ascending sort. + dialect_name = ( + db.bind.dialect.name if db.bind is not None else "postgresql" + ) + if dialect_name == "sqlite": + json_path = f'$."{metric_uuid}".value' + path_value = func.json_extract( + CallImportEvaluationRow.metric_scores, + json_path, + ) + else: + path_value = func.json_extract_path_text( + CallImportEvaluationRow.metric_scores, + str(metric_uuid), + "value", + ) + primary_sort = ( + path_value.desc().nullslast() + if direction_desc + else path_value.asc().nullslast() + ) + + if primary_sort is not None: + query = query.order_by(primary_sort, CallImportRow.row_index.asc()) + elif sort_recognized: + # Explicit ``sort_by=row_index`` request — honour direction. + query = query.order_by(_apply_direction(CallImportRow.row_index)) + else: + # No sort requested OR unrecognized column — safe default of + # ``row_index ASC``. We deliberately ignore ``sort_dir`` here + # so a typo'd / stale ``sort_by`` doesn't quietly invert the + # default order. + query = query.order_by(CallImportRow.row_index.asc()) + from app.db_sharding.eval_rows import fetch_evaluation_row_pairs_page + from app.db_sharding.sessions import is_sharding_enabled + + def _pair_row_index( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> int: + return int(pair[1].row_index or 0) + + def _directed_string(value: Optional[str], desc: bool) -> Tuple[int, ...]: + text = value or "" + if not desc: + return (0, *text.encode("utf-8")) + return (1, *(-byte for byte in text.encode("utf-8"))) + + if sort_by_clean == "conversation_id": + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[Any, ...]: + return ( + _directed_string(pair[1].conversation_id, direction_desc), + _pair_row_index(pair), + ) + elif sort_by_clean == "status": + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[Any, ...]: + return ( + _directed_string(pair[0].status, direction_desc), + _pair_row_index(pair), + ) + elif sort_by_clean.startswith("metric:") and metric_uuid is not None: + metric_id_str = str(metric_uuid) + + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[Any, ...]: + scores = pair[0].metric_scores or {} + entry = scores.get(metric_id_str, {}) + raw_value = entry.get("value") if isinstance(entry, dict) else None + null_rank = 1 if raw_value is None else 0 + return ( + null_rank, + _directed_string( + str(raw_value) if raw_value is not None else None, + direction_desc, + ), + _pair_row_index(pair), + ) + elif sort_recognized and sort_by_clean == "row_index": + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[int, ...]: + idx = _pair_row_index(pair) + return (-idx,) if direction_desc else (idx,) + else: + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[int, ...]: + return (_pair_row_index(pair),) + + if is_sharding_enabled(): + def _build_query(session: Session): + return query.with_session(session) + + total, rows = fetch_evaluation_row_pairs_page( + db, + _build_query, + page=page, + page_size=page_size, + sort_key=_pair_sort_key, + bounded_shard_fetch=( + not sort_recognized or sort_by_clean == "row_index" + ), + ) + else: + total = query.count() + rows = query.offset((page - 1) * page_size).limit(page_size).all() + + # Row detail shows the transcript for this run's chosen source. + items: List[CallImportEvaluationRowResponse] = [ + _to_evaluation_row_response(eval_row_obj, source_row, eval_row) + for eval_row_obj, source_row in rows + ] + + return CallImportEvaluationRowListResponse( + items=items, + total=total, + page=page, + page_size=page_size, + ) + + +@router.get( + "/{eval_id}/export", + operation_id="exportCallImportEvaluationCsv", +) +async def export_call_import_evaluation_csv( + call_import_id: UUID, + eval_id: UUID, + format: Literal["csv", "xlsx"] = Query( + "csv", + description=( + "Output format. ``csv`` returns a UTF-8 BOM CSV; ``xlsx`` " + "returns a native Excel workbook (single sheet)." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> StreamingResponse: + del api_key + call_import = _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + selected_metric_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) + # Include parent metric ids referenced in selected_metric_groups so + # the export shows a parent "Chosen Label" column next to its + # children's true/false columns. + lookup_ids: List[UUID] = list(selected_metric_ids) + groups_raw = ( + evaluation.selected_metric_groups + if isinstance(evaluation.selected_metric_groups, dict) + else {} + ) + for parent_str in groups_raw.keys(): + try: + pid = UUID(parent_str) + if pid not in lookup_ids: + lookup_ids.append(pid) + except (TypeError, ValueError): + continue + metrics = _metrics_for_ids(db, organization_id, lookup_ids) + metric_names = {str(metric.id): metric.name for metric in metrics} + metrics_by_id = {str(metric.id): metric for metric in metrics} + + # Two export-time modes depending on how the batch was uploaded: + # + # * Schema-driven (new): ``call_imports.schema_id`` is set, + # ``parameter_mapping`` records which CSV header fed each + # parameter, and ``raw_columns`` on each row is keyed by + # parameter NAME. Export headers are the parameter names. + # * Legacy (pre-schema): ``column_mapping`` / ``extra_columns`` / + # ``custom_column_mapping`` drive the columns and + # ``raw_columns`` is keyed by the original CSV header. + # + # We bucket entries into ``standard_export_headers`` (raw_columns + # key == export header) and ``custom_export`` (export header + # differs from the raw_columns key) so the row-projection loop + # below stays mode-agnostic. + standard_export_headers: List[str] = [] + custom_export: List[tuple[str, str]] = [] # [(export_header, raw_columns_key)] + + if call_import.schema_id is not None: + # Use the live schema parameter list for column ordering. Falls + # back to whatever's in ``parameter_mapping`` if the schema was + # deleted (defensive - the FK is ON DELETE RESTRICT, but tests + # / future cascades may still hit this branch). + from app.models.database import CallImportSchema as _ImportSchema + + schema_obj = ( + db.query(_ImportSchema) + .filter(_ImportSchema.id == call_import.schema_id) + .first() + ) + if schema_obj is not None: + params_sorted = sorted( + schema_obj.parameters, key=lambda p: p.ordering or 0 + ) + for param in params_sorted: + if param.name and param.name not in standard_export_headers: + standard_export_headers.append(param.name) + else: + for param_name in (call_import.parameter_mapping or {}).keys(): + if param_name and param_name not in standard_export_headers: + standard_export_headers.append(param_name) + else: + mapping = call_import.column_mapping or {} + mapped_headers = [ + mapping.get("external_call_id"), + mapping.get("transcript"), + mapping.get("recording_url"), + ] + for header in [*mapped_headers, *(call_import.extra_columns or [])]: + if ( + isinstance(header, str) + and header + and header not in standard_export_headers + ): + standard_export_headers.append(header) + + custom_mapping = call_import.custom_column_mapping or {} + if isinstance(custom_mapping, dict): + for name, csv_header in custom_mapping.items(): + if not isinstance(name, str) or not isinstance(csv_header, str): + continue + if not name or not csv_header: + continue + if name in standard_export_headers: + continue # would clobber a real column + custom_export.append((name, csv_header)) + + if ( + call_import.source_format == "audio" + and "conversation_id" not in standard_export_headers + ): + standard_export_headers.insert(0, "conversation_id") + + # Build the metric columns: each parent (if any) gets a value column + # and (when capture_rationale=true) a " - LLM Rationale" + # column. The per-child boolean columns are intentionally suppressed + # — categorization metrics now collapse to exactly two columns in + # the export, mirroring the in-app table. + child_ids_in_groups: set[str] = set() + for parent_str, child_strs in groups_raw.items(): + for child_str in child_strs: + if isinstance(child_str, str): + child_ids_in_groups.add(child_str) + + metric_headers: List[str] = [] + rationale_headers: Dict[str, str] = {} # metric_id_str -> rationale column name + seen_metric_ids: set[str] = set() + + def _add_metric_column(metric: Metric) -> None: + mid_str = str(metric.id) + if mid_str in seen_metric_ids: + return + # Skip any child whose parent is part of this run — the parent + # column above already shows the chosen child name as its + # value. + if mid_str in child_ids_in_groups: + return + seen_metric_ids.add(mid_str) + header = metric_names[mid_str] + metric_headers.append(header) + if bool(getattr(metric, "capture_rationale", False)): + rationale_header = f"{header} - LLM Rationale" + metric_headers.append(rationale_header) + rationale_headers[mid_str] = rationale_header + + for parent_str in groups_raw.keys(): + parent = metrics_by_id.get(parent_str) + if parent: + _add_metric_column(parent) + # Children of an in-run parent are deliberately not emitted — + # the ``child_ids_in_groups`` guard inside ``_add_metric_column`` + # is what enforces this. We still iterate the keys above (not + # ``.items()``) so the parent-only emission is explicit. + # Append anything left over (standalone metrics not in any group, or + # legacy runs without ``selected_metric_groups``). + for metric in metrics: + if metric.selection_mode and not metric.parent_metric_id: + continue # already handled above + if str(metric.id) in seen_metric_ids: + continue + _add_metric_column(metric) + + # Three new fixed columns surface the two transcript fields and the + # evaluation's transcript_source as live values pulled from the + # ``CallImportRow`` (not from the frozen ``raw_columns`` snapshot). + # The user can now compare "what was in the CSV" vs "what the + # diarisation worker produced" without round-tripping through the + # UI, and downstream tools can verify which transcript the metrics + # were computed against. + PRODUCTION_TRANSCRIPT_HEADER = "Production Transcript" + DIARISED_TRANSCRIPT_HEADER = "Diarised Transcript" + EVAL_SOURCE_HEADER = "Evaluated Transcript Source" + + fieldnames = [ + *standard_export_headers, + *[h for h, _ in custom_export], + PRODUCTION_TRANSCRIPT_HEADER, + DIARISED_TRANSCRIPT_HEADER, + EVAL_SOURCE_HEADER, + *metric_headers, + ] + + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + rows = sorted( + load_evaluation_row_pairs(db, eval_id), + key=lambda pair: int(pair[1].row_index or 0), + ) + else: + rows = ( + db.query(CallImportEvaluationRow, CallImportRow) + .join( + CallImportRow, + CallImportRow.id == CallImportEvaluationRow.call_import_row_id, + ) + .filter(CallImportEvaluationRow.evaluation_id == eval_id) + .order_by(CallImportRow.row_index.asc()) + .all() + ) + + def _project_rows() -> Iterator[Dict[str, str]]: + for eval_row, source_row in rows: + row_out: Dict[str, str] = {} + raw = ( + source_row.raw_columns + if isinstance(source_row.raw_columns, dict) + else {} + ) + for header in standard_export_headers: + value = raw.get(header) + if value is None and header == "conversation_id": + value = source_row.conversation_id + row_out[header] = "" if value is None else str(value) + for export_header, csv_header in custom_export: + value = raw.get(csv_header) + row_out[export_header] = "" if value is None else str(value) + + # Live transcripts pulled from the row, NOT from raw_columns, + # so re-diarised values are always reflected in the export. + # Both transcript columns are flattened to a single line so the + # spreadsheet cell doesn't balloon vertically — the in-app + # ``TranscriptView`` still has the DB copy with line breaks + # intact for chat-bubble rendering. + row_out[PRODUCTION_TRANSCRIPT_HEADER] = _flatten_transcript( + source_row.transcript + ) + row_out[DIARISED_TRANSCRIPT_HEADER] = _flatten_transcript( + source_row.diarised_transcript + ) + row_out[EVAL_SOURCE_HEADER] = _evaluated_transcript_source_label( + evaluation, + source_row, + ) + + scores = ( + eval_row.metric_scores + if isinstance(eval_row.metric_scores, dict) + else {} + ) + for metric in metrics: + metric_score = ( + scores.get(str(metric.id)) + if isinstance(scores, dict) + else None + ) + value = ( + metric_score.get("value") + if isinstance(metric_score, dict) + else None + ) + # Parent metrics (selection_mode set) render the chosen + # child name for single_choice or the ";"-joined list of + # true child names for multi_label. + if ( + metric.selection_mode + and not metric.parent_metric_id + and isinstance(metric_score, dict) + ): + if metric.selection_mode == "multi_label": + selected = metric_score.get("selected_child_names") + if isinstance(selected, list): + value = ";".join(str(s) for s in selected) + else: + value = ( + metric_score.get("chosen_child_name") + or metric_score.get("value") + ) + row_out[metric.name] = "" if value is None else str(value) + rationale_header = rationale_headers.get(str(metric.id)) + if rationale_header is not None: + rationale = ( + metric_score.get("rationale") + if isinstance(metric_score, dict) + else None + ) + row_out[rationale_header] = ( + "" if rationale is None else str(rationale) + ) + yield row_out + + base_filename = f"call-import-{call_import_id}-evaluation-{eval_id}" + + if format == "xlsx": + # xlsx is unicode-native (Hindi/Devanagari, emoji, etc.) so the + # UTF-8-BOM dance isn't needed here. ``write_only`` mode keeps + # peak memory bounded for large evaluations because openpyxl + # only buffers the current row. + try: + from openpyxl import Workbook # type: ignore + from openpyxl.cell import WriteOnlyCell # type: ignore + from openpyxl.styles import Font # type: ignore + except ImportError as exc: # pragma: no cover - exercised by pyproject lock + raise HTTPException( + status_code=500, + detail=( + "Excel export requires the 'openpyxl' package which is " + "not installed." + ), + ) from exc + + workbook = Workbook(write_only=True) + worksheet = workbook.create_sheet(title="Evaluation") + + bold_font = Font(bold=True) + header_cells = [] + for header in fieldnames: + cell = WriteOnlyCell(worksheet, value=header) + cell.font = bold_font + header_cells.append(cell) + worksheet.append(header_cells) + + for row_dict in _project_rows(): + worksheet.append([row_dict.get(h, "") for h in fieldnames]) + + buffer = io.BytesIO() + workbook.save(buffer) + xlsx_bytes = buffer.getvalue() + filename = f"{base_filename}.xlsx" + return StreamingResponse( + iter([xlsx_bytes]), + media_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + output = io.StringIO() + writer = csv.DictWriter(output, fieldnames=fieldnames, extrasaction="ignore") + writer.writeheader() + for row_dict in _project_rows(): + writer.writerow(row_dict) + + # Excel on Windows defaults to the system ANSI codepage (Windows-1252) + # when a CSV has no encoding marker, which turns UTF-8 Hindi/Devanagari + # / any non-ASCII text into mojibake (e.g. ``ठीक`` → ``ठीक``). + # A UTF-8 BOM tells Excel to switch to UTF-8 decoding and is silently + # skipped by every other UTF-8-aware reader (pandas, LibreOffice, + # Google Sheets, etc.), so the data round-trips correctly everywhere. + csv_text = output.getvalue() + # ``utf-8-sig`` adds the UTF-8 BOM so Excel on Windows decodes the file + # as UTF-8 instead of the system codepage. We also declare the same + # codec in the Content-Type header so well-behaved HTTP clients (incl. + # ``httpx`` / ``requests`` in our tests) strip the BOM during decode. + csv_bytes = csv_text.encode("utf-8-sig") + filename = f"{base_filename}.csv" + return StreamingResponse( + iter([csv_bytes]), + media_type="text/csv; charset=utf-8-sig", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + +def _report_filename_slug(value: str) -> str: + slug = re.sub(r"[^a-zA-Z0-9]+", "-", value.strip().lower()).strip("-") + return slug or "client" + + +def _pdf_report_actor(principal: Principal) -> tuple[Optional[str], Optional[UUID]]: + created_by = principal.email + if not created_by and principal.user_id: + created_by = str(principal.user_id) + return created_by, principal.user_id + + +def _pdf_report_response_from_row( + row: CallImportEvaluationPdfReport, + *, + cache_hit: bool = False, +) -> CallImportEvaluationPdfReportResponse: + filename = row.filename or "report.pdf" + preview_url, download_url = presigned_urls_for_pdf_report( + row.s3_key or "", + filename, + ) + return CallImportEvaluationPdfReportResponse( + id=str(row.id), + filename=filename, + preview_url=preview_url, + download_url=download_url, + created_at=row.created_at or datetime.now(timezone.utc), + created_by=row.created_by, + report_type=row.report_type, + vendor_name=row.vendor_name, + config_summary=config_summary_from_report_config( + row.report_config if isinstance(row.report_config, dict) else {} + ), + storage_available=bool(row.s3_key), + cache_hit=cache_hit, + ) + + +def _pdf_report_list_item_from_row( + row: CallImportEvaluationPdfReport, +) -> CallImportEvaluationPdfReportListItem: + return CallImportEvaluationPdfReportListItem( + id=str(row.id), + filename=row.filename, + vendor_name=row.vendor_name, + report_type=row.report_type, + created_by=row.created_by, + created_at=row.created_at or datetime.now(timezone.utc), + config_summary=config_summary_from_report_config( + row.report_config if isinstance(row.report_config, dict) else {} + ), + cache_fingerprint=row.cache_fingerprint, + ) + + +def _report_branding_for_import_workspace( + db: Session, + organization_id: UUID, + workspace_id: UUID, + *, + internal_brand_image_id: Optional[str] = None, + external_brand_image_id: Optional[str] = None, +) -> tuple[dict[str, str] | list[str], Optional[str]]: + workspace = ( + db.query(Workspace) + .filter( + Workspace.id == workspace_id, + Workspace.organization_id == organization_id, + ) + .first() + ) + raw = workspace.report_branding if workspace and isinstance(workspace.report_branding, dict) else {} + images = raw.get("images") if isinstance(raw.get("images"), list) else [] + loaded_images: list[dict[str, str]] = [] + for item in images: + if not isinstance(item, dict) or not item.get("s3_key"): + continue + content_type = str(item.get("content_type") or "image/png") + try: + from app.services.storage.s3_service import s3_service + + image_bytes = s3_service.download_file_by_key(str(item["s3_key"])) + except Exception as exc: # noqa: BLE001 + logger.warning( + "Unable to load report branding image for workspace {}: {}", + workspace_id, + exc, + ) + continue + encoded = base64.b64encode(image_bytes).decode("ascii") + role = str(item.get("role") or "generic") + if role not in {"internal", "external", "generic"}: + role = "generic" + loaded_images.append( + { + "id": str(item.get("id") or ""), + "role": role, + "data_uri": f"data:{content_type};base64,{encoded}", + } + ) + + def _pick(role: str, selected_id: Optional[str]) -> Optional[str]: + if selected_id: + for loaded in loaded_images: + if loaded["id"] == selected_id: + return loaded["data_uri"] + for loaded in loaded_images: + if loaded["role"] == role: + return loaded["data_uri"] + return None + + logo_data_uris: dict[str, str] = {} + internal_uri = _pick("internal", internal_brand_image_id) + external_uri = _pick("external", external_brand_image_id) + if internal_uri: + logo_data_uris["internal"] = internal_uri + if external_uri: + logo_data_uris["external"] = external_uri + if ( + not logo_data_uris + and not internal_brand_image_id + and not external_brand_image_id + ): + # Backward compatibility for workspaces that only had a generic logo + # library before the two-slot report header existed. + generic_uris = [ + loaded["data_uri"] + for loaded in loaded_images + if loaded.get("data_uri") + ] + if generic_uris: + heading = raw.get("heading") if isinstance(raw.get("heading"), str) else None + return generic_uris[:4], heading + heading = raw.get("heading") if isinstance(raw.get("heading"), str) else None + return logo_data_uris, heading + + +def _display_metrics_for_pdf_report( + db: Session, + organization_id: UUID, + evaluation: CallImportEvaluation, +) -> list[Metric]: + selected_metric_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) + lookup_ids: List[UUID] = list(selected_metric_ids) + groups_raw = ( + evaluation.selected_metric_groups + if isinstance(evaluation.selected_metric_groups, dict) + else {} + ) + for parent_str in groups_raw.keys(): + try: + parent_id = UUID(parent_str) + except (TypeError, ValueError): + continue + if parent_id not in lookup_ids: + lookup_ids.append(parent_id) + + metrics = _metrics_for_ids(db, organization_id, lookup_ids) + child_ids_in_groups: set[str] = set() + for child_strs in groups_raw.values(): + if not isinstance(child_strs, list): + continue + child_ids_in_groups.update(str(child_id) for child_id in child_strs) + + metrics_by_id = {str(metric.id): metric for metric in metrics} + display: list[Metric] = [] + seen: set[str] = set() + + for parent_str in groups_raw.keys(): + parent = metrics_by_id.get(str(parent_str)) + if parent and str(parent.id) not in seen: + display.append(parent) + seen.add(str(parent.id)) + + for metric in metrics: + metric_id = str(metric.id) + if metric_id in seen or metric_id in child_ids_in_groups: + continue + if metric.selection_mode and not metric.parent_metric_id: + continue + display.append(metric) + seen.add(metric_id) + + return display + + +def _metrics_for_clustering( + db: Session, + evaluation: CallImportEvaluation, + eval_rows: List[CallImportEvaluationRow], +) -> List[Metric]: + """All enabled quality metrics scored in this run, normalized for clustering. + + Hierarchical children are collapsed to their parent metric so cluster + groups render at the category level (e.g. ``AI reveal``) instead of the + child label level (e.g. ``Yes`` / ``No``). + """ + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + aggregate_metric_ids: List[UUID] = [] + for agg in aggregates: + if (agg.metric_category or "quality") == "user_insight": + continue + try: + aggregate_metric_ids.append(UUID(agg.metric_id)) + except (TypeError, ValueError): + continue + if not aggregate_metric_ids: + return [] + + aggregate_metrics = _metrics_for_ids( + db, evaluation.organization_id, aggregate_metric_ids + ) + by_id = {metric.id: metric for metric in aggregate_metrics} + + normalized_ids: List[UUID] = [] + seen: set[UUID] = set() + for metric_id in aggregate_metric_ids: + metric = by_id.get(metric_id) + target_id = ( + metric.parent_metric_id + if metric is not None and metric.parent_metric_id + else metric_id + ) + if target_id in seen: + continue + seen.add(target_id) + normalized_ids.append(target_id) + + metrics = _metrics_for_ids(db, evaluation.organization_id, normalized_ids) + return [ + metric + for metric in metrics + if getattr(metric, "enabled", True) and not _metric_is_user_insight(metric) + ] + + +def _metric_is_user_insight(metric: Metric) -> bool: + if (getattr(metric, "metric_category", "quality") or "quality") == "user_insight": + return True + text_value = " ".join( + str(part or "").lower() + for part in (getattr(metric, "name", ""), getattr(metric, "description", "")) + ) + normalized = text_value.replace("-", " ").replace("_", " ") + phrases = ( + "call context", + "caller context", + "product identification", + "out of scope", + "identity match", + "user identity", + "caller identity", + "frustration trigger", + "video call offer", + "video call reception", + ) + return any(phrase in normalized for phrase in phrases) + + +def _evaluation_rows_for_period( + db: Session, + evaluation_id: UUID, +) -> list[tuple[CallImportEvaluationRow, CallImportRow]]: + return ( + db.query(CallImportEvaluationRow, CallImportRow) + .join(CallImportRow, CallImportRow.id == CallImportEvaluationRow.call_import_row_id) + .filter(CallImportEvaluationRow.evaluation_id == evaluation_id) + .order_by(CallImportRow.row_index.asc()) + .all() + ) + + +def _baseline_candidate_evaluations( + db: Session, + organization_id: UUID, + workspace_id: UUID, + current_evaluation: CallImportEvaluation, + current_period_start: Optional[date], + *, + limit: int = 20, +) -> list[dict[str, Any]]: + candidates = ( + db.query(CallImportEvaluation, CallImport) + .join(CallImport, CallImport.id == CallImportEvaluation.call_import_id) + .filter( + CallImportEvaluation.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + CallImportEvaluation.id != current_evaluation.id, + CallImportEvaluation.status == "completed", + CallImportEvaluation.completed_rows > 0, + ) + .order_by(desc(CallImportEvaluation.created_at)) + .limit(limit * 3) + .all() + ) + items: list[dict[str, Any]] = [] + for candidate_eval, candidate_import in candidates: + rows = _evaluation_rows_for_period(db, candidate_eval.id) + period_start, period_end, period_label, period_display = _report_period_from_rows(rows) + if current_period_start and period_start and period_start >= current_period_start: + continue + dataset = ( + (candidate_import.dataset or "").strip() + or (candidate_import.original_filename or candidate_import.filename or "").strip() + or "Unknown dataset" + ) + evaluation_name = ( + (candidate_eval.name or "").strip() + or str(candidate_eval.id)[:8] + ) + items.append( + { + "evaluation_id": str(candidate_eval.id), + "name": evaluation_name, + "dataset": dataset, + "period_label": period_label, + "period_start": period_start, + "period_end": period_end, + "period_display": period_display, + "completed_rows": int(candidate_eval.completed_rows or 0), + "created_at": candidate_eval.created_at, + "is_default": False, + } + ) + if len(items) >= limit: + break + items.sort( + key=lambda item: ( + item["period_start"] or date.min, + item["created_at"] or datetime.min.replace(tzinfo=timezone.utc), + ), + reverse=True, + ) + if items: + items[0]["is_default"] = True + return items + + +def _resolve_baseline_evaluation( + db: Session, + organization_id: UUID, + workspace_id: UUID, + current_evaluation: CallImportEvaluation, + current_period_start: Optional[date], + baseline_evaluation_id: Optional[str], +) -> Optional[CallImportEvaluation]: + candidates = _baseline_candidate_evaluations( + db, + organization_id, + workspace_id, + current_evaluation, + current_period_start, + ) + allowed_ids = {item["evaluation_id"] for item in candidates} + if baseline_evaluation_id: + baseline_id = str(baseline_evaluation_id).strip() + if baseline_id not in allowed_ids: + raise HTTPException( + status_code=400, + detail="Selected baseline evaluation is not a valid prior run for this report.", + ) + return ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == UUID(baseline_id), + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not candidates: + return None + default_id = candidates[0]["evaluation_id"] + return ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == UUID(default_id), + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + + +def _benchmark_context_for_evaluation( + db: Session, + baseline_evaluation: Optional[CallImportEvaluation], +) -> Optional[dict[str, str]]: + if baseline_evaluation is None: + return None + baseline_import = ( + db.query(CallImport) + .filter(CallImport.id == baseline_evaluation.call_import_id) + .first() + ) + rows = _evaluation_rows_for_period(db, baseline_evaluation.id) + period_start, _period_end, period_label, _period_display = _report_period_from_rows(rows) + dataset = ( + (baseline_import.dataset or "").strip() + if baseline_import and baseline_import.dataset + else None + ) + filename = ( + (baseline_import.original_filename or baseline_import.filename or "").strip() + if baseline_import + else None + ) + evaluation_label = ( + (baseline_evaluation.name or "").strip() + if baseline_evaluation.name + else str(baseline_evaluation.id)[:8] + ) + period = period_label or ( + period_start.isoformat() if period_start else "previous report" + ) + return { + "dataset": dataset or filename or "Unknown dataset", + "evaluation": evaluation_label, + "evaluation_id": str(baseline_evaluation.id), + "period": period, + } + + +def _period_deltas_from_evaluation( + db: Session, + baseline_evaluation: CallImportEvaluation, + current_metric_aggregates: list[dict[str, Any]], + current_evaluation: CallImportEvaluation, + current_eval_rows: List[CallImportEvaluationRow], +) -> dict[str, dict[str, str]]: + baseline_rows = _evaluation_rows_for_period(db, baseline_evaluation.id) + baseline_eval_rows = [eval_row for eval_row, _source_row in baseline_rows] + baseline_aggregate_models = _compute_metric_aggregates( + db, + baseline_evaluation, + baseline_eval_rows, + ) + baseline_metric_aggregates = [ + _aggregate_to_dict(aggregate) for aggregate in baseline_aggregate_models + ] + _metrics, _aggs, policies, _source, _child_map = _clustering_context( + db, current_evaluation, current_eval_rows + ) + metric_by_id = {str(m.id): m for m in _metrics} + current_by_id = { + str(item.get("metric_id")): item for item in current_metric_aggregates + } + previous_by_id = { + str(item.get("metric_id")): item + for item in baseline_metric_aggregates + if isinstance(item, dict) + } + deltas: dict[str, dict[str, str]] = {} + for metric_id, current in current_by_id.items(): + metric = metric_by_id.get(metric_id) + policy = policies.get(metric_id) + previous_raw = previous_by_id.get(metric_id) + if metric is None or policy is None: + deltas[metric_id] = { + "label": "No previous-week baseline", + "detail": "No comparable prior report snapshot was found.", + } + continue + current_pct = failure_rate_percent_from_rows( + current_eval_rows, metric, policy + ) + previous_pct = failure_rate_percent_from_rows( + baseline_eval_rows, metric, policy + ) + if current_pct is None or previous_pct is None: + current_pct = current_pct or _aggregate_primary_percent(current, policy) + previous_pct = ( + previous_pct or _aggregate_primary_percent(previous_raw, policy) + if previous_raw + else None + ) + if current_pct is None or previous_pct is None: + deltas[metric_id] = { + "label": "No previous-week baseline", + "detail": "No comparable prior report snapshot was found.", + } + continue + delta = current_pct - previous_pct + sign = "+" if delta >= 0 else "" + deltas[metric_id] = { + "label": f"{sign}{delta:.1f} pp", + "detail": ( + f"Current report {current_pct:.1f}% vs previous report " + f"{previous_pct:.1f}%" + ), + } + return deltas + + +_DELTA_EXPLANATION_SYSTEM_PROMPT = ( + "You are a senior conversation-analytics reviewer. You will receive " + "week-over-week metric failure-rate deltas plus reconciled failure " + "cluster context per metric.\n\n" + "Return STRICT JSON only:\n" + "{\n" + ' "explanations": {"": "<1-2 sentence explanation of why the delta likely occurred>"}\n' + "}\n\n" + "Constraints:\n" + "- Only include metrics supplied in the prompt.\n" + "- Cluster labels are generated independently each run and are NOT stable " + "IDs. Never compare an unmatched current label to 0% baseline.\n" + "- Use matched_theme_shifts for label-aligned comparisons, " + "gap_label_shifts for structural shifts, and new_themes_current_period " + "for themes that emerged without a baseline match.\n" + "- If reconciliation is uncertain, explain using the numeric delta and " + "gap_label_shifts only.\n" + "- Keep each explanation to 1-2 short sentences (~220 chars).\n" + "- Vendor-safe, factual language; no markdown." +) + + +def _period_delta_explanation_cache_key( + baseline_evaluation_id: UUID, + *, + completed_rows: int, + baseline_completed_rows: int, +) -> str: + return ( + f"{baseline_evaluation_id}:{completed_rows}:" + f"{baseline_completed_rows}:reconciled-v2" + ) + + +def _normalize_cluster_label(label: str) -> str: + return re.sub(r"[^a-z0-9]+", " ", (label or "").lower()).strip() + + +_CLUSTER_LABEL_STOPWORDS = frozenset( + { + "a", + "an", + "the", + "and", + "or", + "during", + "while", + "with", + "for", + "from", + "into", + "general", + "user", + "bot", + "agent", + } +) + + +def _cluster_label_tokens(label: str) -> set[str]: + return { + token + for token in _normalize_cluster_label(label).split() + if token and token not in _CLUSTER_LABEL_STOPWORDS and len(token) > 2 + } + + +def _cluster_label_similarity(left: str, right: str) -> float: + tokens_left = _cluster_label_tokens(left) + tokens_right = _cluster_label_tokens(right) + if not tokens_left or not tokens_right: + return 0.0 + intersection = tokens_left & tokens_right + if not intersection: + return 0.0 + union = tokens_left | tokens_right + jaccard = len(intersection) / len(union) + smaller = tokens_left if len(tokens_left) <= len(tokens_right) else tokens_right + overlap_ratio = len(intersection) / len(smaller) + return max(jaccard, overlap_ratio * 0.85) + + +def _group_clusters_by_gap_label( + clusters: list[dict[str, Any]], +) -> dict[str, list[dict[str, Any]]]: + grouped: dict[str, list[dict[str, Any]]] = {} + for cluster in clusters: + gap_label = str(cluster.get("gap_label") or "UNKNOWN") + grouped.setdefault(gap_label, []).append(cluster) + return grouped + + +def _append_matched_cluster_pair( + matched: list[dict[str, Any]], + current: dict[str, Any], + baseline: dict[str, Any], + *, + match_confidence: float, + match_method: str, +) -> None: + matched.append( + { + "current_label": current.get("label"), + "baseline_label": baseline.get("label"), + "gap_label": current.get("gap_label") or baseline.get("gap_label"), + "current_share_pct": current.get("share_pct"), + "baseline_share_pct": baseline.get("share_pct"), + "share_delta_pp": round( + float(current.get("share_pct") or 0.0) + - float(baseline.get("share_pct") or 0.0), + 1, + ), + "match_confidence": round(match_confidence, 2), + "match_method": match_method, + } + ) + + +def _aggregate_share_by_gap_label( + clusters: list[dict[str, Any]], +) -> dict[str, float]: + totals: dict[str, float] = {} + for cluster in clusters: + gap_label = str(cluster.get("gap_label") or "UNKNOWN") + totals[gap_label] = totals.get(gap_label, 0.0) + float( + cluster.get("share_pct") or 0.0 + ) + return {gap: round(share, 1) for gap, share in totals.items()} + + +def _reconcile_cluster_periods( + current_clusters: list[dict[str, Any]], + baseline_clusters: list[dict[str, Any]], + *, + similarity_threshold: float = 0.35, +) -> dict[str, Any]: + """Align independently-generated cluster labels before delta explanation.""" + matched: list[dict[str, Any]] = [] + current_unmatched = list(current_clusters) + remaining_baseline = list(baseline_clusters) + + current_by_gap = _group_clusters_by_gap_label(current_unmatched) + baseline_by_gap = _group_clusters_by_gap_label(remaining_baseline) + for gap_label in list(current_by_gap): + current_group = current_by_gap.get(gap_label) or [] + baseline_group = baseline_by_gap.get(gap_label) or [] + if len(current_group) != 1 or len(baseline_group) != 1: + continue + current = current_group[0] + baseline = baseline_group[0] + _append_matched_cluster_pair( + matched, + current, + baseline, + match_confidence=0.75, + match_method="single_cluster_per_gap_label", + ) + current_unmatched.remove(current) + remaining_baseline.remove(baseline) + current_by_gap[gap_label] = [] + baseline_by_gap[gap_label] = [] + + for current in list(current_unmatched): + best_idx: Optional[int] = None + best_score = 0.0 + for idx, baseline in enumerate(remaining_baseline): + score = _cluster_label_similarity( + str(current.get("label") or ""), + str(baseline.get("label") or ""), + ) + if current.get("gap_label") == baseline.get("gap_label"): + score += 0.1 + if score > best_score: + best_score = score + best_idx = idx + + if best_idx is not None and best_score >= similarity_threshold: + baseline = remaining_baseline.pop(best_idx) + _append_matched_cluster_pair( + matched, + current, + baseline, + match_confidence=best_score, + match_method="label_similarity", + ) + + matched_current_labels = { + str(item.get("current_label") or "") for item in matched + } + matched_baseline_labels = { + str(item.get("baseline_label") or "") for item in matched + } + current_unmatched = [ + cluster + for cluster in current_clusters + if str(cluster.get("label") or "") not in matched_current_labels + ] + remaining_baseline = [ + cluster + for cluster in baseline_clusters + if str(cluster.get("label") or "") not in matched_baseline_labels + ] + + new_themes = [ + { + "label": cluster.get("label"), + "gap_label": cluster.get("gap_label"), + "share_pct": cluster.get("share_pct"), + "note": "New theme in current period (no close baseline match).", + } + for cluster in current_unmatched + ] + + retired_themes = [ + { + "label": baseline.get("label"), + "gap_label": baseline.get("gap_label"), + "share_pct": baseline.get("share_pct"), + "note": "Theme present in baseline only (retired or renamed).", + } + for baseline in remaining_baseline + ] + + current_gap = _aggregate_share_by_gap_label(current_clusters) + baseline_gap = _aggregate_share_by_gap_label(baseline_clusters) + gap_label_shifts: dict[str, dict[str, float]] = {} + for gap_label in set(current_gap) | set(baseline_gap): + current_share = current_gap.get(gap_label, 0.0) + baseline_share = baseline_gap.get(gap_label, 0.0) + if abs(current_share - baseline_share) >= 0.5: + gap_label_shifts[gap_label] = { + "current_share_pct": current_share, + "baseline_share_pct": baseline_share, + "share_delta_pp": round(current_share - baseline_share, 1), + } + + return { + "matched_theme_shifts": matched, + "new_themes_current_period": new_themes, + "retired_themes_baseline_period": retired_themes, + "gap_label_shifts": gap_label_shifts, + "reconciliation_note": ( + "Cluster labels are generated independently each run and may " + "rename the same failure mode. Do not treat unmatched current " + "labels as 0% in the baseline period." + ), + } + + +def _load_period_delta_explanations_cache( + evaluation: CallImportEvaluation, + cache_key: str, +) -> Optional[dict[str, str]]: + raw = getattr(evaluation, "period_delta_explanations", None) + if not isinstance(raw, dict): + return None + entry = raw.get(cache_key) + if not isinstance(entry, dict): + return None + explanations_raw = entry.get("explanations") + if not isinstance(explanations_raw, dict): + return None + return { + str(metric_id): str(why).strip() + for metric_id, why in explanations_raw.items() + if str(metric_id).strip() and isinstance(why, str) and why.strip() + } + + +def _save_period_delta_explanations_cache( + db: Session, + evaluation: CallImportEvaluation, + cache_key: str, + explanations: dict[str, str], +) -> None: + raw = evaluation.period_delta_explanations + if not isinstance(raw, dict): + raw = {} + updated = dict(raw) + updated[cache_key] = { + "explanations": explanations, + "generated_at": datetime.now(timezone.utc).isoformat(), + } + evaluation.period_delta_explanations = updated + flag_modified(evaluation, "period_delta_explanations") + db.commit() + + +def _cluster_summary_for_metric( + state: Optional[EvaluationMetricClustersState], + metric_id: str, +) -> list[dict[str, Any]]: + if state is None or state.status != "completed": + return [] + for group in state.groups: + if str(group.metric_id) != metric_id: + continue + return [ + { + "label": cluster.label, + "gap_label": cluster.gap_label, + "share_pct": round(cluster.share_pct, 1), + "count": cluster.count, + } + for cluster in group.clusters[:5] + ] + return [] + + +def _merge_delta_why( + raw_deltas: dict[str, dict[str, str]], + explanations: dict[str, str], +) -> dict[str, dict[str, str]]: + if not explanations: + return raw_deltas + merged: dict[str, dict[str, str]] = {} + for metric_id, delta in raw_deltas.items(): + updated = dict(delta) + why = explanations.get(metric_id) + if why: + updated["why"] = why + merged[metric_id] = updated + return merged + + +def _explain_period_deltas( + db: Session, + organization_id: UUID, + evaluation: CallImportEvaluation, + baseline_evaluation: CallImportEvaluation, + raw_deltas: dict[str, dict[str, str]], + *, + min_delta_pp: float = 0.5, +) -> dict[str, dict[str, str]]: + """Attach ``why`` explanations to period deltas using cached LLM output.""" + if not raw_deltas: + return raw_deltas + + cache_key = _period_delta_explanation_cache_key( + baseline_evaluation.id, + completed_rows=evaluation.completed_rows, + baseline_completed_rows=baseline_evaluation.completed_rows, + ) + cached = _load_period_delta_explanations_cache(evaluation, cache_key) + if cached is not None: + return _merge_delta_why(raw_deltas, cached) + + current_clusters = _metric_clusters_payload(evaluation) + baseline_clusters = _metric_clusters_payload(baseline_evaluation) + metrics_for_prompt: list[dict[str, Any]] = [] + for metric_id, delta in raw_deltas.items(): + label = delta.get("label") or "" + if "No previous-week baseline" in label: + continue + match = re.search(r"([+-]?\d+(?:\.\d+)?)\s*pp", label) + if match and abs(float(match.group(1))) < min_delta_pp: + continue + current_summary = _cluster_summary_for_metric(current_clusters, metric_id) + baseline_summary = _cluster_summary_for_metric(baseline_clusters, metric_id) + if not current_summary and not baseline_summary: + continue + cluster_reconciliation = _reconcile_cluster_periods( + current_summary, + baseline_summary, + ) + metrics_for_prompt.append( + { + "metric_id": metric_id, + "delta_label": label, + "delta_detail": delta.get("detail") or "", + "cluster_reconciliation": cluster_reconciliation, + } + ) + + if not metrics_for_prompt: + return raw_deltas + + provider_hint: Optional[str] = None + model_hint: Optional[str] = None + tldr_raw = evaluation.tldr_summary + if isinstance(tldr_raw, dict): + if isinstance(tldr_raw.get("provider"), str): + provider_hint = tldr_raw["provider"] + if isinstance(tldr_raw.get("model"), str): + model_hint = tldr_raw["model"] + + from app.services.ai.llm_resolver import get_llm_provider_and_model + from app.services.call_import_user_insights import _call_llm, _parse_json_object + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, provider_hint, model_hint + ) + try: + text = _call_llm( + db, + organization_id, + provider_enum, + model_str, + [ + {"role": "system", "content": _DELTA_EXPLANATION_SYSTEM_PROMPT}, + { + "role": "user", + "content": json.dumps( + {"metrics": metrics_for_prompt}, + ensure_ascii=False, + default=str, + ), + }, + ], + temperature=0.3, + max_tokens=900, + ) + except Exception as exc: + logger.warning("[PeriodDeltaExplain] LLM call failed: {}", exc) + return raw_deltas + + parsed = _parse_json_object(text) + explanations_raw = parsed.get("explanations") + explanations: dict[str, str] = {} + if isinstance(explanations_raw, dict): + for metric_id, why in explanations_raw.items(): + if isinstance(why, str) and why.strip(): + explanations[str(metric_id)] = why.strip() + + if explanations: + _save_period_delta_explanations_cache( + db, evaluation, cache_key, explanations + ) + return _merge_delta_why(raw_deltas, explanations) + + +def _period_deltas_with_explanations( + db: Session, + organization_id: UUID, + evaluation: CallImportEvaluation, + baseline_evaluation: CallImportEvaluation, + raw_deltas: dict[str, dict[str, str]], +) -> dict[str, dict[str, str]]: + return _explain_period_deltas( + db, + organization_id, + evaluation, + baseline_evaluation, + raw_deltas, + ) + + +def _benchmark_context_for_snapshot( + db: Session, + previous_snapshot: Optional[CallImportEvaluationReportSnapshot], +) -> Optional[dict[str, str]]: + if previous_snapshot is None: + return None + previous_import = ( + db.query(CallImport) + .filter(CallImport.id == previous_snapshot.call_import_id) + .first() + ) + previous_eval = ( + db.query(CallImportEvaluation) + .filter(CallImportEvaluation.id == previous_snapshot.evaluation_id) + .first() + ) + dataset = ( + (previous_import.dataset or "").strip() + if previous_import and previous_import.dataset + else None + ) + filename = ( + (previous_import.original_filename or previous_import.filename or "").strip() + if previous_import + else None + ) + evaluation_label = ( + (previous_eval.name or "").strip() + if previous_eval and previous_eval.name + else str(previous_snapshot.evaluation_id)[:8] + ) + period = previous_snapshot.period_label or ( + previous_snapshot.period_start.isoformat() + if previous_snapshot.period_start + else "previous report" + ) + return { + "dataset": dataset or filename or "Unknown dataset", + "evaluation": evaluation_label, + "evaluation_id": str(previous_snapshot.evaluation_id), + "period": period, + } + + +def _clamp_prose_to_sentences( + text: str, + *, + max_sentences: int = 3, + max_chars: int = 300, +) -> str: + """Keep concise audit/TLDR prose within sentence and character limits.""" + cleaned = (text or "").strip() + if not cleaned: + return cleaned + cleaned = re.sub(r"\s*\n+\s*", " ", cleaned).strip() + sentences = [ + sentence.strip() + for sentence in re.split(r"(?<=[.!?])\s+", cleaned) + if sentence.strip() + ] + if sentences: + result = " ".join(sentences[:max_sentences]).strip() + else: + result = cleaned + if len(result) > max_chars: + trimmed = result[: max_chars - 3].rsplit(" ", 1)[0].rstrip(".,;:") + result = f"{trimmed}..." if trimmed else result[:max_chars] + return result + + +def _audit_summary_text_from_tldr( + summary: Optional[EvaluationTldrSummary], +) -> Optional[str]: + if summary is None: + return None + narrative = _clamp_prose_to_sentences(summary.narrative.strip()) + return narrative or None + + +def _metric_insights_from_tldr( + summary: Optional[EvaluationTldrSummary], +) -> dict[str, str]: + if summary is None: + return {} + return { + str(metric_id): insight.strip() + for metric_id, insight in summary.metric_insights.items() + if str(metric_id).strip() and insight.strip() + } + + +def _report_period_from_rows( + rows: list[tuple[CallImportEvaluationRow, CallImportRow]], +) -> tuple[Optional[date], Optional[date], Optional[str], str]: + dates = [ + source_row.recording_date + for eval_row, source_row in rows + if eval_row.status == "completed" and source_row.recording_date + ] + if not dates: + return None, None, None, "Not specified" + start = min(dates) + end = max(dates) + week_anchor = max(dates) + week_start = week_anchor - timedelta(days=week_anchor.weekday()) + week_end = week_start + timedelta(days=6) + iso_year, iso_week, _ = week_anchor.isocalendar() + label = f"{iso_year}-W{iso_week:02d}" + if week_start.year == week_end.year: + week_range = f"{week_start.strftime('%b %d')}ΓÇô{week_end.strftime('%b %d, %Y')}" + else: + week_range = ( + f"{week_start.strftime('%b %d, %Y')}ΓÇô{week_end.strftime('%b %d, %Y')}" + ) + display = f"W{iso_week:02d} · {week_range}" + return start, end, label, display + + +def _aggregate_to_dict(aggregate: CallImportMetricAggregate) -> dict[str, Any]: + if hasattr(aggregate, "model_dump"): + return aggregate.model_dump(mode="json") + return aggregate.dict() + + +def _aggregate_primary_percent( + raw: dict[str, Any], + policy: Optional[MetricFailurePolicy] = None, +) -> Optional[float]: + return aggregate_primary_percent(raw, policy) + + +def _child_names_by_parent( + db: Session, + organization_id: UUID, + parent_metric_ids: Sequence[UUID], +) -> Dict[str, List[str]]: + if not parent_metric_ids: + return {} + children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id.in_(list(parent_metric_ids)), + ) + .all() + ) + out: Dict[str, List[str]] = {} + for child in children: + pid = str(child.parent_metric_id) + out.setdefault(pid, []).append(child.name) + return out + + +def _clustering_context( + db: Session, + evaluation: CallImportEvaluation, + eval_rows: List[CallImportEvaluationRow], +) -> Tuple[ + List[Metric], + List[CallImportMetricAggregate], + Dict[str, MetricFailurePolicy], + Literal["inferred", "user"], + Dict[str, List[str]], +]: + metrics = _metrics_for_clustering(db, evaluation, eval_rows) + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + parent_ids = [ + m.id + for m in metrics + if getattr(m, "selection_mode", None) and not getattr(m, "parent_metric_id", None) + ] + child_names_by_parent = _child_names_by_parent( + db, evaluation.organization_id, parent_ids + ) + policies, source = effective_policies( + evaluation, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + return metrics, aggregates, policies, source, child_names_by_parent + + +def _period_deltas_from_aggregates( + previous_metric_aggregates: list[dict[str, Any]], + current_metric_aggregates: list[dict[str, Any]], + policies: Optional[Dict[str, MetricFailurePolicy]] = None, +) -> dict[str, dict[str, str]]: + current_by_id = {str(item.get("metric_id")): item for item in current_metric_aggregates} + previous_by_id = { + str(item.get("metric_id")): item + for item in previous_metric_aggregates + if isinstance(item, dict) + } + deltas: dict[str, dict[str, str]] = {} + for metric_id, current in current_by_id.items(): + previous_raw = previous_by_id.get(metric_id) + policy = (policies or {}).get(metric_id) + current_pct = _aggregate_primary_percent(current, policy) + previous_pct = ( + _aggregate_primary_percent(previous_raw, policy) + if previous_raw + else None + ) + if current_pct is None or previous_pct is None: + deltas[metric_id] = { + "label": "No previous-week baseline", + "detail": "No comparable prior report snapshot was found.", + } + continue + delta = current_pct - previous_pct + sign = "+" if delta >= 0 else "" + deltas[metric_id] = { + "label": f"{sign}{delta:.1f} pp", + "detail": f"Current report {current_pct:.1f}% vs previous report {previous_pct:.1f}%", + } + return deltas + + +def _period_deltas_from_snapshot( + previous: Optional[CallImportEvaluationReportSnapshot], + current_metric_aggregates: list[dict[str, Any]], +) -> dict[str, dict[str, str]]: + previous_items = ( + previous.metric_aggregates + if previous and isinstance(previous.metric_aggregates, list) + else [] + ) + return _period_deltas_from_aggregates(previous_items, current_metric_aggregates) + + +def _sample_evidence_for_metrics( + rows: list[tuple[CallImportEvaluationRow, CallImportRow]], + metric_ids: set[str], +) -> dict[str, list[dict[str, str]]]: + samples: dict[str, list[dict[str, str]]] = {metric_id: [] for metric_id in metric_ids} + for eval_row, source_row in rows: + scores = eval_row.metric_scores if isinstance(eval_row.metric_scores, dict) else {} + for metric_id in metric_ids: + if len(samples.get(metric_id, [])) >= 4: + continue + score = scores.get(metric_id) + if not isinstance(score, dict): + continue + rationale = score.get("rationale") + transcript = source_row.diarised_transcript or source_row.transcript or "" + quote = rationale if isinstance(rationale, str) and rationale.strip() else transcript[:350] + if quote: + samples.setdefault(metric_id, []).append( + { + "conversation_id": source_row.conversation_id, + "quote": str(quote).strip()[:500], + } + ) + return samples + + +def _fallback_report_narrative( + insight_aggregates: list[dict[str, Any]], + evidence_samples: dict[str, list[dict[str, str]]], +) -> dict[str, Any]: + observations: dict[str, str] = {} + evidence: dict[str, dict[str, str]] = {} + design_notes: list[str] = [] + for aggregate in insight_aggregates: + metric_id = str(aggregate.get("metric_id") or "") + name = str(aggregate.get("metric_name") or "Insight") + counts = aggregate.get("value_counts") if isinstance(aggregate.get("value_counts"), list) else [] + if counts: + top = counts[0] + total = int(aggregate.get("count") or 0) or sum( + int(item.get("count") or 0) for item in counts if isinstance(item, dict) + ) + pct = (int(top.get("count") or 0) / total) * 100 if total else 0 + observations[metric_id] = ( + f"{top.get('label')} is the dominant {name.lower()} category at {pct:.1f}% of classified calls." + ) + design_notes.append( + f"{name}: {top.get('label')} is the largest segment and should be reviewed for workflow or prompt improvements." + ) + sample = (evidence_samples.get(metric_id) or [{}])[0] + if sample: + evidence[metric_id] = sample + return { + "observations": observations, + "evidence": evidence, + "design_notes": design_notes[:7], + "audit_summary": None, + } + + +def _generate_report_narrative( + db: Session, + organization_id: UUID, + *, + metric_aggregates: list[dict[str, Any]], + insight_aggregates: list[dict[str, Any]], + period_delta_by_metric: dict[str, dict[str, str]], + evidence_samples: dict[str, list[dict[str, str]]], + report_config: dict[str, Any], +) -> dict[str, Any]: + if not insight_aggregates: + return {"observations": {}, "evidence": {}, "design_notes": [], "audit_summary": None} + try: + from app.services.ai.llm_resolver import get_llm_provider_and_model + from app.services.ai.llm_service import llm_service + + provider_enum, model_str = get_llm_provider_and_model(organization_id, db, None, None) + prompt = ( + "You are writing a vendor-safe external call quality audit report. " + "Return strict JSON with keys observations (object keyed by metric_id), " + "evidence (object keyed by metric_id with conversation_id and quote), " + "design_notes (array of concise numbered-note strings), and audit_summary (string). " + "Use only the supplied aggregates and evidence samples.\n\n" + + json.dumps( + { + "metric_aggregates": metric_aggregates[:30], + "insight_aggregates": insight_aggregates, + "period_deltas": period_delta_by_metric, + "evidence_samples": evidence_samples, + "report_config": report_config, + }, + default=str, + ) + ) + llm_result = llm_service.generate_response( + messages=[ + {"role": "system", "content": "Return JSON only. No markdown."}, + {"role": "user", "content": prompt}, + ], + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + temperature=0.2, + max_tokens=1200, + ) + parsed = json.loads(str(llm_result.content or "{}")) + if isinstance(parsed, dict): + fallback = _fallback_report_narrative(insight_aggregates, evidence_samples) + return { + "observations": parsed.get("observations") or fallback["observations"], + "evidence": parsed.get("evidence") or fallback["evidence"], + "design_notes": parsed.get("design_notes") or fallback["design_notes"], + "audit_summary": parsed.get("audit_summary") or fallback["audit_summary"], + } + except Exception as exc: # noqa: BLE001 + logger.warning("Report narrative LLM generation fell back to deterministic text: {}", exc) + return _fallback_report_narrative(insight_aggregates, evidence_samples) + + +@router.get( + "/{eval_id}/baseline-candidates", + response_model=CallImportEvaluationBaselineCandidatesResponse, + operation_id="listCallImportEvaluationBaselineCandidates", +) +async def list_call_import_evaluation_baseline_candidates( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationBaselineCandidatesResponse: + del api_key + call_import = _require_import(db, call_import_id, organization_id) + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + rows = _evaluation_rows_for_period(db, evaluation.id) + period_start, _period_end, _derived_period_label, _period_display = _report_period_from_rows( + rows + ) + candidates = _baseline_candidate_evaluations( + db, + organization_id, + call_import.workspace_id, + evaluation, + period_start, + ) + default_evaluation_id = next( + (item["evaluation_id"] for item in candidates if item.get("is_default")), + None, + ) + return CallImportEvaluationBaselineCandidatesResponse( + items=[CallImportEvaluationBaselineCandidate(**item) for item in candidates], + default_evaluation_id=default_evaluation_id, + ) + + +@router.post( + "/{eval_id}/pdf-report", + operation_id="generateCallImportEvaluationPdfReport", + dependencies=[Depends(require_call_import_capability(REPORTS_GENERATE))], +) +async def generate_call_import_evaluation_pdf_report( + call_import_id: UUID, + eval_id: UUID, + payload: CallImportEvaluationPdfReportRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +): + del api_key + call_import = _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + is_internal = payload.report_type == "internal" + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + rows = sorted( + load_evaluation_row_pairs(db, eval_id), + key=lambda pair: int(pair[1].row_index or 0), + ) + else: + rows = ( + db.query(CallImportEvaluationRow, CallImportRow) + .join( + CallImportRow, + CallImportRow.id == CallImportEvaluationRow.call_import_row_id, + ) + .filter(CallImportEvaluationRow.evaluation_id == eval_id) + .order_by(CallImportRow.row_index.asc()) + .all() + ) + report_config = payload.report_config if isinstance(payload.report_config, dict) else {} + metrics = _display_metrics_for_pdf_report(db, organization_id, evaluation) + configured_quality_ids = { + str(item) + for item in report_config.get("quality_metric_ids", []) + if item + } + configured_insight_ids = { + str(item.get("metric_id") or item) + for item in report_config.get("insights", []) + if item + } + if configured_quality_ids or configured_insight_ids: + allowed_ids = configured_quality_ids | configured_insight_ids + metrics = [metric for metric in metrics if str(metric.id) in allowed_ids] + + eval_rows = [eval_row for eval_row, _source_row in rows] + aggregate_models = _compute_metric_aggregates(db, evaluation, eval_rows) + selected_report_metric_ids = {str(metric.id) for metric in metrics} + aggregate_dicts = [ + _aggregate_to_dict(aggregate) + for aggregate in aggregate_models + if aggregate.metric_id in selected_report_metric_ids + ] + insight_metric_ids = { + str(metric.id) + for metric in metrics + if _metric_is_user_insight(metric) + } + metric_aggregates = [ + item for item in aggregate_dicts if str(item.get("metric_id")) not in insight_metric_ids + ] + insight_aggregates = [ + item for item in aggregate_dicts if str(item.get("metric_id")) in insight_metric_ids + ] + period_start, period_end, derived_period_label, period_display = _report_period_from_rows(rows) + period_label = (payload.period_label or derived_period_label or "").strip() or None + include_period_delta = ( + payload.include_period_delta or payload.include_weekly_delta + ) + previous_snapshot = None + period_delta_by_metric: dict[str, dict[str, str]] = {} + baseline_evaluation: Optional[CallImportEvaluation] = None + if include_period_delta and period_start: + baseline_evaluation = _resolve_baseline_evaluation( + db, + organization_id, + call_import.workspace_id, + evaluation, + period_start, + payload.baseline_evaluation_id, + ) + if baseline_evaluation: + period_delta_by_metric = _period_deltas_from_evaluation( + db, + baseline_evaluation, + metric_aggregates, + evaluation, + [eval_row for eval_row, _ in rows], + ) + period_delta_by_metric = _period_deltas_with_explanations( + db, + organization_id, + evaluation, + baseline_evaluation, + period_delta_by_metric, + ) + benchmark_context = _benchmark_context_for_evaluation(db, baseline_evaluation) + evidence_samples = _sample_evidence_for_metrics(rows, insight_metric_ids) + cached_tldr_summary = _tldr_summary_payload(evaluation) + cached_user_insights = _user_insights_payload(evaluation) + cached_metric_clusters = _metric_clusters_payload(evaluation) + cached_prompt_improvements = _prompt_improvements_payload(evaluation) + generated_insights_for_pdf = _selected_generated_user_insights( + cached_user_insights, + report_config, + ) + metric_clusters_for_pdf = _selected_metric_clusters_for_pdf( + cached_metric_clusters, + report_config, + ) + prompt_improvements_for_pdf = _selected_prompt_improvements_for_pdf( + cached_prompt_improvements, + report_config, + ) + branding_images, custom_heading = _report_branding_for_import_workspace( + db, + organization_id, + call_import.workspace_id, + internal_brand_image_id=payload.internal_brand_image_id, + external_brand_image_id=payload.external_brand_image_id, + ) + eval_row_list = [eval_row for eval_row, _ in rows] + pdf_aggregates = _compute_metric_aggregates(db, evaluation, eval_row_list) + pdf_parent_ids = [ + m.id + for m in metrics + if getattr(m, "selection_mode", None) + and not getattr(m, "parent_metric_id", None) + ] + pdf_child_map = _child_names_by_parent( + db, evaluation.organization_id, pdf_parent_ids + ) + failure_policies_for_pdf, _fp_source = effective_policies( + evaluation, + metrics, + pdf_aggregates, + child_names_by_parent=pdf_child_map, + ) + + from app.services.storage.s3_service import s3_service + + config_fingerprint = compute_pdf_report_config_fingerprint( + report_type=payload.report_type, + include_period_delta=bool(payload.include_period_delta), + include_weekly_delta=bool(payload.include_weekly_delta), + baseline_evaluation_id=payload.baseline_evaluation_id, + internal_brand_image_id=payload.internal_brand_image_id, + external_brand_image_id=payload.external_brand_image_id, + use_case=payload.use_case, + report_config=report_config, + report_heading=custom_heading, + vendor_name=payload.vendor_name, + platform_base_url=payload.platform_base_url, + period_label=period_label, + ) + content_fingerprint = compute_pdf_report_content_fingerprint( + evaluation_status=evaluation.status, + completed_rows=int(evaluation.completed_rows or 0), + total_rows=int(evaluation.total_rows or 0), + failed_rows=int(evaluation.failed_rows or 0), + metric_aggregates=metric_aggregates, + insight_aggregates=insight_aggregates, + period_delta_by_metric=period_delta_by_metric, + benchmark_context=benchmark_context, + metric_metadata=[ + { + "id": str(metric.id), + "name": metric.name, + "description": metric.description, + } + for metric in metrics + ], + failure_policies=failure_policies_for_pdf, + tldr_summary=cached_tldr_summary, + user_insights_for_pdf=generated_insights_for_pdf, + metric_clusters_for_pdf=metric_clusters_for_pdf, + prompt_improvements_for_pdf=prompt_improvements_for_pdf, + ) + cache_fingerprint = compute_pdf_report_cache_fingerprint( + config_fingerprint=config_fingerprint, + content_fingerprint=content_fingerprint, + ) + if s3_service.is_enabled(): + cached_pdf_report = find_cached_pdf_report( + db, + evaluation_id=evaluation.id, + organization_id=organization_id, + cache_fingerprint=cache_fingerprint, + ) + if cached_pdf_report is not None: + logger.info( + "Reusing stored PDF report {} for evaluation {} (cache fingerprint match)", + cached_pdf_report.id, + eval_id, + ) + return _pdf_report_response_from_row(cached_pdf_report, cache_hit=True) + + narrative = _generate_report_narrative( + db, + organization_id, + metric_aggregates=metric_aggregates, + insight_aggregates=insight_aggregates if is_internal else [], + period_delta_by_metric=period_delta_by_metric, + evidence_samples=evidence_samples if is_internal else {}, + report_config=report_config, + ) + + generated_at = datetime.now(timezone.utc) + try: + pdf_started = datetime.now(timezone.utc) + pdf_bytes = await asyncio.to_thread( + call_import_evaluation_pdf_report_service.render_pdf, + vendor_name=payload.vendor_name, + call_import=call_import, + evaluation=evaluation, + metrics=metrics, + rows=rows, + failure_policies=failure_policies_for_pdf, + generated_at=generated_at, + internal=is_internal, + logo_data_uris=branding_images, + custom_heading=custom_heading, + include_weekly_delta=include_period_delta, + period_delta_by_metric=period_delta_by_metric, + use_case=payload.use_case, + period_display=period_display, + total_metric_count=db.query(Metric) + .filter(Metric.organization_id == organization_id, Metric.enabled.is_(True)) + .count(), + report_config=report_config, + narrative=narrative, + audit_summary=_audit_summary_text_from_tldr(cached_tldr_summary), + metric_insights=_metric_insights_from_tldr(cached_tldr_summary), + benchmark_context=benchmark_context, + generated_user_insights=generated_insights_for_pdf, + user_insights_overview=( + cached_user_insights.overview if cached_user_insights else None + ), + metric_clusters=metric_clusters_for_pdf, + metric_clusters_overview=( + cached_metric_clusters.overview if cached_metric_clusters else None + ), + prompt_improvements=prompt_improvements_for_pdf, + platform_base_url=payload.platform_base_url, + ) + logger.info( + "PDF report render finished in {:.1f}s for evaluation {}", + (datetime.now(timezone.utc) - pdf_started).total_seconds(), + eval_id, + ) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to generate PDF report for call import {} evaluation {}", + call_import_id, + eval_id, + ) + raise HTTPException( + status_code=500, + detail=f"Failed to generate PDF report: {exc}", + ) from exc + + snapshot = CallImportEvaluationReportSnapshot( + evaluation_id=evaluation.id, + call_import_id=call_import.id, + organization_id=organization_id, + workspace_id=call_import.workspace_id, + period_label=period_label, + period_start=period_start, + period_end=period_end, + report_config=report_config, + selected_metric_ids=[str(metric.id) for metric in metrics], + metric_aggregates=metric_aggregates, + insight_aggregates=insight_aggregates, + narrative=narrative, + total_calls=evaluation.total_rows, + selected_metric_count=len(metrics), + total_metric_count=db.query(Metric) + .filter(Metric.organization_id == organization_id, Metric.enabled.is_(True)) + .count(), + ) + db.add(snapshot) + db.flush() + + filename = ( + f"{_report_filename_slug(payload.vendor_name)}-" + f"{payload.report_type}-quality-metric-audit-{eval_id}.pdf" + ) + + if not s3_service.is_enabled(): + db.commit() + return StreamingResponse( + iter([pdf_bytes]), + media_type="application/pdf", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + report_id = uuid4() + s3_key = build_pdf_report_s3_key( + organization_id=organization_id, + call_import_id=call_import.id, + evaluation_id=evaluation.id, + report_id=report_id, + ) + try: + s3_service.upload_file_by_key( + file_content=pdf_bytes, + key=s3_key, + content_type="application/pdf", + ) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to upload PDF report for evaluation {} to object storage", + eval_id, + ) + db.rollback() + raise HTTPException( + status_code=500, + detail=f"Failed to store PDF report: {exc}", + ) from exc + + created_by, created_by_user_id = _pdf_report_actor(principal) + pdf_report = CallImportEvaluationPdfReport( + id=report_id, + evaluation_id=evaluation.id, + call_import_id=call_import.id, + organization_id=organization_id, + workspace_id=call_import.workspace_id, + snapshot_id=snapshot.id, + vendor_name=payload.vendor_name, + report_type=payload.report_type, + filename=filename, + s3_key=s3_key, + report_config=report_config, + cache_fingerprint=cache_fingerprint, + created_by=created_by, + created_by_user_id=created_by_user_id, + ) + db.add(pdf_report) + try: + db.commit() + except IntegrityError: + db.rollback() + try: + s3_service.delete_file_by_key(s3_key) + except Exception: # noqa: BLE001 + logger.warning( + "Failed to delete orphan PDF after cache race for evaluation {}", + eval_id, + ) + raced_winner = find_cached_pdf_report( + db, + evaluation_id=evaluation.id, + organization_id=organization_id, + cache_fingerprint=cache_fingerprint, + ) + if raced_winner is not None: + logger.info( + "PDF report cache race resolved for evaluation {} (winner {})", + eval_id, + raced_winner.id, + ) + return _pdf_report_response_from_row(raced_winner, cache_hit=True) + raise HTTPException( + status_code=500, + detail="Failed to store PDF report due to a concurrent duplicate request.", + ) from None + db.refresh(pdf_report) + return _pdf_report_response_from_row(pdf_report) + + +@router.get( + "/{eval_id}/pdf-reports", + response_model=CallImportEvaluationPdfReportListResponse, + operation_id="listCallImportEvaluationPdfReports", +) +async def list_call_import_evaluation_pdf_reports( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationPdfReportListResponse: + del api_key + _require_import(db, call_import_id, organization_id) + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + rows = ( + db.query(CallImportEvaluationPdfReport) + .filter( + CallImportEvaluationPdfReport.evaluation_id == eval_id, + CallImportEvaluationPdfReport.organization_id == organization_id, + ) + .order_by(desc(CallImportEvaluationPdfReport.created_at)) + .all() + ) + return CallImportEvaluationPdfReportListResponse( + items=[_pdf_report_list_item_from_row(row) for row in rows], + ) + + +@router.get( + "/{eval_id}/pdf-reports/{report_id}", + response_model=CallImportEvaluationPdfReportResponse, + operation_id="getCallImportEvaluationPdfReport", +) +async def get_call_import_evaluation_pdf_report( + call_import_id: UUID, + eval_id: UUID, + report_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationPdfReportResponse: + del api_key + _require_import(db, call_import_id, organization_id) + row = ( + db.query(CallImportEvaluationPdfReport) + .filter( + CallImportEvaluationPdfReport.id == report_id, + CallImportEvaluationPdfReport.evaluation_id == eval_id, + CallImportEvaluationPdfReport.call_import_id == call_import_id, + CallImportEvaluationPdfReport.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="PDF report not found") + if not row.s3_key: + raise HTTPException( + status_code=404, + detail="PDF report file is not available in object storage", + ) + return _pdf_report_response_from_row(row) + + +@router.get( + "/{eval_id}/pdf-reports/{report_id}/download", + operation_id="downloadCallImportEvaluationPdfReport", +) +async def download_call_import_evaluation_pdf_report( + call_import_id: UUID, + eval_id: UUID, + report_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +): + del api_key + _require_import(db, call_import_id, organization_id) + row = ( + db.query(CallImportEvaluationPdfReport) + .filter( + CallImportEvaluationPdfReport.id == report_id, + CallImportEvaluationPdfReport.evaluation_id == eval_id, + CallImportEvaluationPdfReport.call_import_id == call_import_id, + CallImportEvaluationPdfReport.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="PDF report not found") + if not row.s3_key: + raise HTTPException( + status_code=404, + detail="PDF report file is not available in object storage", + ) + from app.services.storage.s3_service import s3_service + + if not s3_service.is_enabled(): + raise HTTPException( + status_code=503, + detail="Object storage is not enabled or not configured.", + ) + try: + file_bytes = s3_service.download_file_by_key(row.s3_key) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to download PDF report {} for evaluation {}", + report_id, + eval_id, + ) + raise HTTPException( + status_code=500, + detail=f"Failed to download PDF report: {exc}", + ) from exc + filename = row.filename or "report.pdf" + safe_name = filename.replace('"', "'") + return StreamingResponse( + iter([file_bytes]), + media_type="application/pdf", + headers={"Content-Disposition": f'attachment; filename="{safe_name}"'}, + ) + + +@router.patch( + "/{eval_id}", + response_model=CallImportEvaluationResponse, + operation_id="updateCallImportEvaluation", +) +async def update_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + payload: CallImportEvaluationUpdate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationResponse: + """Edit metadata on an existing evaluation run (currently just ``name``).""" + + del api_key + _require_import(db, call_import_id, organization_id) + + row = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + # Treat unset vs explicit ``None`` differently: unset = leave alone, + # explicit ``None`` or empty string = clear the name. + payload_data = payload.model_dump(exclude_unset=True) + if "name" in payload_data: + row.name = _normalize_name(payload_data["name"]) + + stamp_evaluation_actor(row, principal) + db.commit() + db.refresh(row) + return _serialize_eval(db, row) + + +def _revoke_pending_tasks(evaluation: CallImportEvaluation) -> None: + """Best-effort cancel of any in-flight Celery tasks for an evaluation.""" + + if not evaluation.celery_group_id and not any( + r.celery_task_id for r in evaluation.row_results + ): + return + try: + from app.workers.celery_app import celery_app + + pending_task_ids = [ + eval_row.celery_task_id + for eval_row in evaluation.row_results + if eval_row.celery_task_id + and eval_row.status in {"pending", "running"} + ] + if pending_task_ids: + celery_app.control.revoke(pending_task_ids, terminate=False) + except Exception: + # Best effort — DB delete remains the source of truth. + pass + + +# --------------------------------------------------------------------------- +# User-initiated cancel for in-flight evaluation rows +# --------------------------------------------------------------------------- +# +# Evaluation rows can sit in ``running`` for many minutes when the underlying +# LLM / audio metric call is slow or wedged (the worker carries an 8 min +# soft / 10 min hard time limit). Without a cancel affordance the operator's +# only recourse is to wait for Celery's time limit to fire — or to manually +# mutate the DB. These helpers + the two endpoints below give the UI a +# first-class "Abort" button mirroring the diarisation cancel pattern at +# ``app.api.v1.routes.call_imports`` (``_apply_diarisation_cancel`` etc.). +# +# Why ``terminate=True``: the legacy ``_revoke_pending_tasks`` above uses +# ``terminate=False`` because it's called from delete-flow paths where the +# task may simply not get to run (a worker pulls it off the queue and drops +# it). For a user-initiated cancel we want SIGTERM to interrupt the worker +# mid-LLM/audio call so the in-flight HTTP request actually aborts. +# ``terminate=True`` routes the signal to the executing process; we spell +# ``signal="SIGTERM"`` out for clarity even though it's the default. + +# Sentinel error message stamped on cancelled rows. Read by the eval worker's +# ``_was_cancelled_externally`` guard (see +# :mod:`app.workers.tasks.evaluate_call_import_row`) so a worker that's already +# past its slowest operation can't overwrite the cancelled state with its own +# terminal status. Touching either copy means touching both. +EVAL_CANCELLED_BY_USER_ERROR: str = "Evaluation cancelled by user" + + +def _cancellable_eval_states() -> Tuple[str, ...]: + """States that an evaluation row can be cancelled from. + + Kept as a tiny helper so adding a future ``"queued"`` / ``"retrying"`` + state only needs one edit. + """ + return ("pending", "running") + + +def _revoke_eval_task(eval_row: CallImportEvaluationRow) -> None: + """Best-effort revoke of a single eval row's Celery task. + + Always swallows control-plane exceptions — Celery's control bus is + inherently best-effort and a missed revoke is not catastrophic + because the DB row is already flipped to ``failed`` by the caller + before this runs (so the UI immediately reflects the cancel; if + the task happens to finish anyway, the worker's finaliser skips + over the row via :data:`EVAL_CANCELLED_BY_USER_ERROR`). + """ + task_id = (eval_row.celery_task_id or "").strip() + if not task_id: + return + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke( + task_id, terminate=True, signal="SIGTERM" + ) + logger.info( + "Revoked evaluation task {} for eval row {}", + task_id, + eval_row.id, + ) + except Exception as exc: # noqa: BLE001 — revoke is best-effort + logger.warning( + "Failed to revoke evaluation task {} for eval row {}: {}", + task_id, + eval_row.id, + exc, + ) + + +def _cancel_eval_row_with_source_diarisation( + eval_row: CallImportEvaluationRow, + source_row: CallImportRow | None, + *, + cascade_diarisation: bool, + now: datetime | None = None, +) -> list[str]: + """Mark an eval row cancelled; optionally fail in-flight diarisation. + + Returns Celery task ids to revoke (deduped). The caller may batch-revoke + them; when ``cascade_diarisation`` is true, diarisation revoke also runs + inside :func:`_apply_diarisation_cancel` for the source row's task id. + """ + cancellable_states = _cancellable_eval_states() + if (eval_row.status or "").lower() not in cancellable_states: + return [] + + stamp = now or datetime.now(timezone.utc) + task_ids: list[str] = [] + eval_task_id = (eval_row.celery_task_id or "").strip() + if eval_task_id: + task_ids.append(eval_task_id) + + eval_row.status = "failed" + eval_row.error_message = EVAL_CANCELLED_BY_USER_ERROR + eval_row.finished_at = stamp + eval_row.celery_task_id = None + + if cascade_diarisation and source_row is not None: + from app.api.v1.routes.call_imports import _apply_diarisation_cancel + + source_task_id = (source_row.celery_task_id or "").strip() + if source_task_id and source_task_id not in task_ids: + task_ids.append(source_task_id) + _apply_diarisation_cancel([source_row]) + + return list(dict.fromkeys(task_ids)) + + +def _apply_evaluation_cancel( + eval_rows: List[CallImportEvaluationRow], + *, + source_rows: dict[UUID, CallImportRow] | None = None, + cascade_diarisation: bool = False, +) -> Tuple[int, int]: + """Cancel every cancellable row in ``eval_rows``. + + Returns ``(cancelled, skipped)`` so the caller can build a typed + response without re-querying the DB. The caller is responsible for + ``db.commit()`` after this returns — we deliberately don't commit + here so a batch endpoint can flush all rows in one transaction. + """ + cancellable_states = _cancellable_eval_states() + cancelled = 0 + skipped = 0 + now = datetime.now(timezone.utc) + source_rows = source_rows or {} + for eval_row in eval_rows: + if (eval_row.status or "").lower() not in cancellable_states: + skipped += 1 + continue + source_row = source_rows.get(eval_row.call_import_row_id) + task_ids = _cancel_eval_row_with_source_diarisation( + eval_row, + source_row, + cascade_diarisation=cascade_diarisation, + now=now, + ) + for task_id in task_ids: + _revoke_eval_task_by_id(task_id, eval_row_id=eval_row.id) + cancelled += 1 + return cancelled, skipped + + +def _revoke_eval_task_by_id(task_id: str, *, eval_row_id: UUID) -> None: + """Best-effort revoke when the task id is known outside the ORM row.""" + cleaned = (task_id or "").strip() + if not cleaned: + return + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke( + cleaned, terminate=True, signal="SIGTERM" + ) + logger.info( + "Revoked evaluation task {} for eval row {}", + cleaned, + eval_row_id, + ) + except Exception as exc: # noqa: BLE001 — revoke is best-effort + logger.warning( + "Failed to revoke evaluation task {} for eval row {}: {}", + cleaned, + eval_row_id, + exc, + ) + + +def _claim_evaluation_bulk_operation( + evaluation_id: UUID, + operation: str, +) -> None: + """Reserve the run for a single bulk worker pass; 409 if one is active.""" + from app.services.call_imports.evaluation_bulk_op import ( + get_evaluation_bulk_operation, + try_set_evaluation_bulk_operation, + ) + + if try_set_evaluation_bulk_operation(evaluation_id, operation): # type: ignore[arg-type] + return + existing = get_evaluation_bulk_operation(evaluation_id) or operation + raise HTTPException( + status_code=409, + detail=( + f"A bulk {existing.replace('_', ' ')} operation is already in " + "progress for this evaluation. Wait for it to finish before " + "starting another action." + ), + ) + + +def _require_no_evaluation_bulk_operation(evaluation_id: UUID) -> None: + from app.services.call_imports.evaluation_bulk_op import ( + get_evaluation_bulk_operation, + ) + + existing = get_evaluation_bulk_operation(evaluation_id) + if existing: + raise HTTPException( + status_code=409, + detail=( + f"A bulk {existing.replace('_', ' ')} operation is already in " + "progress for this evaluation. Wait for it to finish before " + "starting another action." + ), + ) + + +@router.post( + "/{eval_id}/cancel", + response_model=CallImportEvaluationBulkActionResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="cancelCallImportEvaluation", +) +async def cancel_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationBulkActionResponse: + """Abort all in-flight (or queued) rows in a single evaluation run. + + Idempotent: calling on a run whose rows are already terminal returns + ``target_count=0`` with 202 so the UI can fire this from an + "Abort" button without having to pre-check the state. + + Heavy row resets and Celery revokes run in a background worker so + large batches do not block the API thread. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + from app.services.call_imports.bulk_ops import count_evaluation_cancel_targets + + target_count = count_evaluation_cancel_targets(db, eval_id, mode="abort") + if target_count == 0: + from app.services.call_imports.bulk_ops import ( + _sweep_evaluation_diarisation_cancel, + ) + + swept = _sweep_evaluation_diarisation_cancel(db, eval_id) + if swept: + db.commit() + return CallImportEvaluationBulkActionResponse( + accepted=True, + target_count=0, + evaluation_id=eval_id, + ) + + _claim_evaluation_bulk_operation(eval_id, "abort") + evaluation.status = "cancelled" + stamp_evaluation_actor(evaluation, principal) + db.commit() + + from app.workers.tasks.call_import_bulk_ops import ( + cancel_call_import_evaluation_task, + ) + + cancel_call_import_evaluation_task.delay(str(eval_id), mode="abort") + return CallImportEvaluationBulkActionResponse( + accepted=True, + target_count=target_count, + evaluation_id=eval_id, + ) + + +@router.post( + "/{eval_id}/force-fail-pending", + response_model=CallImportEvaluationBulkActionResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="forceFailCallImportEvaluationPending", +) +async def force_fail_pending_call_import_evaluation_rows( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationBulkActionResponse: + """Force-fail only rows currently in ``pending`` for a single run. + + This is narrower than :func:`cancel_call_import_evaluation`: it leaves + ``running`` rows untouched so operators can clear permanently queued rows + without interrupting in-flight evaluations. + + Row updates run in a background worker so large batches do not block + the API thread. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + from app.services.call_imports.bulk_ops import count_evaluation_cancel_targets + + target_count = count_evaluation_cancel_targets( + db, eval_id, mode="force_fail_pending" + ) + if target_count == 0: + return CallImportEvaluationBulkActionResponse( + accepted=True, + target_count=0, + evaluation_id=eval_id, + ) + + _claim_evaluation_bulk_operation(eval_id, "force_fail_pending") + stamp_evaluation_actor(evaluation, principal) + db.commit() + + from app.workers.tasks.call_import_bulk_ops import ( + cancel_call_import_evaluation_task, + ) + + cancel_call_import_evaluation_task.delay( + str(eval_id), mode="force_fail_pending" + ) + return CallImportEvaluationBulkActionResponse( + accepted=True, + target_count=target_count, + evaluation_id=eval_id, + ) + + +@router.post( + "/{eval_id}/rows/{eval_row_id}/cancel", + response_model=CallImportEvaluationRowResponse, + status_code=status.HTTP_200_OK, + operation_id="cancelCallImportEvaluationRow", +) +async def cancel_call_import_evaluation_row( + call_import_id: UUID, + eval_id: UUID, + eval_row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationRowResponse: + """Abort an in-flight (or queued) evaluation for a single row. + + Idempotent: calling on a row that's already terminal (``completed`` + / ``failed``) returns the row unchanged with a 200 so the UI can + wire this to a "Stop" button without having to pre-check the + state. Updates the parent run's rollup so its counters reflect + the cancel immediately. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + _require_no_evaluation_bulk_operation(eval_id) + + from app.db_sharding.eval_rows import evaluation_row_session + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + try: + with evaluation_row_session(eval_row_id) as ( + row_db, + _catalog_db, + eval_row, + source_row, + _shard_id, + ): + if eval_row.evaluation_id != eval_id: + raise HTTPException( + status_code=404, + detail="Evaluation row not found in this run", + ) + _apply_evaluation_cancel( + [eval_row], + source_rows={source_row.id: source_row}, + cascade_diarisation=True, + ) + row_db.commit() + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + row_db.refresh(eval_row) + return _to_evaluation_row_response(eval_row, source_row, evaluation) + except LookupError as exc: + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) from exc + + eval_row = ( + db.query(CallImportEvaluationRow) + .filter( + CallImportEvaluationRow.id == eval_row_id, + CallImportEvaluationRow.evaluation_id == eval_id, + ) + .first() + ) + if not eval_row: + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) + + source_row = ( + db.query(CallImportRow) + .filter(CallImportRow.id == eval_row.call_import_row_id) + .first() + ) + source_rows = ( + {source_row.id: source_row} if source_row is not None else None + ) + _apply_evaluation_cancel( + [eval_row], + source_rows=source_rows, + cascade_diarisation=True, + ) + db.flush() + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(eval_row) + + return _to_evaluation_row_response(eval_row, source_row, evaluation) + + +@router.delete( + "/{eval_id}", + status_code=status.HTTP_204_NO_CONTENT, + operation_id="deleteCallImportEvaluation", +) +async def delete_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> Response: + del api_key + _require_import(db, call_import_id, organization_id) + + row = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + _revoke_pending_tasks(row) + + db.delete(row) + db.commit() + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +@router.post( + "/bulk-delete", + status_code=status.HTTP_200_OK, + operation_id="bulkDeleteCallImportEvaluations", +) +async def bulk_delete_call_import_evaluations( + call_import_id: UUID, + payload: CallImportEvaluationBulkDelete, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> Dict[str, int]: + """Delete multiple evaluation runs scoped to one call import. + + Mirrors :func:`delete_call_import_evaluation` but in bulk so the UI + can clear out a multi-select. Unknown ids (already deleted, or + belonging to a different org/import) are silently skipped — the + response just reports how many actually went away. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + if not payload.evaluation_ids: + return {"deleted": 0} + + rows = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id.in_(payload.evaluation_ids), + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .all() + ) + deleted = 0 + for row in rows: + _revoke_pending_tasks(row) + db.delete(row) + deleted += 1 + db.commit() + return {"deleted": deleted} + + +# --------------------------------------------------------------------------- +# Aggregation: turns per-row metric scores into histograms / value counts. +# +# Designed to be cheap enough to call on every page load: we read each +# evaluation row once, bucket numeric values into a fixed 10-bin +# histogram, and tally the top categorical values. Scaling concerns +# (millions of rows) are deferred — at that point we'd push this into a +# Postgres aggregate query, but for typical CSV imports (<10k rows) the +# Python pass is fast enough and dramatically simpler. +# --------------------------------------------------------------------------- + + +_HISTOGRAM_BUCKETS = 10 +_TOP_VALUE_COUNTS = 10 + + +def _coerce_numeric(value: Any) -> Optional[float]: + """Return ``value`` as ``float`` when it's numeric; ``None`` otherwise.""" + if isinstance(value, bool): + # Booleans are ints in Python; treat them as categorical so + # pass/fail metrics show up in value_counts instead of becoming + # a degenerate {0,1} histogram. + return None + if isinstance(value, (int, float)) and math.isfinite(value): + return float(value) + if isinstance(value, str): + try: + f = float(value) + if math.isfinite(f): + return f + except ValueError: + return None + return None + + +def _coerce_category(value: Any) -> Optional[str]: + """Render ``value`` as a label suitable for a value_counts bucket.""" + if value is None: + return None + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, str): + text = value.strip() + return text or None + # Lists / dicts: stringify so they still group sensibly without + # exploding the cardinality (worst case: everything is "[…]" once). + return str(value) + + +def _build_histogram( + values: List[float], +) -> List[CallImportMetricHistogramBucket]: + """Fixed-bin histogram over ``values``; returns [] for <2 values.""" + if len(values) < 2: + return [] + lo = min(values) + hi = max(values) + if lo == hi: + # All values identical — render a single bucket so the UI shows a + # spike rather than empty space. + return [ + CallImportMetricHistogramBucket(x0=lo, x1=hi, count=len(values)) + ] + width = (hi - lo) / _HISTOGRAM_BUCKETS + buckets: List[List[float]] = [[] for _ in range(_HISTOGRAM_BUCKETS)] + for v in values: + # Right-edge inclusive on the last bucket so ``hi`` doesn't fall + # off into a non-existent bucket index. + idx = int((v - lo) / width) + if idx >= _HISTOGRAM_BUCKETS: + idx = _HISTOGRAM_BUCKETS - 1 + buckets[idx].append(v) + return [ + CallImportMetricHistogramBucket( + x0=lo + i * width, + x1=lo + (i + 1) * width, + count=len(bucket), + ) + for i, bucket in enumerate(buckets) + ] + + +def _percentile(values: List[float], pct: float) -> Optional[float]: + """Linear-interpolated percentile compatible with NumPy default.""" + if not values: + return None + sorted_vals = sorted(values) + if len(sorted_vals) == 1: + return sorted_vals[0] + rank = (pct / 100.0) * (len(sorted_vals) - 1) + lo = int(math.floor(rank)) + hi = int(math.ceil(rank)) + if lo == hi: + return sorted_vals[lo] + frac = rank - lo + return sorted_vals[lo] + (sorted_vals[hi] - sorted_vals[lo]) * frac + + +def _compute_metric_aggregates( + db: Session, + evaluation: CallImportEvaluation, + eval_rows: List[CallImportEvaluationRow], +) -> List[CallImportMetricAggregate]: + """Collapse per-row ``metric_scores`` into one aggregate per metric. + + Selected metrics are read fresh from the DB so the response always + surfaces the current ``metric.name`` / ``metric_type`` even when a + metric was renamed after the run finished. + """ + + selected_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) + # Include parent metrics from selected_metric_groups so they appear + # alongside their children in the aggregate response. Use ``getattr`` + # with a default so the helper still works for callers that pass + # lightweight objects (tests, in-memory shims) that don't carry the + # attribute at all. + groups_raw_candidate = getattr(evaluation, "selected_metric_groups", None) + groups_raw = ( + groups_raw_candidate if isinstance(groups_raw_candidate, dict) else {} + ) + for parent_str in groups_raw.keys(): + try: + pid = UUID(parent_str) + if pid not in selected_ids: + selected_ids.append(pid) + except (TypeError, ValueError): + continue + + metrics = _metrics_for_ids(db, evaluation.organization_id, selected_ids) + metric_meta: Dict[str, Metric] = {str(m.id): m for m in metrics} + + # Default to selected metrics, but also include any metric ids that + # surface in row scores even if missing from the metric registry — + # otherwise renaming/deleting a metric mid-run would silently drop + # results from the chart. + discovered_ids: List[str] = list(metric_meta.keys()) + for row in eval_rows: + scores = row.metric_scores if isinstance(row.metric_scores, dict) else {} + for metric_id_str in scores.keys(): + if metric_id_str not in metric_meta and metric_id_str not in discovered_ids: + discovered_ids.append(metric_id_str) + + results: List[CallImportMetricAggregate] = [] + + for metric_id_str in discovered_ids: + meta = metric_meta.get(metric_id_str) + numeric_values: List[float] = [] + category_counts: Dict[str, int] = {} + # For multi-label parents we still need to know how many rows + # were scored (each row votes for >=1 label) so the n-badge in + # the UI shows "n=50" instead of the misleading "n=208" sum. + multi_label_rows_scored = 0 + # Unordered pair tally for the co-occurrence heatmap. Keys are + # ``(label_a, label_b)`` with ``a < b`` so we never double-count + # the same unordered pair. Only populated for multi-label + # parents — every other metric leaves this empty. + multi_label_pair_counts: Dict[Tuple[str, str], int] = {} + skipped = 0 + errored = 0 + observed_metric_type: Optional[str] = None + observed_name: Optional[str] = None + + # ``meta`` is a real ``Metric`` row in production, but tests + # frequently pass a lightweight stub. Pull the two attributes + # we need via ``getattr`` so a stub that only sets ``id`` / + # ``name`` / ``metric_type`` doesn't blow up here. + is_multi_label_parent = bool( + meta + and getattr(meta, "selection_mode", None) == "multi_label" + and not getattr(meta, "parent_metric_id", None) + ) + + for row in eval_rows: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else {} + ) + entry = scores.get(metric_id_str) + if not isinstance(entry, dict): + continue + if entry.get("metric_name"): + observed_name = entry.get("metric_name") + if entry.get("type"): + observed_metric_type = entry.get("type") + if entry.get("skipped"): + skipped += 1 + continue + if entry.get("error"): + errored += 1 + continue + + # Multi-label parents store a comma-joined value that + # isn't useful as a single category; instead tally each + # selected child individually so the chart shows per-label + # counts that mirror the children's own boolean histograms. + if is_multi_label_parent: + selected = entry.get("selected_child_names") + if isinstance(selected, list) and selected: + multi_label_rows_scored += 1 + cleaned: List[str] = [] + for label in selected: + text_label = str(label).strip() or None + if text_label: + cleaned.append(text_label) + category_counts[text_label] = ( + category_counts.get(text_label, 0) + 1 + ) + # Emit one increment per unordered pair of distinct + # labels that fired together on this row. ``cleaned`` + # is deduplicated first because the LLM occasionally + # repeats a label inside ``selected_child_names``. + distinct = sorted(set(cleaned)) + for i in range(len(distinct)): + for j in range(i + 1, len(distinct)): + pair = (distinct[i], distinct[j]) + multi_label_pair_counts[pair] = ( + multi_label_pair_counts.get(pair, 0) + 1 + ) + continue + + value = entry.get("value") + numeric = _coerce_numeric(value) + if numeric is not None: + numeric_values.append(numeric) + continue + category = _coerce_category(value) + if category is not None: + category_counts[category] = category_counts.get(category, 0) + 1 + + # ``count`` is "rows scored". For numeric / single-choice + # metrics that's the same as ``len(numeric) + sum(categories)`` + # because each scored row contributes exactly one observation. + # Multi-label parents however contribute one observation per + # selected child, so summing ``category_counts`` over-counts — + # we tracked rows-scored separately above and use it here. + rows_scored = ( + multi_label_rows_scored + if is_multi_label_parent + else len(numeric_values) + sum(category_counts.values()) + ) + + # Build numeric stats first, then categorical (both can coexist). + agg = CallImportMetricAggregate( + metric_id=metric_id_str, + metric_name=( + (meta.name if meta else observed_name) or "Unknown metric" + ), + metric_type=( + meta.metric_type if meta else observed_metric_type + ), + metric_category=( + "user_insight" + if meta is not None and _metric_is_user_insight(meta) + else "quality" + ) + or "quality", + is_multi_label_parent=is_multi_label_parent, + count=rows_scored, + skipped_count=skipped, + error_count=errored, + ) + if numeric_values: + agg.mean = float(statistics.fmean(numeric_values)) + agg.median = float(statistics.median(numeric_values)) + agg.min = min(numeric_values) + agg.max = max(numeric_values) + agg.stddev = ( + float(statistics.pstdev(numeric_values)) + if len(numeric_values) > 1 + else 0.0 + ) + agg.p25 = _percentile(numeric_values, 25) + agg.p75 = _percentile(numeric_values, 75) + agg.p95 = _percentile(numeric_values, 95) + agg.histogram_buckets = _build_histogram(numeric_values) + if category_counts: + sorted_counts = sorted( + category_counts.items(), key=lambda kv: kv[1], reverse=True + ) + agg.value_counts = [ + CallImportMetricValueCount(label=label, count=count) + for label, count in sorted_counts[:_TOP_VALUE_COUNTS] + ] + # Restrict the heatmap to pairs of labels we actually + # rendered above so the frontend never has to match + # against truncated/missing rows. Sorted desc by pair + # count to keep the most informative cells in the + # response when ``_TOP_VALUE_COUNTS`` clipped the matrix. + if is_multi_label_parent and multi_label_pair_counts: + kept_labels = { + label for label, _ in sorted_counts[:_TOP_VALUE_COUNTS] + } + pair_items = [ + (a, b, count) + for (a, b), count in multi_label_pair_counts.items() + if a in kept_labels and b in kept_labels + ] + pair_items.sort(key=lambda t: t[2], reverse=True) + agg.co_occurrence = [ + CallImportMetricLabelPair(a=a, b=b, count=count) + for a, b, count in pair_items + ] + + results.append(agg) + + # Sort so each parent metric immediately precedes its children. + # The Visualizations grid renders metrics top-to-bottom in this + # order, so multi-label parents (the "summary" chart) sit above + # the per-child boolean histograms that drill into them. Metrics + # whose ``meta`` row was deleted mid-run (``meta is None``) sink + # to the bottom but keep their relative order. + enumerated = list(enumerate(results)) + + def _sort_key(item: Tuple[int, CallImportMetricAggregate]): + original_idx, agg = item + meta = metric_meta.get(agg.metric_id) + if meta is None: + return (1, "", 1, "", original_idx) + parent_id = getattr(meta, "parent_metric_id", None) + # Group key: a child shares its parent's UUID; a parent + # uses its own UUID. Within a group, depth=0 (parent) sorts + # before depth=1 (child); ties break alphabetically by name + # so children render in a stable order regardless of which + # row scored which label first. + if parent_id is None: + group_key = str(meta.id) + depth = 0 + else: + group_key = str(parent_id) + depth = 1 + return ( + 0, + group_key, + depth, + (getattr(meta, "name", "") or "").lower(), + original_idx, + ) + + enumerated.sort(key=_sort_key) + return [agg for _idx, agg in enumerated] + + +@router.get( + "/{eval_id}/aggregate", + response_model=CallImportEvaluationAggregateResponse, + operation_id="getCallImportEvaluationAggregate", +) +async def get_call_import_evaluation_aggregate( + call_import_id: UUID, + eval_id: UUID, + baseline_evaluation_id: Optional[UUID] = Query(None), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationAggregateResponse: + """Return per-metric distributions for the Visualizations tab. + + The shape is intentionally chart-friendly: histograms for numeric + metrics, top-N value counts for categorical/text metrics, plus + summary stats (mean/p50/p95) so the UI can render summary cards + without recomputing on the client. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + eval_rows = _load_eval_rows(db, eval_id) + + metrics = _compute_metric_aggregates(db, evaluation, eval_rows) + + period_deltas: dict[str, MetricPeriodDelta] = {} + resolved_baseline_id: Optional[UUID] = None + if baseline_evaluation_id is not None: + call_import = _require_import(db, call_import_id, organization_id) + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + + rows = load_evaluation_row_pairs(db, eval_id) + period_start, _, _, _ = _report_period_from_rows(rows) + baseline_evaluation = _resolve_baseline_evaluation( + db, + organization_id, + call_import.workspace_id, + evaluation, + period_start, + str(baseline_evaluation_id), + ) + if baseline_evaluation: + resolved_baseline_id = baseline_evaluation.id + metric_aggregates_dicts = [ + _aggregate_to_dict(agg) for agg in metrics + ] + raw_deltas = _period_deltas_from_evaluation( + db, + baseline_evaluation, + metric_aggregates_dicts, + evaluation, + eval_rows, + ) + raw_deltas = _period_deltas_with_explanations( + db, + organization_id, + evaluation, + baseline_evaluation, + raw_deltas, + ) + period_deltas = { + metric_id: MetricPeriodDelta( + label=delta.get("label") or "", + detail=delta.get("detail") or "", + why=(delta.get("why") or "").strip() or None, + ) + for metric_id, delta in raw_deltas.items() + } + + _fp_stored, failure_policies_source = policies_from_evaluation_raw( + evaluation.metric_clusters + ) + return CallImportEvaluationAggregateResponse( + evaluation_id=eval_id, + total_rows=evaluation.total_rows, + completed_rows=evaluation.completed_rows, + failed_rows=evaluation.failed_rows, + metrics=metrics, + period_deltas=period_deltas, + baseline_evaluation_id=resolved_baseline_id, + failure_policies_source=failure_policies_source, + ) + + +# --------------------------------------------------------------------------- +# TLDR insights: LLM-generated narrative + bullet patterns rendered above +# the Visualizations charts. Cached on ``CallImportEvaluation.tldr_summary`` +# so the page never auto-burns LLM tokens; the user explicitly clicks +# "Generate summary" or "Regenerate" from the empty-state CTA. +# --------------------------------------------------------------------------- + + +_INSIGHTS_SYSTEM_PROMPT = ( + "You are a senior conversation-analytics reviewer. You will be " + "given aggregated metric statistics + a sample of rationales for " + "the rows of a single call-import evaluation. Identify the most " + "useful PATTERNS that hold ACROSS the calls -- not just per-metric " + "numbers. Look for combinations (e.g. `when X happens, Y also " + "tends to happen`), notable outliers, frequent failure modes, and " + "any signal that would change how a reviewer triages the run.\n\n" + "Return STRICT JSON only, with this shape and no extra keys:\n" + "{\n" + ' "narrative": "",\n' + ' "patterns": ["", "", ...],\n' + ' "metric_insights": {"": "<2-3 line business meaning>"}\n' + "}\n\n" + "Constraints:\n" + "- narrative is the ONLY text shown in the external audit summary and " + "Visualizations TLDR; keep it to at most 3 short sentences (~300 chars).\n" + "- patterns are optional supporting notes and are NOT rendered in the " + "audit summary; keep 0 to 3 bullets if supplied, each <= 120 characters.\n" + "- metric_insights must include one entry for each top-level metric id supplied.\n" + "- Each metric insight should explain what the metric means for the business and what the current distribution suggests, not restate the metric rubric.\n" + "- Avoid restating raw counts unless they reveal a pattern.\n" + "- Use neutral, factual language ('frustration appeared in...') " + "rather than judgemental ('the agents failed to...')." +) + + +def _tldr_summary_payload( + evaluation: CallImportEvaluation, +) -> Optional[EvaluationTldrSummary]: + """Return the cached TLDR (with ``is_stale`` set) or ``None``. + + ``CallImportEvaluation.tldr_summary`` is a ``JSON`` column so we + have to validate shape defensively -- a half-written or hand-edited + blob should not break the aggregate response. Returns ``None`` when + no cached summary exists. + """ + raw = evaluation.tldr_summary + if not isinstance(raw, dict): + return None + narrative = raw.get("narrative") + if not isinstance(narrative, str) or not narrative.strip(): + return None + patterns_raw = raw.get("patterns") + patterns = ( + [str(p) for p in patterns_raw if isinstance(p, str) and p.strip()] + if isinstance(patterns_raw, list) + else [] + ) + metric_insights_raw = raw.get("metric_insights") + metric_insights = ( + { + str(metric_id): str(insight).strip() + for metric_id, insight in metric_insights_raw.items() + if str(metric_id).strip() + and isinstance(insight, str) + and insight.strip() + } + if isinstance(metric_insights_raw, dict) + else {} + ) + generated_at_raw = raw.get("generated_at") + try: + generated_at = ( + datetime.fromisoformat(generated_at_raw) + if isinstance(generated_at_raw, str) + else evaluation.updated_at or datetime.now(timezone.utc) + ) + except ValueError: + generated_at = evaluation.updated_at or datetime.now(timezone.utc) + snapshot = raw.get("generated_at_completed_rows") + snapshot_int = int(snapshot) if isinstance(snapshot, (int, float)) else 0 + return EvaluationTldrSummary( + narrative=_clamp_prose_to_sentences(narrative.strip()), + patterns=patterns, + metric_insights=metric_insights, + generated_at=generated_at, + generated_at_completed_rows=snapshot_int, + provider=raw.get("provider") if isinstance(raw.get("provider"), str) else None, + model=raw.get("model") if isinstance(raw.get("model"), str) else None, + is_stale=evaluation.completed_rows > snapshot_int, + ) + + +def _sample_rationales_per_metric( + eval_rows: List[CallImportEvaluationRow], + *, + per_metric_cap: int = 3, + rationale_char_cap: int = 600, +) -> Dict[str, List[str]]: + """Collect up to ``per_metric_cap`` distinct rationales per metric. + + Distinctness is case- and whitespace-insensitive. We truncate each + rationale to ``rationale_char_cap`` so a few unusually verbose rows + can't dominate the prompt budget. Empty / non-string rationales are + skipped. + """ + out: Dict[str, List[str]] = {} + seen: Dict[str, set[str]] = {} + for row in eval_rows: + scores = row.metric_scores if isinstance(row.metric_scores, dict) else {} + for metric_id, entry in scores.items(): + if not isinstance(entry, dict): + continue + rationale = entry.get("rationale") + if not isinstance(rationale, str): + continue + text = rationale.strip() + if not text: + continue + bucket = out.setdefault(metric_id, []) + if len(bucket) >= per_metric_cap: + continue + key = " ".join(text.lower().split()) + seen_set = seen.setdefault(metric_id, set()) + if key in seen_set: + continue + seen_set.add(key) + bucket.append(text[:rationale_char_cap]) + return out + + +def _build_insights_messages( + evaluation: CallImportEvaluation, + aggregate: List[CallImportMetricAggregate], + rationale_samples: Dict[str, List[str]], + metric_meta: Dict[str, Metric], +) -> List[Dict[str, str]]: + """Render the user prompt fed to the LLM. + + The shape is plain markdown-ish text instead of JSON so the LLM can + skim it without us spending tokens on verbose schema delimiters. + Parent metrics surface their child metrics nested underneath so the + model sees the hierarchy and can talk about "X often co-occurred + with Y" rather than treating sub-labels as standalone metrics. + """ + name = evaluation.name or f"Run {str(evaluation.id)[:8]}" + lines: List[str] = [ + f"Evaluation: {name}", + ( + f"Rows: total={evaluation.total_rows} " + f"completed={evaluation.completed_rows} " + f"failed={evaluation.failed_rows}" + ), + "", + "## Per-metric aggregate", + ] + + # Group metrics by parent so the prompt mirrors the hierarchy. Any + # aggregate row whose ``metric_id`` is missing from ``metric_meta`` + # is rendered as a leaf at the top-level list (handles renamed / + # deleted parents). + children_by_parent: Dict[str, List[CallImportMetricAggregate]] = {} + top_level: List[CallImportMetricAggregate] = [] + for agg in aggregate: + meta = metric_meta.get(agg.metric_id) + parent_id = ( + str(meta.parent_metric_id) + if meta is not None and getattr(meta, "parent_metric_id", None) + else None + ) + if parent_id: + children_by_parent.setdefault(parent_id, []).append(agg) + else: + top_level.append(agg) + + def _format_metric_block(agg: CallImportMetricAggregate, indent: int) -> List[str]: + prefix = " " * indent + "- " + bits: List[str] = [f"{prefix}{agg.metric_name} [id={agg.metric_id}] (n={agg.count}"] + if agg.skipped_count: + bits.append(f", skipped={agg.skipped_count}") + if agg.error_count: + bits.append(f", errors={agg.error_count}") + bits.append(")") + meta = metric_meta.get(agg.metric_id) + description = (meta.description or "").strip() if meta else "" + if description: + bits.append(f" | definition={description[:500]}") + if agg.mean is not None: + mean_s = f"{agg.mean:.2f}" + stddev_s = f"{agg.stddev:.2f}" if agg.stddev is not None else "-" + bits.append(f" | mean={mean_s} stddev={stddev_s}") + if agg.min is not None and agg.max is not None: + bits.append(f" range=[{agg.min:.2f}, {agg.max:.2f}]") + if agg.value_counts: + total = sum(v.count for v in agg.value_counts) or 1 + top = agg.value_counts[:3] + shares = ", ".join( + f'"{v.label}"={v.count}/{total}' for v in top + ) + bits.append(f" | top={shares}") + result = ["".join(bits)] + rationales = rationale_samples.get(agg.metric_id, []) + for r in rationales: + result.append(" " * (indent + 1) + f"- rationale: {r}") + return result + + for agg in top_level: + lines.extend(_format_metric_block(agg, indent=0)) + meta = metric_meta.get(agg.metric_id) + children = children_by_parent.get(str(meta.id), []) if meta else [] + for child in children: + lines.extend(_format_metric_block(child, indent=1)) + + lines.append("") + top_level_ids = [agg.metric_id for agg in top_level] + if top_level_ids: + lines.append( + "metric_insights keys must exactly use these top-level metric ids: " + + ", ".join(top_level_ids) + ) + lines.append("") + lines.append( + "Write the JSON object as instructed. Do not include " + "preamble, code fences, or trailing commentary." + ) + + return [ + {"role": "system", "content": _INSIGHTS_SYSTEM_PROMPT}, + {"role": "user", "content": "\n".join(lines)}, + ] + + +def _parse_insights_response(text: str) -> EvaluationTldrSummary: + """Coerce the LLM response into ``narrative`` + ``patterns``. + + Matches the JSON-with-fallback pattern used by + ``app.api.v1.routes.metrics._parse_metric_generation_response``: try + ``json.loads`` first, then fall back to regex extraction of the + first ``{...}`` block. Raises ``HTTPException`` with a 502 when the + response can't be parsed at all. + """ + cleaned = (text or "").strip() + if not cleaned: + raise HTTPException( + status_code=502, detail="LLM returned an empty insights response" + ) + try: + parsed = json.loads(cleaned) + except json.JSONDecodeError: + import re + + match = re.search(r"\{.*\}", cleaned, re.DOTALL) + if not match: + raise HTTPException( + status_code=502, + detail="Could not parse LLM insights response as JSON", + ) + try: + parsed = json.loads(match.group(0)) + except json.JSONDecodeError as e: + raise HTTPException( + status_code=502, + detail=f"Could not parse LLM insights response: {e}", + ) + + if not isinstance(parsed, dict): + raise HTTPException( + status_code=502, detail="LLM insights JSON was not an object" + ) + + narrative = parsed.get("narrative") + if not isinstance(narrative, str) or not narrative.strip(): + raise HTTPException( + status_code=502, + detail="LLM insights JSON missing 'narrative' string", + ) + + patterns_raw = parsed.get("patterns") + if patterns_raw is None: + patterns: List[str] = [] + elif isinstance(patterns_raw, list): + patterns = [ + str(p).strip() + for p in patterns_raw + if isinstance(p, str) and p.strip() + ] + else: + raise HTTPException( + status_code=502, + detail="LLM insights JSON 'patterns' must be a list of strings", + ) + metric_insights_raw = parsed.get("metric_insights") + if metric_insights_raw is None: + metric_insights: Dict[str, str] = {} + elif isinstance(metric_insights_raw, dict): + metric_insights = { + str(metric_id): str(insight).strip() + for metric_id, insight in metric_insights_raw.items() + if str(metric_id).strip() + and isinstance(insight, str) + and insight.strip() + } + else: + raise HTTPException( + status_code=502, + detail="LLM insights JSON 'metric_insights' must be an object", + ) + + return EvaluationTldrSummary( + narrative=_clamp_prose_to_sentences(narrative.strip()), + patterns=patterns, + metric_insights=metric_insights, + generated_at=datetime.now(timezone.utc), + generated_at_completed_rows=0, # filled in by caller + is_stale=False, + ) + + +def _generate_and_persist_tldr_summary( + db: Session, + evaluation: CallImportEvaluation, + *, + organization_id: UUID, + provider: Optional[str] = None, + model: Optional[str] = None, +) -> EvaluationTldrSummary: + """LLM TLDR generation used by the imports-queue Celery worker.""" + eval_id = evaluation.id + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + + pairs = load_evaluation_row_pairs(db, eval_id) + eval_rows = [eval_row for eval_row, _ in pairs] + aggregate = _compute_metric_aggregates(db, evaluation, eval_rows) + if not aggregate: + raise HTTPException( + status_code=400, + detail=( + "No metric data yet. Wait for at least one row to " + "finish scoring before generating a summary." + ), + ) + + metric_ids: List[UUID] = [] + for agg in aggregate: + try: + metric_ids.append(UUID(agg.metric_id)) + except (TypeError, ValueError): + continue + metrics = _metrics_for_ids(db, organization_id, metric_ids) + metric_meta: Dict[str, Metric] = {str(m.id): m for m in metrics} + + rationale_samples = _sample_rationales_per_metric(eval_rows) + messages = _build_insights_messages( + evaluation, aggregate, rationale_samples, metric_meta + ) + + from app.services.ai.llm_resolver import get_llm_provider_and_model + from app.services.ai.llm_service import llm_service + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, provider, model + ) + + try: + llm_result = llm_service.generate_response( + messages=messages, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + temperature=0.4, + max_tokens=1400, + ) + except Exception as e: + logger.error(f"[CallImportInsights] LLM call failed: {e}") + raise HTTPException( + status_code=502, detail=f"LLM call failed: {e}" + ) from e + + summary = _parse_insights_response(llm_result.get("text", "")) + total = int(evaluation.total_rows or 0) + ui_completed = min(int(evaluation.completed_rows or 0), total) if total else int( + evaluation.completed_rows or 0 + ) + summary.generated_at_completed_rows = ui_completed + summary.provider = provider_enum.value + summary.model = model_str + summary.is_stale = False + + evaluation.tldr_summary = { + "narrative": summary.narrative, + "patterns": summary.patterns, + "metric_insights": summary.metric_insights, + "generated_at": summary.generated_at.isoformat(), + "generated_at_completed_rows": summary.generated_at_completed_rows, + "provider": summary.provider, + "model": summary.model, + } + flag_modified(evaluation, "tldr_summary") + db.commit() + db.refresh(evaluation) + return summary + + +@router.get( + "/{eval_id}/insights", + response_model=Optional[EvaluationTldrSummary], + operation_id="getCallImportEvaluationInsights", +) +async def get_call_import_evaluation_insights( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> Optional[EvaluationTldrSummary]: + """Return the cached TLDR (or ``null``) without contacting the LLM. + + Used by the Visualizations tab on first paint so the empty-state + CTA can show up before the user opts into generation. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + return _tldr_summary_payload(evaluation) + + +@router.post( + "/{eval_id}/insights", + response_model=EvaluationTldrSummary, + operation_id="generateCallImportEvaluationInsights", +) +async def generate_call_import_evaluation_insights( + call_import_id: UUID, + eval_id: UUID, + body: EvaluationInsightsRequest = Body(default_factory=EvaluationInsightsRequest), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationTldrSummary: + """Generate (or return-cached) the LLM TLDR for an evaluation run. + + Behavior: + + * ``body.regenerate=False`` and a cached summary at the current + ``completed_rows`` watermark exists -> return it as-is. + * ``body.regenerate=False`` and a stale cached summary exists + (``generated_at_completed_rows < completed_rows``) -> return it + with ``is_stale=True``; the UI prompts the user to regenerate. + * Otherwise -> resolve provider+model (auto-detect when omitted), + call the LLM, persist the new summary, return it. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + if not body.regenerate: + cached = _tldr_summary_payload(evaluation) + if cached is not None: + return cached + + # Run the TLDR LLM on the imports worker (not the default worker or API). + from app.workers.tasks.generate_evaluation_tldr_insights import ( + generate_evaluation_tldr_insights_task, + ) + + try: + task_result = generate_evaluation_tldr_insights_task.apply_async( + kwargs={ + "evaluation_id": str(eval_id), + "call_import_id": str(call_import_id), + "organization_id": str(organization_id), + "provider": body.provider, + "model": body.model, + }, + ).get(timeout=25 * 60) + except Exception as exc: + logger.error( + "[CallImportInsights] TLDR task failed for evaluation {}: {}", + eval_id, + exc, + ) + raise HTTPException( + status_code=502, + detail=f"Summary generation failed: {exc}", + ) from exc + + if isinstance(task_result, dict) and task_result.get("error"): + status_code = int(task_result.get("status_code") or 502) + raise HTTPException( + status_code=status_code, + detail=str(task_result["error"]), + ) + + summary = EvaluationTldrSummary.model_validate(task_result) + db.refresh(evaluation) + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(evaluation) + + from app.services.ai.llm_resolver import get_llm_provider_and_model + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, body.provider, body.model, body.credential_id + ) + + _enqueue_user_insights_job( + evaluation, + provider=summary.provider or provider_enum.value, + model=summary.model or model_str, + force=body.regenerate, + max_llm_calls=body.max_llm_calls, + db=db, + principal=principal, + ) + + return summary + + +def _user_insights_payload( + evaluation: CallImportEvaluation, +) -> Optional[EvaluationUserInsightsState]: + raw = getattr(evaluation, "user_insights", None) + if raw is None: + return None + return user_insights_state_from_raw( + raw, + completed_rows=evaluation.completed_rows, + ) + + +def _selected_generated_user_insights( + state: Optional[EvaluationUserInsightsState], + report_config: dict[str, Any], +) -> list[dict[str, Any]]: + """Filter and order generated insights for PDF section 03.""" + if state is None or state.status != "completed" or not state.insights: + return [] + + selected_ids = report_config.get("user_insight_ids") + if isinstance(selected_ids, list) and selected_ids: + allowed = {str(item) for item in selected_ids if item} + items = [item for item in state.insights if item.id in allowed] + else: + items = list(state.insights) + + order_raw = report_config.get("order") + order_ids: list[str] = [] + if isinstance(order_raw, dict): + user_order = order_raw.get("user_insights") + if isinstance(user_order, list): + order_ids = [str(item) for item in user_order if item] + + if order_ids: + by_id = {item.id: item for item in items} + ordered = [by_id[iid] for iid in order_ids if iid in by_id] + seen = set(order_ids) + ordered.extend(item for item in items if item.id not in seen) + items = ordered + + return [item.model_dump(mode="json") for item in items] + + +def _enqueue_user_insights_job( + evaluation: CallImportEvaluation, + *, + provider: Optional[str] = None, + model: Optional[str] = None, + force: bool = False, + max_llm_calls: Optional[int] = None, + db: Optional[Session] = None, + principal: Optional[Principal] = None, +) -> None: + """Enqueue background user-insights generation unless already running.""" + current = _user_insights_payload(evaluation) + if current is not None and current.status == "running" and not force: + return + + llm_budget = normalize_max_llm_calls(max_llm_calls) + + completed_count = ( + _count_completed_eval_rows(db, evaluation.id) + if db is not None + else evaluation.completed_rows + ) + total_calls = total_llm_calls_for_rows(completed_count, max_llm_calls=llm_budget) + evaluation.user_insights = { + "status": "running", + "insights": ( + (evaluation.user_insights or {}).get("insights", []) + if isinstance(evaluation.user_insights, dict) + else [] + ), + "generated_at": datetime.now(timezone.utc).isoformat(), + "generated_at_completed_rows": evaluation.completed_rows, + "progress": {"completed_llm_calls": 0, "total_llm_calls": total_calls}, + "provider": provider, + "model": model, + "max_llm_calls": llm_budget, + "llm_calls_used": 0, + "error_message": None, + } + if db is not None: + if principal is not None: + stamp_evaluation_actor(evaluation, principal) + flag_modified(evaluation, "user_insights") + db.commit() + + from app.workers.tasks.generate_evaluation_user_insights import ( + generate_evaluation_user_insights_task, + ) + + generate_evaluation_user_insights_task.delay( + str(evaluation.id), + provider=provider, + model=model, + max_llm_calls=llm_budget, + ) + + +@router.get( + "/{eval_id}/user-insights", + response_model=Optional[EvaluationUserInsightsState], + operation_id="getCallImportEvaluationUserInsights", +) +async def get_call_import_evaluation_user_insights( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> Optional[EvaluationUserInsightsState]: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + return _user_insights_payload(evaluation) + + +@router.post( + "/{eval_id}/user-insights", + response_model=EvaluationUserInsightsState, + operation_id="generateCallImportEvaluationUserInsights", +) +async def generate_call_import_evaluation_user_insights( + call_import_id: UUID, + eval_id: UUID, + body: EvaluationUserInsightsRequest = Body( + default_factory=EvaluationUserInsightsRequest + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationUserInsightsState: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + if not body.regenerate and not body.force: + cached = _user_insights_payload(evaluation) + if cached is not None and cached.status in {"running", "completed"}: + return cached + + eval_rows = _load_eval_rows(db, eval_id) + if not any(row.status == "completed" for row in eval_rows): + raise HTTPException( + status_code=400, + detail=( + "No completed rows yet. Wait for at least one row to " + "finish scoring before generating user insights." + ), + ) + + from app.services.ai.llm_resolver import get_llm_provider_and_model + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, body.provider, body.model, body.credential_id + ) + + _enqueue_user_insights_job( + evaluation, + provider=provider_enum.value, + model=model_str, + force=body.force or body.regenerate, + max_llm_calls=body.max_llm_calls, + db=db, + principal=principal, + ) + + db.refresh(evaluation) + return _user_insights_payload(evaluation) or EvaluationUserInsightsState( + status="running" + ) + + +def _metric_clusters_payload( + evaluation: CallImportEvaluation, +) -> Optional[EvaluationMetricClustersState]: + raw = getattr(evaluation, "metric_clusters", None) + if raw is None: + return None + return metric_clusters_state_from_raw( + raw, + completed_rows=evaluation.completed_rows, + ) + + +def _selected_metric_clusters_for_pdf( + state: Optional[EvaluationMetricClustersState], + report_config: dict[str, Any], +) -> dict[str, Any]: + if state is None or state.status != "completed": + return {} + sections = report_config.get("sections") + if isinstance(sections, dict) and sections.get("failure_diagnostics") is False: + return {} + payload: dict[str, Any] = { + "groups": [g.model_dump(mode="json") for g in state.groups], + "discovered_problems": [ + d.model_dump(mode="json") for d in state.discovered_problems + ], + } + if state.rca_summary is not None: + payload["rca_summary"] = state.rca_summary.model_dump(mode="json") + return payload + + +def _prompt_improvements_payload( + evaluation: CallImportEvaluation, +) -> Optional[EvaluationPromptImprovementsState]: + from app.services.call_import_prompt_improvements import ( + prompt_improvements_state_from_raw, + ) + + raw = getattr(evaluation, "prompt_improvements", None) + if raw is None: + return None + return prompt_improvements_state_from_raw( + raw, + completed_rows=evaluation.completed_rows, + ) + + +def _selected_prompt_improvements_for_pdf( + state: Optional[EvaluationPromptImprovementsState], + report_config: dict[str, Any], +) -> dict[str, Any]: + if state is None or state.status != "completed": + return {} + sections = report_config.get("sections") + if isinstance(sections, dict) and sections.get("prompt_improvements") is False: + return {} + return { + "imported_agent_id": state.imported_agent_id, + "imported_agent_name": state.imported_agent_name, + "overview": state.overview, + "suggestions": [s.model_dump(mode="json") for s in state.suggestions], + } + + +def _enqueue_prompt_improvements_job( + evaluation: CallImportEvaluation, + *, + imported_agent_id: UUID, + imported_agent_name: str, + provider: Optional[str] = None, + model: Optional[str] = None, + credential_id: Optional[UUID] = None, + force: bool = False, + db: Optional[Session] = None, + principal: Optional[Principal] = None, +) -> None: + current = _prompt_improvements_payload(evaluation) + if current is not None and current.status == "running" and not force: + return + + evaluation.prompt_improvements = { + "status": "running", + "imported_agent_id": str(imported_agent_id), + "imported_agent_name": imported_agent_name, + "suggestions": [], + "generated_at": datetime.now(timezone.utc).isoformat(), + "generated_at_completed_rows": evaluation.completed_rows, + "provider": provider, + "model": model, + "error_message": None, + } + if db is not None: + if principal is not None: + stamp_evaluation_actor(evaluation, principal) + flag_modified(evaluation, "prompt_improvements") + db.commit() + + from app.workers.tasks.generate_evaluation_prompt_improvements import ( + generate_evaluation_prompt_improvements_task, + ) + + async_result = generate_evaluation_prompt_improvements_task.apply_async( + kwargs={ + "evaluation_id": str(evaluation.id), + "imported_agent_id": str(imported_agent_id), + "provider": provider, + "model": model, + "credential_id": str(credential_id) if credential_id else None, + }, + queue="imports", + ) + if db is not None and isinstance(evaluation.prompt_improvements, dict): + evaluation.prompt_improvements["celery_task_id"] = async_result.id + flag_modified(evaluation, "prompt_improvements") + db.commit() + + +def _load_eval_rows(db: Session, evaluation_id: UUID) -> List[CallImportEvaluationRow]: + from app.db_sharding.eval_rows import load_evaluation_rows_for_run + + return load_evaluation_rows_for_run(db, evaluation_id) + + +def _count_completed_eval_rows(db: Session, evaluation_id: UUID) -> int: + from app.db_sharding.eval_rows import count_evaluation_rows_for_run + + return count_evaluation_rows_for_run( + db, evaluation_id, statuses=["completed"] + ) + + +def _completed_row_pairs_for_evaluation( + db: Session, + evaluation_id: UUID, +) -> List[Tuple[CallImportEvaluationRow, CallImportRow]]: + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + + row_pairs = load_evaluation_row_pairs(db, evaluation_id) + return [ + (eval_row, source_row) + for eval_row, source_row in row_pairs + if eval_row.status == "completed" + ] + + +def _resolve_metric_cluster_row_selection( + db: Session, + evaluation: CallImportEvaluation, + eval_rows: List[CallImportEvaluationRow], + evaluation_row_ids: Optional[List[UUID]], + *, + row_limit: Optional[int] = None, + policies: Optional[Dict[str, MetricFailurePolicy]] = None, +) -> Tuple[List[Tuple[CallImportEvaluationRow, CallImportRow]], List[str]]: + """Return filtered completed row pairs and the selected row id strings.""" + completed_pairs = _completed_row_pairs_for_evaluation(db, evaluation.id) + metrics = _metrics_for_clustering(db, evaluation, eval_rows) + if policies is None: + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + parent_ids = [ + m.id + for m in metrics + if getattr(m, "selection_mode", None) + and not getattr(m, "parent_metric_id", None) + ] + child_names_by_parent = _child_names_by_parent( + db, evaluation.organization_id, parent_ids + ) + policies, _ = effective_policies( + evaluation, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + eligible = list_eligible_cluster_rows( + evaluation, completed_pairs, metrics, policies + ) + eligible_ordered_ids = [str(item["evaluation_row_id"]) for item in eligible] + eligible_id_set = set(eligible_ordered_ids) + + if evaluation_row_ids is None and row_limit is not None: + selected_ids = eligible_ordered_ids[:row_limit] + filtered = filter_completed_row_pairs( + completed_pairs, + [UUID(rid) for rid in selected_ids], + ) + return filtered, selected_ids + + if evaluation_row_ids is None: + selected_ids = eligible_ordered_ids + filtered = filter_completed_row_pairs( + completed_pairs, + [UUID(rid) for rid in selected_ids], + ) + return filtered, selected_ids + + requested = {str(rid) for rid in evaluation_row_ids} + completed_id_set = {str(eval_row.id) for eval_row, _ in completed_pairs} + unknown = sorted(requested - completed_id_set) + if unknown: + raise HTTPException( + status_code=400, + detail=( + "One or more evaluation_row_ids are missing or not completed: " + + ", ".join(unknown[:5]) + + ("…" if len(unknown) > 5 else "") + ), + ) + not_eligible = sorted(requested - eligible_id_set) + if not_eligible: + raise HTTPException( + status_code=400, + detail=( + "Each selected row must have at least one flagged quality metric. " + "Ineligible row(s): " + + ", ".join(not_eligible[:5]) + + ("…" if len(not_eligible) > 5 else "") + ), + ) + selected_ids = sorted(requested) + filtered = filter_completed_row_pairs(completed_pairs, evaluation_row_ids) + return filtered, selected_ids + + +def _enqueue_metric_clusters_job( + evaluation: CallImportEvaluation, + *, + provider: Optional[str] = None, + model: Optional[str] = None, + credential_id: Optional[UUID] = None, + force: bool = False, + max_llm_calls: Optional[int] = None, + evaluation_row_ids: Optional[List[UUID]] = None, + selected_evaluation_row_ids: Optional[List[str]] = None, + failure_policies: Optional[Dict[str, MetricFailurePolicy]] = None, + db: Optional[Session] = None, + principal: Optional[Principal] = None, +) -> None: + current = _metric_clusters_payload(evaluation) + if current is not None and current.status == "running" and not force: + return + + llm_budget = normalize_max_llm_calls(max_llm_calls) + total_calls = 1 + row_ids_for_task: Optional[List[str]] = None + if db is not None: + eval_rows = _load_eval_rows(db, evaluation.id) + if selected_evaluation_row_ids is None: + _, selected_evaluation_row_ids = _resolve_metric_cluster_row_selection( + db, + evaluation, + eval_rows, + evaluation_row_ids, + ) + completed_pairs = filter_completed_row_pairs( + _completed_row_pairs_for_evaluation(db, evaluation.id), + [UUID(rid) for rid in selected_evaluation_row_ids], + ) + metrics = _metrics_for_clustering(db, evaluation, eval_rows) + policies_for_estimate = failure_policies + if policies_for_estimate is None: + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + parent_ids = [ + m.id + for m in metrics + if getattr(m, "selection_mode", None) + and not getattr(m, "parent_metric_id", None) + ] + child_names_by_parent = _child_names_by_parent( + db, evaluation.organization_id, parent_ids + ) + policies_for_estimate, _ = effective_policies( + evaluation, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + _, total_calls = estimate_metric_clusters_llm_calls( + evaluation, + metrics, + completed_pairs, + policies_for_estimate, + max_llm_calls=llm_budget, + ) + row_ids_for_task = list(selected_evaluation_row_ids) + + prior_raw = ( + evaluation.metric_clusters + if isinstance(evaluation.metric_clusters, dict) + else {} + ) + policy_blob: Dict[str, Any] = {} + if failure_policies: + policy_blob = failure_policies_to_db(failure_policies, source="user") + + evaluation.metric_clusters = { + "status": "running", + "groups": prior_raw.get("groups", []) if isinstance(prior_raw, dict) else [], + "discovered_problems": ( + prior_raw.get("discovered_problems", []) + if isinstance(prior_raw, dict) + else [] + ), + "generated_at": datetime.now(timezone.utc).isoformat(), + "generated_at_completed_rows": evaluation.completed_rows, + "progress": {"completed_llm_calls": 0, "total_llm_calls": total_calls}, + "provider": provider, + "model": model, + "max_llm_calls": llm_budget, + "llm_calls_used": 0, + "error_message": None, + "selected_evaluation_row_ids": selected_evaluation_row_ids or [], + **policy_blob, + } + if db is not None: + if principal is not None: + stamp_evaluation_actor(evaluation, principal) + flag_modified(evaluation, "metric_clusters") + db.commit() + + from app.workers.tasks.generate_evaluation_metric_clusters import ( + generate_evaluation_metric_clusters_task, + ) + + async_result = generate_evaluation_metric_clusters_task.apply_async( + kwargs={ + "evaluation_id": str(evaluation.id), + "provider": provider, + "model": model, + "credential_id": str(credential_id) if credential_id else None, + "max_llm_calls": llm_budget, + "evaluation_row_ids": row_ids_for_task, + }, + queue="imports", + ) + if db is not None and isinstance(evaluation.metric_clusters, dict): + evaluation.metric_clusters["celery_task_id"] = async_result.id + flag_modified(evaluation, "metric_clusters") + if principal is not None: + stamp_evaluation_actor(evaluation, principal) + db.commit() + + +def _revoke_metric_clusters_task(evaluation: CallImportEvaluation) -> None: + """Best-effort SIGTERM revoke of the in-flight clustering Celery task.""" + raw = evaluation.metric_clusters + if not isinstance(raw, dict): + return + task_id = str(raw.get("celery_task_id") or "").strip() + if not task_id: + return + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke(task_id, terminate=True, signal="SIGTERM") + logger.info( + "Revoked metric-clusters task {} for evaluation {}", + task_id, + evaluation.id, + ) + except Exception as exc: # noqa: BLE001 + logger.warning( + "Failed to revoke metric-clusters task {} for evaluation {}: {}", + task_id, + evaluation.id, + exc, + ) + + +def _apply_metric_clusters_cancel(evaluation: CallImportEvaluation) -> bool: + """Mark clustering as cancelled and revoke the worker task. + + Returns True if a running job was cancelled, False if already terminal. + """ + raw = evaluation.metric_clusters + if not isinstance(raw, dict): + return False + if (raw.get("status") or "").lower() != "running": + return False + + _revoke_metric_clusters_task(evaluation) + progress = raw.get("progress") if isinstance(raw.get("progress"), dict) else {} + evaluation.metric_clusters = { + **raw, + "status": "cancelled", + "error_message": METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + "progress": progress, + "celery_task_id": None, + } + return True + + +@router.get( + "/{eval_id}/metric-clusters/failure-policies", + response_model=MetricFailurePoliciesResponse, + operation_id="getCallImportEvaluationMetricClusterFailurePolicies", +) +async def get_call_import_evaluation_metric_cluster_failure_policies( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> MetricFailurePoliciesResponse: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + eval_rows = _load_eval_rows(db, eval_id) + metrics, aggregates, policies, source, child_names_by_parent = _clustering_context( + db, evaluation, eval_rows + ) + previews = build_failure_policy_previews( + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + effective=policies, + ) + updated_at = None + raw_mc = evaluation.metric_clusters + if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): + try: + updated_at = datetime.fromisoformat( + str(raw_mc["failure_policies_updated_at"]) + ) + except ValueError: + updated_at = None + return MetricFailurePoliciesResponse( + previews=previews, + policies=policies, + source=source, + updated_at=updated_at, + ) + + +@router.put( + "/{eval_id}/metric-clusters/failure-policies", + response_model=MetricFailurePoliciesResponse, + operation_id="saveCallImportEvaluationMetricClusterFailurePolicies", +) +async def save_call_import_evaluation_metric_cluster_failure_policies( + call_import_id: UUID, + eval_id: UUID, + body: MetricFailurePoliciesSaveRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> MetricFailurePoliciesResponse: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + eval_rows = _load_eval_rows(db, eval_id) + metrics, aggregates, _existing, _source, child_names_by_parent = _clustering_context( + db, evaluation, eval_rows + ) + try: + validate_failure_policies_for_metrics(body.policies, metrics) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + prior = ( + evaluation.metric_clusters + if isinstance(evaluation.metric_clusters, dict) + else {} + ) + evaluation.metric_clusters = merge_failure_policies_into_raw( + prior, + body.policies, + source="user", + ) + flag_modified(evaluation, "metric_clusters") + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(evaluation) + + policies, source = policies_from_evaluation_raw(evaluation.metric_clusters) + if source != "user": + source = "user" + previews = build_failure_policy_previews( + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + effective=policies, + ) + updated_at = None + raw_mc = evaluation.metric_clusters + if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): + try: + updated_at = datetime.fromisoformat( + str(raw_mc["failure_policies_updated_at"]) + ) + except ValueError: + updated_at = None + return MetricFailurePoliciesResponse( + previews=previews, + policies=policies, + source="user", + updated_at=updated_at, + ) + + +@router.get( + "/{eval_id}/metric-clusters/eligible-rows", + response_model=MetricClusterEligibleRowsResponse, + operation_id="listCallImportEvaluationMetricClusterEligibleRows", +) +async def list_call_import_evaluation_metric_cluster_eligible_rows( + call_import_id: UUID, + eval_id: UUID, + limit: Optional[int] = Query(default=None, ge=1), + count_only: bool = Query(default=False), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> MetricClusterEligibleRowsResponse: + """Completed rows that have at least one flagged quality metric.""" + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + eval_rows = _load_eval_rows(db, eval_id) + completed_pairs = _completed_row_pairs_for_evaluation(db, eval_id) + metrics, _aggregates, policies, _source, _child_map = _clustering_context( + db, evaluation, eval_rows + ) + all_eligible = list_eligible_cluster_rows( + evaluation, completed_pairs, metrics, policies + ) + total = len(all_eligible) + if count_only: + return MetricClusterEligibleRowsResponse(items=[], total=total) + raw_items = all_eligible if limit is None else all_eligible[:limit] + items = [MetricClusterEligibleRow.model_validate(item) for item in raw_items] + return MetricClusterEligibleRowsResponse(items=items, total=total) + + +@router.get( + "/{eval_id}/metric-clusters", + response_model=Optional[EvaluationMetricClustersState], + operation_id="getCallImportEvaluationMetricClusters", +) +async def get_call_import_evaluation_metric_clusters( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> Optional[EvaluationMetricClustersState]: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + return _metric_clusters_payload(evaluation) + + +@router.post( + "/{eval_id}/metric-clusters", + response_model=EvaluationMetricClustersState, + operation_id="generateCallImportEvaluationMetricClusters", +) +async def generate_call_import_evaluation_metric_clusters( + call_import_id: UUID, + eval_id: UUID, + body: EvaluationMetricClustersRequest = Body( + default_factory=EvaluationMetricClustersRequest + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationMetricClustersState: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + if not body.regenerate and not body.force: + cached = _metric_clusters_payload(evaluation) + if cached is not None and cached.status in {"running", "completed"}: + return cached + + eval_rows = _load_eval_rows(db, eval_id) + if not any(row.status == "completed" for row in eval_rows): + raise HTTPException( + status_code=400, + detail=( + "No completed rows yet. Wait for at least one row to " + "finish scoring before generating metric clusters." + ), + ) + + if body.evaluation_row_ids and body.row_limit is not None: + raise HTTPException( + status_code=400, + detail="Specify either evaluation_row_ids or row_limit, not both.", + ) + + if body.evaluation_row_ids: + completed_pairs = _completed_row_pairs_for_evaluation(db, evaluation.id) + completed_id_set = {str(eval_row.id) for eval_row, _ in completed_pairs} + requested = {str(rid) for rid in body.evaluation_row_ids} + unknown = sorted(requested - completed_id_set) + if unknown: + raise HTTPException( + status_code=400, + detail=( + "One or more evaluation_row_ids are missing or not completed: " + + ", ".join(unknown[:5]) + + ("…" if len(unknown) > 5 else "") + ), + ) + + from app.services.ai.llm_resolver import get_llm_provider_and_model + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, body.provider, body.model, body.credential_id + ) + + metrics, aggregates, _inferred, _source, child_names_by_parent = _clustering_context( + db, evaluation, eval_rows + ) + merged_policies = merge_clustering_policies( + body.failure_policies, + evaluation, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + try: + validate_failure_policies_for_metrics( + body.failure_policies or merged_policies, metrics + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + if not has_clusterable_metrics(metrics, merged_policies, eval_rows): + raise HTTPException( + status_code=400, + detail=( + "No calls match any failure policy. Select failure values on " + "metrics that have matching rows, or leave metrics with no " + "failures unchecked — they are skipped automatically." + ), + ) + + filtered_pairs, selected_row_ids = _resolve_metric_cluster_row_selection( + db, + evaluation, + eval_rows, + body.evaluation_row_ids, + row_limit=body.row_limit, + policies=merged_policies, + ) + if not selected_row_ids: + raise HTTPException( + status_code=400, + detail=( + "No eligible rows to cluster. Select completed calls that match " + "at least one configured failure policy." + ), + ) + if not filtered_pairs: + raise HTTPException( + status_code=400, + detail="No completed rows match the selected evaluation_row_ids.", + ) + + _enqueue_metric_clusters_job( + evaluation, + provider=provider_enum.value, + model=model_str, + credential_id=body.credential_id, + force=body.force or body.regenerate, + max_llm_calls=body.max_llm_calls, + evaluation_row_ids=body.evaluation_row_ids, + selected_evaluation_row_ids=selected_row_ids, + failure_policies=merged_policies, + db=db, + principal=principal, + ) + + db.refresh(evaluation) + return _metric_clusters_payload(evaluation) or EvaluationMetricClustersState( + status="running" + ) + + +@router.post( + "/{eval_id}/metric-clusters/cancel", + response_model=EvaluationMetricClustersState, + operation_id="cancelCallImportEvaluationMetricClusters", +) +async def cancel_call_import_evaluation_metric_clusters( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationMetricClustersState: + """Abort in-flight failure-diagnostics clustering. + + Idempotent: if clustering is not ``running``, returns the current state + unchanged. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + _apply_metric_clusters_cancel(evaluation) + flag_modified(evaluation, "metric_clusters") + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(evaluation) + + return _metric_clusters_payload(evaluation) or EvaluationMetricClustersState( + status="idle" + ) + + +@router.get( + "/{eval_id}/prompt-improvements", + response_model=Optional[EvaluationPromptImprovementsState], + operation_id="getCallImportEvaluationPromptImprovements", +) +async def get_call_import_evaluation_prompt_improvements( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> Optional[EvaluationPromptImprovementsState]: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + return _prompt_improvements_payload(evaluation) + + +@router.post( + "/{eval_id}/prompt-improvements", + response_model=EvaluationPromptImprovementsState, + operation_id="generateCallImportEvaluationPromptImprovements", +) +async def generate_call_import_evaluation_prompt_improvements( + call_import_id: UUID, + eval_id: UUID, + body: EvaluationPromptImprovementsRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationPromptImprovementsState: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + clusters = _metric_clusters_payload(evaluation) + if clusters is None or clusters.status != "completed": + raise HTTPException( + status_code=400, + detail=( + "Metric clusters must be completed before generating prompt " + "improvements. Run failure diagnostics first." + ), + ) + + from app.services.call_import_prompt_improvements import is_imported_agent + from app.services.ai.llm_resolver import get_llm_provider_and_model + + imported_agent = ( + db.query(PromptPartial) + .filter( + PromptPartial.id == body.imported_agent_id, + PromptPartial.organization_id == organization_id, + PromptPartial.workspace_id == workspace_id, + ) + .first() + ) + if imported_agent is None or not is_imported_agent(imported_agent): + raise HTTPException( + status_code=404, + detail="Imported agent not found in the active workspace", + ) + + if not body.regenerate and not body.force: + cached = _prompt_improvements_payload(evaluation) + if ( + cached is not None + and cached.status in {"running", "completed"} + and cached.imported_agent_id == str(body.imported_agent_id) + ): + return cached + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, body.provider, body.model, body.credential_id + ) + + _enqueue_prompt_improvements_job( + evaluation, + imported_agent_id=body.imported_agent_id, + imported_agent_name=imported_agent.name, + provider=provider_enum.value, + model=model_str, + credential_id=body.credential_id, + force=body.force or body.regenerate, + db=db, + principal=principal, + ) + + db.refresh(evaluation) + return _prompt_improvements_payload(evaluation) or EvaluationPromptImprovementsState( + status="running", + imported_agent_id=str(body.imported_agent_id), + imported_agent_name=imported_agent.name, + ) + + +# --------------------------------------------------------------------------- +# Flow chart: turns per-row LLM-inferred ``sequence`` arrays into a +# directed graph of (label -> label) transitions across the whole run. +# Powers the aggregate Sankey-style React Flow chart on the evaluation +# overview; per-call flow charts are built client-side from the same +# ``sequence`` field on a single row's metric_scores entry. +# --------------------------------------------------------------------------- + + +_FLOW_TERMINAL_THRESHOLD = 0.2 # Mark as terminal when >=20% of sequences end here. +_FLOW_START_NODE_ID = "__START__" +_DISCOVERED_NODE_PREFIX = "disc:" + + +def _slug_label(value: Any) -> str: + """Lowercase + whitespace-collapse + underscore-join. + + Used everywhere we need a stable key for a metric/label name — + matching the same convention the worker uses when emitting + ``sequence`` entries and discovered keys. + """ + if value is None: + return "" + return "_".join(str(value).strip().lower().split()) + + +def _resolve_alias(alias_map: Dict[str, str], key: str) -> str: + """Walk the alias map until we hit a slug that doesn't redirect. + + The merge endpoint stores ``from_slug -> to_slug`` pairs. The delete + endpoint stores ``from_slug -> ""`` (empty string sentinel) to mark + a slug as tombstoned. Chains can accumulate when the user merges + A→B and later merges B→C; this helper collapses them so callers + always land on the final canonical slug. + + Returns: + * the canonical slug if it still resolves to a real label, + * an empty string if the slug has been tombstoned (callers MUST + treat an empty result as "drop this entry entirely"), + * the input ``key`` if it isn't aliased. + + Cycles are guarded by a hard step limit since the alias map is + user-driven. + """ + if not key: + return "" + if not alias_map: + return key + current = key + seen: set[str] = set() + for _ in range(16): + if current in seen: + return current + seen.add(current) + if current not in alias_map: + return current + nxt = alias_map[current] + if nxt == current: + return current + if nxt == "": + # Deletion sentinel — the user has explicitly retired this + # slug. Propagate the empty string up so callers drop it. + return "" + current = nxt + return current + + +# Reserved JSON key under which the worker stores top-level metric +# discoveries on each row's ``metric_scores`` dict. Mirrors the constant +# in ``app/workers/tasks/helpers/llm_evaluation.py`` — kept local here to +# avoid a worker import cycle from the routes module. +DISCOVERED_METRICS_KEY = "__discovered_metrics__" + +# Allowed values for an LLM-suggested top-level metric type. Kept in +# sync with ``DiscoveredMetricSuggestedType`` in +# ``app/models/schemas.py``. +_DISCOVERED_METRIC_TYPES = ("boolean", "rating", "category") + + +def normalize_scores_with_aliases( + metric_scores: Dict[str, Any], + evaluation: CallImportEvaluation, + db: Session, + organization_id: UUID, +) -> Dict[str, Any]: + """Rewrite per-row ``metric_scores`` to honor merges + promotions. + + Called by the worker right after ``evaluate_with_llm`` returns so + every row that finishes AFTER a user has merged or promoted a + discovered label persists data already reflecting that decision. + Without this hook, a worker holding a stale prompt could re-emit a + ``from_key`` slug long after the user merged it away. + + For every parent entry (``selection_mode != null`` and a + ``discovered_labels`` / ``sequence`` field) we: + + * resolve discovered slugs through the evaluation's + ``discovered_label_aliases`` map (transitively), + * drop any discovered_labels entry whose canonical slug now + matches a real promoted child of the parent (merging them out + of the panel for free), and + * collapse adjacent duplicate sequence entries that result. + + Returns ``metric_scores`` (mutated in place) for chaining. + """ + if not isinstance(metric_scores, dict): + return metric_scores + + aliases_top = ( + evaluation.discovered_label_aliases + if isinstance(evaluation.discovered_label_aliases, dict) + else {} + ) + + # Identify the parent entries inside metric_scores. They're the + # dicts that carry a ``selection_mode`` key (set by the LLM + # hierarchy parser) and either a ``sequence`` or a + # ``discovered_labels`` list. + for key, entry in list(metric_scores.items()): + if not isinstance(entry, dict): + continue + if entry.get("type") != "category" and not entry.get("selection_mode"): + continue + try: + parent_uuid = UUID(str(key)) + except (TypeError, ValueError): + continue + + alias_map = {} + sub = aliases_top.get(str(parent_uuid)) + if isinstance(sub, dict): + alias_map = { + str(k): str(v) + for k, v in sub.items() + if isinstance(k, str) and isinstance(v, str) + } + promoted = _promoted_child_slugs(db, parent_uuid, organization_id) + + # Rewrite discovered_labels: alias-resolve keys, drop duplicates + # post-resolution, and drop entries that have been promoted. + discovered = entry.get("discovered_labels") + if isinstance(discovered, list): + kept_disc: List[Dict[str, Any]] = [] + seen: set[str] = set() + for d in discovered: + if not isinstance(d, dict): + continue + slug = _slug_label(d.get("key") or d.get("name")) + slug = _resolve_alias(alias_map, slug) + if not slug or slug in promoted or slug in seen: + continue + seen.add(slug) + new_entry = dict(d) + new_entry["key"] = slug + kept_disc.append(new_entry) + entry["discovered_labels"] = kept_disc + + # Rewrite sequence: alias-resolve every entry; collapse adjacent + # duplicates that result. We DON'T drop slugs that match + # promoted children — the promoted child slug is still a valid + # sequence entry; the flow chart will resolve it to the real + # child node. + seq = entry.get("sequence") + if isinstance(seq, list): + new_seq: List[str] = [] + last: Optional[str] = None + for item in seq: + if not isinstance(item, str): + continue + slug = _resolve_alias(alias_map, _slug_label(item)) + if not slug or slug == last: + continue + new_seq.append(slug) + last = slug + entry["sequence"] = new_seq + + # Top-level metric discoveries live alongside the parent entries + # under the reserved ``DISCOVERED_METRICS_KEY`` slot. Apply the + # flat evaluation-level alias/tombstone map + suppress slugs that + # already correspond to a real top-level Metric so workers that + # finish AFTER the user has merged / deleted / promoted can't + # resurrect a retired candidate. + discovered_metrics_payload = metric_scores.get(DISCOVERED_METRICS_KEY) + if isinstance(discovered_metrics_payload, list): + flat_alias_map = ( + evaluation.discovered_metric_aliases + if isinstance(evaluation.discovered_metric_aliases, dict) + else {} + ) + promoted_metric_slugs = _promoted_top_level_metric_slugs( + db, organization_id + ) + kept_metrics: List[Dict[str, Any]] = [] + seen_metrics: set[str] = set() + for d in discovered_metrics_payload: + if not isinstance(d, dict): + continue + slug = _slug_label(d.get("key") or d.get("name")) + slug = _resolve_alias(flat_alias_map, slug) + if ( + not slug + or slug in promoted_metric_slugs + or slug in seen_metrics + ): + continue + seen_metrics.add(slug) + new_entry = dict(d) + new_entry["key"] = slug + kept_metrics.append(new_entry) + if kept_metrics: + metric_scores[DISCOVERED_METRICS_KEY] = kept_metrics + else: + # No survivors — drop the empty array so empty-discovery rows + # keep their pre-feature payload shape. + metric_scores.pop(DISCOVERED_METRICS_KEY, None) + + return metric_scores + + +def _alias_map_for_parent( + evaluation: CallImportEvaluation, parent_metric_id: UUID +) -> Dict[str, str]: + """Pull ``{from_slug: to_slug}`` for one parent out of the eval's blob. + + Stored shape on the evaluation row is + ``{parent_id_str: {from_slug: to_slug, ...}}``. Returns an empty + dict for parents that have never had a merge applied. + """ + raw = getattr(evaluation, "discovered_label_aliases", None) + if not isinstance(raw, dict): + return {} + submap = raw.get(str(parent_metric_id)) + if not isinstance(submap, dict): + return {} + return { + str(k): str(v) + for k, v in submap.items() + if isinstance(k, str) and isinstance(v, str) + } + + +def _promoted_child_slugs( + db: Session, parent_metric_id: UUID, organization_id: UUID +) -> set[str]: + """Slugs of every real child currently sitting under the parent. + + The Discovered Labels panel hides any candidate whose slug already + matches a real child — that covers both freshly-promoted candidates + and legacy children the LLM happened to re-discover. We pull from + the live ``metrics`` table rather than the eval's + ``selected_metric_groups`` snapshot so newly-promoted children take + effect immediately, even on evaluations that ran before the + promotion. + """ + children = ( + db.query(Metric.name) + .filter( + Metric.parent_metric_id == parent_metric_id, + Metric.organization_id == organization_id, + ) + .all() + ) + out: set[str] = set() + for (name,) in children: + slug = _slug_label(name) + if slug: + out.add(slug) + return out + + +def _promoted_top_level_metric_slugs( + db: Session, organization_id: UUID +) -> set[str]: + """Slugs of every top-level (non-child) Metric in the organization. + + Used to suppress discovered-metric candidates whose slug already + matches a real standalone metric. We intentionally include both + standalone metrics AND parent category metrics — a top-level + discovery that collides with either name is a duplicate by + definition. + """ + rows = ( + db.query(Metric.name) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id.is_(None), + ) + .all() + ) + out: set[str] = set() + for (name,) in rows: + slug = _slug_label(name) + if slug: + out.add(slug) + return out + + +def _get_running_discovered_labels( + db: Session, + eval_id: UUID, + parent_metric_id: UUID, + organization_id: Optional[UUID] = None, + alias_map: Optional[Dict[str, str]] = None, +) -> List[Dict[str, Any]]: + """Slug-deduped view of every discovered label seen in this eval so far. + + Walks each ``call_import_evaluation_rows`` row's + ``metric_scores[parent_id]["discovered_labels"]`` and folds entries + that share the same slug. Returns a list ordered by descending + count and stable on label key, shaped like:: + + [{"key": "customer_on_hold", "name": "Customer put on hold", + "description": "...", "sample_rationale": "...", "count": 12}] + + Powers two callers: + * The worker prompt builder ("REUSE the existing key if it fits") + — invoked just before each row's LLM call to feed the model the + running list of previously-discovered labels in this evaluation. + * The ``/discovered-labels`` API surface used by the frontend + Discovered Labels panel to render candidates with counts + + sample rationales. + + Non-completed rows are skipped: an in-flight row's discoveries are + not yet reliable (the row could fail and never produce final + metric_scores). We accept the tradeoff that rows running + concurrently won't see each other's labels — slug-collision dedup + catches identical re-inventions, and near-paraphrases surface in + the UI panel where the user can manually merge. + """ + + parent_id_str = str(parent_metric_id) + from app.db_sharding.eval_rows import load_evaluation_rows_for_run + + eval_rows = load_evaluation_rows_for_run(db, eval_id) + rows = [ + (row.metric_scores,) + for row in eval_rows + if row.status == CallImportRowStatus.COMPLETED.value + ] + + # Suppress slugs that have either: + # * been promoted to a real child of the parent (so the panel doesn't + # keep nagging the user about a candidate they've already + # accepted), or + # * been merged INTO another slug (the "from" side of a merge) — + # those occurrences fold into the canonical target instead. + promoted_slugs: set[str] = set() + if organization_id is not None: + promoted_slugs = _promoted_child_slugs( + db, parent_metric_id, organization_id + ) + aliases = alias_map or {} + + by_key: Dict[str, Dict[str, Any]] = {} + for (scores,) in rows: + if not isinstance(scores, dict): + continue + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + continue + discovered = parent_entry.get("discovered_labels") + if not isinstance(discovered, list): + continue + for entry in discovered: + if not isinstance(entry, dict): + continue + raw_key = entry.get("key") or entry.get("name") + key = _slug_label(raw_key) + if not key: + continue + # Apply user merges + deletions first, THEN drop anything + # that ended up on a real child slug. Order matters: a + # candidate that was merged into a slug which has since + # been promoted should disappear, not show up at the + # canonical slug. An empty resolved key means the slug was + # tombstoned via the delete endpoint. + key = _resolve_alias(aliases, key) + if not key or key in promoted_slugs: + continue + name = (entry.get("name") or "").strip() or key.replace("_", " ") + description = (entry.get("description") or "").strip() or None + sample = (entry.get("rationale") or "").strip() or None + + existing = by_key.get(key) + if existing is None: + # Track up to N=3 distinct rationales per candidate so + # the Promote-to-child flow can pre-fill the new + # sub-metric's rubric with concrete LLM examples + # without the user copy-pasting from the row table. + # ``sample_rationale`` is preserved for back-compat + # with older clients; ``examples`` is the new field. + examples = [sample] if sample else [] + by_key[key] = { + "key": key, + "name": name, + "description": description, + "sample_rationale": sample, + "examples": examples, + "count": 1, + } + continue + + existing["count"] += 1 + if not existing["description"] and description: + existing["description"] = description + if not existing["sample_rationale"] and sample: + existing["sample_rationale"] = sample + # Append distinct rationales (case-insensitive trim) up + # to a small cap. Headroom is intentionally one above + # what the UI surfaces (2) so we have a backup when the + # first rationale is unhelpful. + if sample: + ex_list: List[str] = existing.setdefault("examples", []) + if len(ex_list) < 3 and not any( + s.strip().lower() == sample.strip().lower() for s in ex_list + ): + ex_list.append(sample) + + return sorted( + by_key.values(), + key=lambda item: (-item["count"], item["key"]), + ) + + +def _get_running_discovered_metrics( + db: Session, + eval_id: UUID, + organization_id: Optional[UUID] = None, + alias_map: Optional[Dict[str, str]] = None, +) -> List[Dict[str, Any]]: + """Slug-deduped view of every discovered top-level metric in this eval. + + Mirrors :func:`_get_running_discovered_labels` but is keyed at the + evaluation level (no ``parent_metric_id``). Walks each completed + row's ``metric_scores[DISCOVERED_METRICS_KEY]`` list, folds entries + that share the same slug (post-alias resolution), and suppresses + slugs that already correspond to a real top-level :class:`Metric` + in the organization. + + Each returned entry is shaped:: + + {"key": "customer_satisfaction", + "name": "Customer Satisfaction", + "description": "...", + "suggested_type": "boolean" | "rating" | "category", + "sample_rationale": "...", + "examples": ["..."], + "count": 12} + """ + + from app.db_sharding.eval_rows import load_evaluation_rows_for_run + + eval_rows = load_evaluation_rows_for_run(db, eval_id) + rows = [ + (row.metric_scores,) + for row in eval_rows + if row.status == CallImportRowStatus.COMPLETED.value + ] + + promoted_slugs: set[str] = set() + if organization_id is not None: + promoted_slugs = _promoted_top_level_metric_slugs( + db, organization_id + ) + aliases = alias_map or {} + + by_key: Dict[str, Dict[str, Any]] = {} + for (scores,) in rows: + if not isinstance(scores, dict): + continue + discovered = scores.get(DISCOVERED_METRICS_KEY) + if not isinstance(discovered, list): + continue + for entry in discovered: + if not isinstance(entry, dict): + continue + raw_key = entry.get("key") or entry.get("name") + key = _slug_label(raw_key) + if not key: + continue + # Apply user merges + deletions first, THEN drop anything + # that ended up on an already-existing top-level metric + # slug. Empty resolved key = tombstoned. + key = _resolve_alias(aliases, key) + if not key or key in promoted_slugs: + continue + name = (entry.get("name") or "").strip() or key.replace( + "_", " " + ) + description = (entry.get("description") or "").strip() or None + sample = (entry.get("rationale") or "").strip() or None + raw_type = str(entry.get("suggested_type") or "").strip().lower() + if raw_type not in _DISCOVERED_METRIC_TYPES: + raw_type = "boolean" + + existing = by_key.get(key) + if existing is None: + examples = [sample] if sample else [] + by_key[key] = { + "key": key, + "name": name, + "description": description, + "suggested_type": raw_type, + "sample_rationale": sample, + "examples": examples, + "count": 1, + } + continue + + existing["count"] += 1 + if not existing["description"] and description: + existing["description"] = description + if not existing["sample_rationale"] and sample: + existing["sample_rationale"] = sample + # Keep the most-frequently-suggested type. We don't track + # per-type frequency yet; defer to the first non-default + # type encountered when the existing entry has the default. + if existing.get("suggested_type") == "boolean" and raw_type != "boolean": + existing["suggested_type"] = raw_type + if sample: + ex_list: List[str] = existing.setdefault("examples", []) + if len(ex_list) < 3 and not any( + s.strip().lower() == sample.strip().lower() for s in ex_list + ): + ex_list.append(sample) + + return sorted( + by_key.values(), + key=lambda item: (-item["count"], item["key"]), + ) + + +def _build_flow_graph( + eval_rows: List[CallImportEvaluationRow], + parent_metric: Metric, + children: List[Metric], + alias_map: Optional[Dict[str, str]] = None, + extra_children: Optional[List[Metric]] = None, +) -> MetricFlowResponse: + """Walk per-row ``sequence`` arrays and produce aggregate nodes/edges. + + A synthetic ``START`` node is prepended to every sequence so the + diagram has a single origin. Children that never appear in any + sequence are still emitted as nodes (count=0) so the UI can render + them in the legend. + + ``alias_map`` lets callers fold merged-out discovered slugs into + their canonical target before building the graph; ``extra_children`` + are children of the parent that aren't in the legend list (e.g. + children promoted *after* the evaluation was created and therefore + missing from ``selected_metric_groups``) but should still resolve in + sequences so the slug doesn't get redrawn as a discovered candidate. + """ + parent_id_str = str(parent_metric.id) + aliases = alias_map or {} + # Build a fast lookup keyed by both the lower_snake child key (what the + # LLM emits in ``sequence``) and the child UUID (what some clients may + # store) so legacy / drifted payloads still resolve. + child_lookup: Dict[str, Metric] = {} + for child in children: + slug = _slug_label(child.name) + child_lookup[slug] = child + child_lookup[str(child.id)] = child + # ``extra_children`` are resolved-only — they shouldn't add legend + # nodes (those come from the explicit ``children`` argument), but + # they need to be in ``child_lookup`` so a sequence step that + # matches a freshly-promoted child resolves to the real child UUID + # instead of falling through to ``discovered_lookup`` and rendering + # as a "discovered" node. + if extra_children: + for child in extra_children: + slug = _slug_label(child.name) + if slug and slug not in child_lookup: + child_lookup[slug] = child + cid = str(child.id) + child_lookup.setdefault(cid, child) + + # Discovered labels: walk every row's discovered_labels first so we + # know which discovered slugs are valid before resolving sequences. + # Discovered nodes get a ``disc:`` prefixed id so they can't collide + # with real child UUIDs in the node/edge graph. We apply + # ``alias_map`` first so merged-out source slugs fold into their + # canonical target — preserving the user's "merge" intent on still- + # in-flight rows whose JSON wasn't rewritten by the merge endpoint. + discovered_lookup: Dict[str, Dict[str, Any]] = {} + for row in eval_rows: + scores = ( + row.metric_scores if isinstance(row.metric_scores, dict) else {} + ) + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + continue + raw_discovered = parent_entry.get("discovered_labels") + if not isinstance(raw_discovered, list): + continue + for entry in raw_discovered: + if not isinstance(entry, dict): + continue + slug = _slug_label(entry.get("key") or entry.get("name")) + slug = _resolve_alias(aliases, slug) + if not slug or slug in child_lookup: + continue + name = (entry.get("name") or "").strip() or slug.replace("_", " ") + existing = discovered_lookup.get(slug) + if existing is None: + discovered_lookup[slug] = { + "id": f"{_DISCOVERED_NODE_PREFIX}{slug}", + "name": name, + } + + node_counts: Dict[str, int] = {} + edge_counts: Dict[tuple[str, str], int] = {} + terminal_counts: Dict[str, int] = {} + + total_rows = len(eval_rows) + rows_with_sequence = 0 + + for row in eval_rows: + scores = ( + row.metric_scores if isinstance(row.metric_scores, dict) else {} + ) + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + continue + raw_sequence = parent_entry.get("sequence") + if not isinstance(raw_sequence, list): + continue + + resolved_ids: List[str] = [] + last_resolved: Optional[str] = None + for item in raw_sequence: + if not isinstance(item, str): + continue + normalized = _resolve_alias(aliases, _slug_label(item)) + child = child_lookup.get(normalized) or child_lookup.get(item) + if child is not None: + cid = str(child.id) + # Adjacent dedupe AFTER alias resolution so two + # different raw slugs that fold to the same target + # don't draw a self-edge through the chart. + if cid == last_resolved: + continue + resolved_ids.append(cid) + last_resolved = cid + continue + disc = discovered_lookup.get(normalized) + if disc is not None: + if disc["id"] == last_resolved: + continue + resolved_ids.append(disc["id"]) + last_resolved = disc["id"] + + if not resolved_ids: + continue + + rows_with_sequence += 1 + for nid in resolved_ids: + node_counts[nid] = node_counts.get(nid, 0) + 1 + + edge_counts[(_FLOW_START_NODE_ID, resolved_ids[0])] = ( + edge_counts.get((_FLOW_START_NODE_ID, resolved_ids[0]), 0) + 1 + ) + for src, tgt in zip(resolved_ids, resolved_ids[1:]): + if src == tgt: + continue + edge_counts[(src, tgt)] = edge_counts.get((src, tgt), 0) + 1 + + terminal_id = resolved_ids[-1] + terminal_counts[terminal_id] = terminal_counts.get(terminal_id, 0) + 1 + + nodes: List[MetricFlowNode] = [] + # Always include a START node so the UI has a stable entry point. + nodes.append( + MetricFlowNode( + id=_FLOW_START_NODE_ID, + label="Start", + count=rows_with_sequence, + is_terminal=False, + ) + ) + + def _emit_child_node(child: Metric) -> None: + cid = str(child.id) + count = node_counts.get(cid, 0) + terminal_count = terminal_counts.get(cid, 0) + is_terminal = False + if rows_with_sequence > 0: + is_terminal = ( + terminal_count / rows_with_sequence + ) >= _FLOW_TERMINAL_THRESHOLD + nodes.append( + MetricFlowNode( + id=cid, + label=child.name, + count=count, + is_terminal=is_terminal, + ) + ) + + emitted_child_ids: set[str] = set() + for child in children: + cid = str(child.id) + if cid in emitted_child_ids: + continue + emitted_child_ids.add(cid) + _emit_child_node(child) + # Extra children (promoted after the eval was created) only get + # legend nodes if they actually appear in the data — otherwise we'd + # pollute the diagram with every standalone promotion the user has + # ever made under this parent. + if extra_children: + for child in extra_children: + cid = str(child.id) + if cid in emitted_child_ids: + continue + if node_counts.get(cid, 0) == 0: + continue + emitted_child_ids.add(cid) + _emit_child_node(child) + # Append discovered nodes after the real children so legend ordering + # keeps user-defined labels first. + for slug, info in discovered_lookup.items(): + nid = info["id"] + count = node_counts.get(nid, 0) + terminal_count = terminal_counts.get(nid, 0) + is_terminal = False + if rows_with_sequence > 0: + is_terminal = ( + terminal_count / rows_with_sequence + ) >= _FLOW_TERMINAL_THRESHOLD + nodes.append( + MetricFlowNode( + id=nid, + label=info["name"], + count=count, + is_terminal=is_terminal, + is_discovered=True, + ) + ) + + edges: List[MetricFlowEdge] = [ + MetricFlowEdge(source=src, target=tgt, count=count) + for (src, tgt), count in sorted( + edge_counts.items(), key=lambda kv: kv[1], reverse=True + ) + ] + + return MetricFlowResponse( + parent_metric_id=parent_id_str, + parent_metric_name=parent_metric.name, + selection_mode=parent_metric.selection_mode, + nodes=nodes, + edges=edges, + total_rows=total_rows, + rows_with_sequence=rows_with_sequence, + ) + + +@router.get( + "/{eval_id}/flow", + response_model=MetricFlowResponse, + operation_id="getCallImportEvaluationFlow", +) +async def get_call_import_evaluation_flow( + call_import_id: UUID, + eval_id: UUID, + parent_metric_id: UUID = Query( + ..., + description=( + "Parent (category) metric whose children's sequences should be " + "aggregated into a flow graph." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> MetricFlowResponse: + """Aggregate the LLM-inferred per-row sequences into one flow graph. + + Returns ``nodes`` (one per child of the parent metric, plus a + synthetic ``START`` node) and ``edges`` (counts of consecutive + label transitions across every row that produced a sequence). The + frontend feeds this directly into a React Flow / xyflow canvas; + edge thickness should scale with ``count / total_rows`` and + ``is_terminal`` nodes should be styled as outcomes. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + parent = ( + db.query(Metric) + .filter( + Metric.id == parent_metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if not parent: + raise HTTPException( + status_code=404, + detail="Parent metric not found in this organization.", + ) + if not parent.selection_mode: + raise HTTPException( + status_code=400, + detail=( + "Flow charts are only meaningful for parent metrics " + "(selection_mode set). This metric is standalone." + ), + ) + + # Children are taken from selected_metric_groups when present so the + # flow chart reflects exactly the subset that ran in this + # evaluation; otherwise fall back to every enabled child of the + # parent. + groups_raw = ( + evaluation.selected_metric_groups + if isinstance(evaluation.selected_metric_groups, dict) + else {} + ) + parent_id_str = str(parent.id) + children: List[Metric] = [] + if parent_id_str in groups_raw and isinstance( + groups_raw[parent_id_str], list + ): + child_ids: List[UUID] = [] + for c in groups_raw[parent_id_str]: + try: + child_ids.append(UUID(str(c))) + except (TypeError, ValueError): + continue + if child_ids: + children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.id.in_(child_ids), + ) + .order_by(Metric.created_at.asc()) + .all() + ) + if not children: + children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id == parent.id, + ) + .order_by(Metric.created_at.asc()) + .all() + ) + + # Children promoted AFTER this evaluation was created aren't in + # ``selected_metric_groups`` but their slugs still appear in already- + # scored rows' sequences. Pass them as ``extra_children`` so those + # sequence entries resolve against the real (now promoted) child + # instead of being redrawn as discovered candidates. + extra_children: List[Metric] = [] + if children: + existing_ids = {child.id for child in children} + all_children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id == parent.id, + ) + .all() + ) + extra_children = [c for c in all_children if c.id not in existing_ids] + + eval_rows = _load_eval_rows(db, eval_id) + + alias_map = _alias_map_for_parent(evaluation, parent.id) + return _build_flow_graph( + eval_rows, + parent, + children, + alias_map=alias_map, + extra_children=extra_children, + ) + + +@router.get( + "/{eval_id}/discovered-labels", + response_model=DiscoveredLabelsResponse, + operation_id="getCallImportEvaluationDiscoveredLabels", +) +async def get_call_import_evaluation_discovered_labels( + call_import_id: UUID, + eval_id: UUID, + parent_metric_id: UUID = Query( + ..., + description=( + "Parent (category) metric whose LLM-discovered candidate " + "sub-labels should be aggregated across rows." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> DiscoveredLabelsResponse: + """Aggregate candidate sub-labels the LLM discovered during this eval. + + Only meaningful for parents with ``allow_discovery=true``; for other + parents we just return an empty ``items`` list rather than 400-ing + so the frontend can call the endpoint unconditionally for every + parent on the Flow tab without branching. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + parent = ( + db.query(Metric) + .filter( + Metric.id == parent_metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if not parent: + raise HTTPException( + status_code=404, + detail="Parent metric not found in this organization.", + ) + + alias_map = _alias_map_for_parent(evaluation, parent_metric_id) + items_raw = _get_running_discovered_labels( + db, + eval_id, + parent_metric_id, + organization_id=organization_id, + alias_map=alias_map, + ) + items = [DiscoveredLabelItem(**item) for item in items_raw] + return DiscoveredLabelsResponse( + parent_metric_id=str(parent.id), items=items + ) + + +@router.post( + "/{eval_id}/discovered-labels/merge", + response_model=DiscoveredLabelsResponse, + operation_id="mergeCallImportEvaluationDiscoveredLabels", +) +async def merge_call_import_evaluation_discovered_labels( + call_import_id: UUID, + eval_id: UUID, + body: DiscoveredLabelMergeRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> DiscoveredLabelsResponse: + """Rewrite every row's ``discovered_labels`` entry from from_key -> to_key. + + Idempotent — re-merging the same pair is a no-op. Discovered slugs + inside per-row ``sequence`` arrays are also rewritten so the flow + chart stays consistent with the panel. When a row already has + ``to_key`` and we're merging ``from_key`` into it, we drop the + ``from_key`` entry instead of producing two entries with the same + slug. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + parent = ( + db.query(Metric) + .filter( + Metric.id == body.parent_metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if not parent: + raise HTTPException( + status_code=404, + detail="Parent metric not found in this organization.", + ) + + from_key = _slug_label(body.from_key) + to_key = _slug_label(body.to_key) + if not from_key or not to_key: + raise HTTPException( + status_code=400, + detail="from_key and to_key must be non-empty slugs.", + ) + if from_key == to_key: + # No-op; just return the current aggregate so the client can + # refresh its view. + alias_map_existing = _alias_map_for_parent(evaluation, parent.id) + items_raw = _get_running_discovered_labels( + db, + eval_id, + body.parent_metric_id, + organization_id=organization_id, + alias_map=alias_map_existing, + ) + return DiscoveredLabelsResponse( + parent_metric_id=str(parent.id), + items=[DiscoveredLabelItem(**item) for item in items_raw], + ) + + parent_id_str = str(parent.id) + from app.db_sharding.eval_rows import foreach_evaluation_row_mutating + + def _merge_discovered_label_row(row: CallImportEvaluationRow) -> bool: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else None + ) + if not scores: + return False + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + return False + + mutated = False + discovered = parent_entry.get("discovered_labels") + if isinstance(discovered, list): + kept: List[Dict[str, Any]] = [] + existing_to = next( + ( + e + for e in discovered + if isinstance(e, dict) + and _slug_label(e.get("key") or e.get("name")) == to_key + ), + None, + ) + for entry in discovered: + if not isinstance(entry, dict): + kept.append(entry) + continue + key = _slug_label(entry.get("key") or entry.get("name")) + if key == from_key: + if existing_to is not None: + mutated = True + continue + new_entry = dict(entry) + new_entry["key"] = to_key + kept.append(new_entry) + mutated = True + else: + kept.append(entry) + if mutated: + parent_entry["discovered_labels"] = kept + + seq = parent_entry.get("sequence") + if isinstance(seq, list): + new_seq: List[str] = [] + seq_changed = False + last_added: Optional[str] = None + for item in seq: + if isinstance(item, str) and _slug_label(item) == from_key: + seq_changed = True + if last_added == to_key: + continue + new_seq.append(to_key) + last_added = to_key + else: + new_seq.append(item) + last_added = ( + _slug_label(item) if isinstance(item, str) else None + ) + if seq_changed: + parent_entry["sequence"] = new_seq + mutated = True + + if mutated: + row.metric_scores = dict(scores) + return mutated + + foreach_evaluation_row_mutating(db, eval_id, _merge_discovered_label_row) + + # Persist the merge at the evaluation level too. This is what makes + # the merge survive future scoring: rows that finish AFTER this + # call (e.g. retries, in-flight workers) will go through the + # alias map in the API surface even if the per-row JSON they + # write still mentions ``from_key``. We chain through any existing + # alias so merging A→B and then B→C resolves A→C in the panel. + raw_aliases = ( + evaluation.discovered_label_aliases + if isinstance(evaluation.discovered_label_aliases, dict) + else {} + ) + aliases_top = dict(raw_aliases) + parent_aliases = dict(aliases_top.get(parent_id_str) or {}) + # Resolve transitively: if to_key itself was previously merged into + # something else, point from_key at the canonical end-of-chain. + canonical_to = _resolve_alias(parent_aliases, to_key) + parent_aliases[from_key] = canonical_to + # Re-target any earlier aliases that pointed AT from_key — without + # this, A→B and then B→C would leave A still pointing to B (now a + # broken pointer because B is gone). Rewriting them keeps the + # alias map self-consistent. + for k, v in list(parent_aliases.items()): + if v == from_key: + parent_aliases[k] = canonical_to + aliases_top[parent_id_str] = parent_aliases + evaluation.discovered_label_aliases = aliases_top + + stamp_evaluation_actor(evaluation, principal) + db.commit() + + alias_map_after = _alias_map_for_parent(evaluation, parent.id) + items_raw = _get_running_discovered_labels( + db, + eval_id, + body.parent_metric_id, + organization_id=organization_id, + alias_map=alias_map_after, + ) + return DiscoveredLabelsResponse( + parent_metric_id=str(parent.id), + items=[DiscoveredLabelItem(**item) for item in items_raw], + ) + + +@router.post( + "/{eval_id}/discovered-labels/delete", + response_model=DiscoveredLabelsResponse, + operation_id="deleteCallImportEvaluationDiscoveredLabel", +) +async def delete_call_import_evaluation_discovered_label( + call_import_id: UUID, + eval_id: UUID, + body: DiscoveredLabelDeleteRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> DiscoveredLabelsResponse: + """Tombstone a single LLM-discovered candidate for this evaluation. + + Symmetric with the merge endpoint, but instead of redirecting the + slug at another candidate we mark it as deleted. After this call: + + * the slug is stripped from every row's + ``metric_scores[parent].discovered_labels`` list, and from + every row's ``sequence`` array (so the flow chart no longer + draws a node for it); + * the slug is recorded in + ``evaluation.discovered_label_aliases[parent][slug] = ""`` + so any worker that finishes a row AFTER this call (e.g. a row + still in flight when the user clicked Delete) silently drops + the slug instead of resurrecting it. + + Idempotent: deleting an already-deleted slug is a no-op. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + parent = ( + db.query(Metric) + .filter( + Metric.id == body.parent_metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if not parent: + raise HTTPException( + status_code=404, + detail="Parent metric not found in this organization.", + ) + + target_key = _slug_label(body.key) + if not target_key: + raise HTTPException( + status_code=400, + detail="key must be a non-empty slug.", + ) + + parent_id_str = str(parent.id) + from app.db_sharding.eval_rows import foreach_evaluation_row_mutating + + def _delete_discovered_label_row(row: CallImportEvaluationRow) -> bool: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else None + ) + if not scores: + return False + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + return False + + mutated = False + discovered = parent_entry.get("discovered_labels") + if isinstance(discovered, list): + kept = [ + e + for e in discovered + if not ( + isinstance(e, dict) + and _slug_label(e.get("key") or e.get("name")) + == target_key + ) + ] + if len(kept) != len(discovered): + parent_entry["discovered_labels"] = kept + mutated = True + + seq = parent_entry.get("sequence") + if isinstance(seq, list): + new_seq: List[str] = [] + seq_changed = False + last_added: Optional[str] = None + for item in seq: + if isinstance(item, str) and _slug_label(item) == target_key: + seq_changed = True + continue + if isinstance(item, str): + norm = _slug_label(item) + if norm == last_added: + seq_changed = True + continue + last_added = norm + new_seq.append(item) + if seq_changed: + parent_entry["sequence"] = new_seq + mutated = True + + if mutated: + row.metric_scores = dict(scores) + return mutated + + foreach_evaluation_row_mutating(db, eval_id, _delete_discovered_label_row) + + # 3. Persist the tombstone on the evaluation so workers that finish + # later don't re-surface the deleted slug. We also retarget any + # existing aliases whose ``to_key`` was the deleted slug — without + # this, a previous merge that pointed at this slug would leave a + # dangling pointer. + raw_aliases = ( + evaluation.discovered_label_aliases + if isinstance(evaluation.discovered_label_aliases, dict) + else {} + ) + aliases_top = dict(raw_aliases) + parent_aliases = dict(aliases_top.get(parent_id_str) or {}) + parent_aliases[target_key] = "" # deletion sentinel + for k, v in list(parent_aliases.items()): + if v == target_key: + parent_aliases[k] = "" + aliases_top[parent_id_str] = parent_aliases + evaluation.discovered_label_aliases = aliases_top + + stamp_evaluation_actor(evaluation, principal) + db.commit() + + alias_map_after = _alias_map_for_parent(evaluation, parent.id) + items_raw = _get_running_discovered_labels( + db, + eval_id, + body.parent_metric_id, + organization_id=organization_id, + alias_map=alias_map_after, + ) + return DiscoveredLabelsResponse( + parent_metric_id=str(parent.id), + items=[DiscoveredLabelItem(**item) for item in items_raw], + ) + + +# --------------------------------------------------------------------------- +# Discovered TOP-LEVEL METRICS (per-evaluation discovery) +# +# These endpoints are the parallel of the discovered-labels trio above but +# scoped to the evaluation as a whole instead of to a parent metric. They +# all live under ``/{eval_id}/discovered-metrics`` and operate on the +# reserved ``DISCOVERED_METRICS_KEY`` slot of each per-row +# ``metric_scores`` plus the flat ``CallImportEvaluation.discovered_metric_aliases`` +# map (no parent-id nesting). +# --------------------------------------------------------------------------- + + +def _flat_metric_aliases( + evaluation: CallImportEvaluation, +) -> Dict[str, str]: + """Pull the flat ``{from_slug: to_slug}`` map for an evaluation.""" + raw = getattr(evaluation, "discovered_metric_aliases", None) + if not isinstance(raw, dict): + return {} + return { + str(k): str(v) + for k, v in raw.items() + if isinstance(k, str) and isinstance(v, str) + } + + +@router.get( + "/{eval_id}/discovered-metrics", + response_model=DiscoveredMetricsResponse, + operation_id="getCallImportEvaluationDiscoveredMetrics", +) +async def get_call_import_evaluation_discovered_metrics( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> DiscoveredMetricsResponse: + """Aggregate top-level metric candidates the LLM discovered during this eval. + + Returns an empty ``items`` list when the evaluation did not opt + into top-level metric discovery; this keeps the frontend able to + call the endpoint unconditionally without branching on the + evaluation's ``discover_new_metrics`` flag. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + if not bool(getattr(evaluation, "discover_new_metrics", False)): + return DiscoveredMetricsResponse(evaluation_id=evaluation.id, items=[]) + + items_raw = _get_running_discovered_metrics( + db, + eval_id, + organization_id=organization_id, + alias_map=_flat_metric_aliases(evaluation), + ) + return DiscoveredMetricsResponse( + evaluation_id=evaluation.id, + items=[DiscoveredMetricItem(**item) for item in items_raw], + ) + + +@router.post( + "/{eval_id}/discovered-metrics/merge", + response_model=DiscoveredMetricsResponse, + operation_id="mergeCallImportEvaluationDiscoveredMetrics", +) +async def merge_call_import_evaluation_discovered_metrics( + call_import_id: UUID, + eval_id: UUID, + body: DiscoveredMetricMergeRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> DiscoveredMetricsResponse: + """Rewrite every row's ``__discovered_metrics__`` entry from→to. + + Mirrors the discovered-labels merge endpoint but operates on the + flat top-level metric list. Idempotent — re-merging is a no-op. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + from_key = _slug_label(body.from_key) + to_key = _slug_label(body.to_key) + if not from_key or not to_key: + raise HTTPException( + status_code=400, + detail="from_key and to_key must be non-empty slugs.", + ) + if from_key == to_key: + items_raw = _get_running_discovered_metrics( + db, + eval_id, + organization_id=organization_id, + alias_map=_flat_metric_aliases(evaluation), + ) + return DiscoveredMetricsResponse( + evaluation_id=evaluation.id, + items=[DiscoveredMetricItem(**item) for item in items_raw], + ) + + from app.db_sharding.eval_rows import foreach_evaluation_row_mutating + + def _merge_discovered_metric_row(row: CallImportEvaluationRow) -> bool: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else None + ) + if not scores: + return False + discovered = scores.get(DISCOVERED_METRICS_KEY) + if not isinstance(discovered, list): + return False + + kept: List[Dict[str, Any]] = [] + mutated = False + existing_to = next( + ( + e + for e in discovered + if isinstance(e, dict) + and _slug_label(e.get("key") or e.get("name")) == to_key + ), + None, + ) + for entry in discovered: + if not isinstance(entry, dict): + kept.append(entry) + continue + key = _slug_label(entry.get("key") or entry.get("name")) + if key == from_key: + if existing_to is not None: + mutated = True + continue + new_entry = dict(entry) + new_entry["key"] = to_key + kept.append(new_entry) + mutated = True + else: + kept.append(entry) + if mutated: + scores[DISCOVERED_METRICS_KEY] = kept + row.metric_scores = dict(scores) + return mutated + + foreach_evaluation_row_mutating(db, eval_id, _merge_discovered_metric_row) + + raw_aliases = ( + evaluation.discovered_metric_aliases + if isinstance(evaluation.discovered_metric_aliases, dict) + else {} + ) + aliases = dict(raw_aliases) + canonical_to = _resolve_alias(aliases, to_key) + aliases[from_key] = canonical_to + for k, v in list(aliases.items()): + if v == from_key: + aliases[k] = canonical_to + evaluation.discovered_metric_aliases = aliases + + stamp_evaluation_actor(evaluation, principal) + db.commit() + + items_raw = _get_running_discovered_metrics( + db, + eval_id, + organization_id=organization_id, + alias_map=_flat_metric_aliases(evaluation), + ) + return DiscoveredMetricsResponse( + evaluation_id=evaluation.id, + items=[DiscoveredMetricItem(**item) for item in items_raw], + ) + + +@router.post( + "/{eval_id}/discovered-metrics/delete", + response_model=DiscoveredMetricsResponse, + operation_id="deleteCallImportEvaluationDiscoveredMetric", +) +async def delete_call_import_evaluation_discovered_metric( + call_import_id: UUID, + eval_id: UUID, + body: DiscoveredMetricDeleteRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> DiscoveredMetricsResponse: + """Tombstone a single LLM-discovered top-level metric candidate.""" + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + target_key = _slug_label(body.key) + if not target_key: + raise HTTPException( + status_code=400, + detail="key must be a non-empty slug.", + ) + + from app.db_sharding.eval_rows import foreach_evaluation_row_mutating + + def _delete_discovered_metric_row(row: CallImportEvaluationRow) -> bool: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else None + ) + if not scores: + return False + discovered = scores.get(DISCOVERED_METRICS_KEY) + if not isinstance(discovered, list): + return False + kept = [ + e + for e in discovered + if not ( + isinstance(e, dict) + and _slug_label(e.get("key") or e.get("name")) + == target_key + ) + ] + if len(kept) == len(discovered): + return False + if kept: + scores[DISCOVERED_METRICS_KEY] = kept + else: + scores.pop(DISCOVERED_METRICS_KEY, None) + row.metric_scores = dict(scores) + return True + + foreach_evaluation_row_mutating(db, eval_id, _delete_discovered_metric_row) + + raw_aliases = ( + evaluation.discovered_metric_aliases + if isinstance(evaluation.discovered_metric_aliases, dict) + else {} + ) + aliases = dict(raw_aliases) + aliases[target_key] = "" # tombstone + for k, v in list(aliases.items()): + if v == target_key: + aliases[k] = "" + evaluation.discovered_metric_aliases = aliases + + stamp_evaluation_actor(evaluation, principal) + db.commit() + + items_raw = _get_running_discovered_metrics( + db, + eval_id, + organization_id=organization_id, + alias_map=_flat_metric_aliases(evaluation), + ) + return DiscoveredMetricsResponse( + evaluation_id=evaluation.id, + items=[DiscoveredMetricItem(**item) for item in items_raw], + ) + + +@router.delete( + "/{eval_id}/rows/{eval_row_id}", + status_code=status.HTTP_204_NO_CONTENT, + operation_id="deleteCallImportEvaluationRow", +) +async def delete_call_import_evaluation_row( + call_import_id: UUID, + eval_id: UUID, + eval_row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> Response: + """Delete a single per-row scoring entry within an evaluation run. + + Useful when the user wants to drop a noisy row before re-exporting + the CSV — e.g. a row whose audio was corrupt and skewed the + aggregate. Counters on the parent are recomputed so the rolled-up + status stays accurate. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + from app.db_sharding.eval_rows import delete_evaluation_row_on_shards + + if not delete_evaluation_row_on_shards(eval_row_id, eval_id): + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + return Response(status_code=status.HTTP_204_NO_CONTENT) + + eval_row = ( + db.query(CallImportEvaluationRow) + .filter( + CallImportEvaluationRow.id == eval_row_id, + CallImportEvaluationRow.evaluation_id == eval_id, + ) + .first() + ) + if not eval_row: + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) + + # If the row was still in flight, best-effort revoke the worker task + # so it doesn't try to write into a deleted DB row mid-execution. + if eval_row.celery_task_id and eval_row.status in {"pending", "running"}: + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke(eval_row.celery_task_id, terminate=False) + except Exception: + pass + + db.delete(eval_row) + db.flush() + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +# --------------------------------------------------------------------------- +# Retry endpoints +# --------------------------------------------------------------------------- +# +# The create endpoint enqueues every row of a fresh run; these endpoints +# let the user re-enqueue a *subset* of rows in an existing run — most +# commonly the ones that failed. We keep the worker contract identical +# (``evaluate_call_import_row_task(eval_row_id)``), so the retry path +# only has to reset row state and re-fan-out. When a row is missing its +# diarised transcript and the run was configured for diarised +# transcripts, we chain through ``transcribe_call_import_row_task`` the +# same way the create endpoint does — that's what makes "retry" feel +# like "just fix it" instead of "fail again immediately". + + +def _prepare_source_row_for_retry( + source_row: CallImportRow, + *, + transcribe_overwrite: bool, +) -> None: + """Clear stale diarisation markers so retry dispatch can re-run the pipeline.""" + source_row.celery_task_id = None + + # Re-fetch recordings when a prior import failed or stalled without S3 audio. + # Mirrors retry_failed_call_import_rows so eval retry can re-enqueue imports. + from app.api.v1.routes.call_imports import _is_stuck_pending_import_row + + if ( + source_row.status + in ( + CallImportRowStatus.FAILED, + CallImportRowStatus.PROCESSING, + ) + and not (source_row.recording_s3_key or "").strip() + ) or ( + _is_stuck_pending_import_row(source_row) + and not (source_row.recording_s3_key or "").strip() + ): + source_row.status = CallImportRowStatus.PENDING + source_row.error_message = None + source_row.attempts = 0 + + if transcribe_overwrite and (source_row.diarised_transcript or "").strip(): + source_row.diarised_transcript = None + + has_dia = bool((source_row.diarised_transcript or "").strip()) + dia_status = (source_row.diarised_transcript_status or "").strip().lower() + + if has_dia and not transcribe_overwrite: + source_row.diarised_transcript_status = "completed" + source_row.diarised_transcript_error = None + return + + if dia_status in {"failed", "pending", "running", "idle"}: + source_row.diarised_transcript_status = "idle" + source_row.diarised_transcript_error = None + + +def _reset_eval_row_for_retry( + eval_row: CallImportEvaluationRow, + *, + metric_ids: Optional[List[UUID]] = None, + skip_revoke: bool = False, +) -> None: + """Wipe per-row state so the worker can re-run it cleanly. + + Mirrors the initial state used by ``create_call_import_evaluation`` + when it first inserts a row, with the addition of revoking any + lingering Celery task id. + + When ``metric_ids`` is provided, this is a **metric-subset retry**: + only the scores for those metrics are removed from + ``metric_scores`` (other metrics' previously-computed values are + preserved so the worker's partial-merge write keeps them intact). + Otherwise the entire ``metric_scores`` dict is reset, matching the + legacy behaviour. + """ + if ( + not skip_revoke + and eval_row.celery_task_id + and eval_row.status in {"pending", "running"} + ): + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke(eval_row.celery_task_id, terminate=False) + except Exception: # noqa: BLE001 — revoke is best-effort + pass + eval_row.status = "pending" + eval_row.error_message = None + if metric_ids: + # Strip ONLY the targeted metric keys. Both string and UUID + # forms can appear in ``metric_scores`` depending on which + # code path wrote the dict, so we normalise to lower-case + # strings for the comparison. + existing = ( + eval_row.metric_scores if isinstance(eval_row.metric_scores, dict) else {} + ) + target_keys = {str(mid).lower() for mid in metric_ids} + eval_row.metric_scores = { + key: value + for key, value in existing.items() + if str(key).lower() not in target_keys + } + else: + eval_row.metric_scores = {} + eval_row.started_at = None + eval_row.finished_at = None + eval_row.celery_task_id = None + + +def _enqueue_eval_rows_with_optional_transcribe( + db: Session, + evaluation: CallImportEvaluation, + eval_rows_with_source: List[ + Tuple[CallImportEvaluationRow, CallImportRow] + ], + *, + transcribe_overwrite: bool = False, + restricted_metric_ids: Optional[List[UUID]] = None, +) -> Tuple[int, int]: + """Schedule throttled evaluation dispatch for pending eval rows. + + Returns ``(evaluate_only_count, transcribe_then_evaluate_count)`` for + logging/UI compatibility. Actual Celery fan-out is handled by + :func:`dispatch_evaluation_rows_task` under Redis fair-share limits. + """ + from app.workers.concurrency.eval_dispatch import _needs_transcribe_for_eval + from app.workers.concurrency.fair_dispatch import ( + schedule_fair_dispatch, + store_evaluation_transcribe_overwrite, + store_row_restricted_metrics, + ) + + eval_only_count = 0 + transcribe_count = 0 + if eval_rows_with_source: + for eval_row, source_row in eval_rows_with_source: + if _needs_transcribe_for_eval( + evaluation, + source_row, + transcribe_overwrite=transcribe_overwrite, + ): + transcribe_count += 1 + else: + eval_only_count += 1 + + restricted_metric_ids_str: Optional[List[str]] = ( + [str(mid) for mid in restricted_metric_ids] + if restricted_metric_ids + else None + ) + if restricted_metric_ids_str: + for eval_row, _ in eval_rows_with_source: + store_row_restricted_metrics(eval_row.id, restricted_metric_ids_str) + else: + restricted_metric_ids_str = ( + [str(mid) for mid in restricted_metric_ids] if restricted_metric_ids else None + ) + store_evaluation_transcribe_overwrite( + evaluation.id, + overwrite=transcribe_overwrite, + ) + schedule_fair_dispatch(max_workspace_turns=999) + return eval_only_count, transcribe_count + + +def _apply_telephony_retry_overrides( + db: Session, + *, + call_import: CallImport, + organization_id: UUID, + payload: CallImportEvaluationRetryRequest, +) -> None: + """Pin or clear telephony credentials on the batch for this retry pass.""" + fields_set = payload.model_fields_set + if ( + "provider" not in fields_set + and "telephony_integration_id" not in fields_set + ): + return + + from app.api.v1.routes.call_imports import ( + _resolve_telephony_integration, + _validate_telephony_credentials_live, + ) + + if payload.telephony_integration_id is not None: + integration = _resolve_telephony_integration( + db, + organization_id, + payload.telephony_integration_id, + payload.provider or "", + ) + _validate_telephony_credentials_live(db, organization_id, integration) + call_import.provider = integration.provider + call_import.telephony_integration_id = integration.id + else: + call_import.provider = None + call_import.telephony_integration_id = None + db.flush() + + +def _apply_retry_overrides( + db: Session, + evaluation: CallImportEvaluation, + organization_id: UUID, + payload: CallImportEvaluationRetryRequest, +) -> None: + """Validate + persist the LLM/STT override fields on the run. + + Mirrors the validation in ``create_call_import_evaluation`` but + only touches the fields the caller actually sent — leaving any + field ``None`` preserves the run's existing value. Raises + ``HTTPException(400)`` on bad input so the route handler can let + FastAPI turn it into a clean 400 response. + """ + # --- LLM provider + model (must be sent together) --- + if payload.llm_provider is not None or payload.llm_model is not None: + if not (payload.llm_provider and payload.llm_model): + raise HTTPException( + status_code=400, + detail=( + "Both llm_provider and llm_model are required when " + "overriding the run LLM on retry." + ), + ) + try: + evaluation.llm_provider = ModelProvider( + payload.llm_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Unknown LLM provider '{payload.llm_provider}'. " + "Valid keys are documented in ModelProvider." + ), + ) + new_model = payload.llm_model.strip() or None + if not new_model: + raise HTTPException( + status_code=400, detail="llm_model cannot be empty." + ) + evaluation.llm_model = new_model + + # --- LLM credential pin --- + if payload.llm_credential_id is not None: + cred = ( + db.query(AIProvider) + .filter( + AIProvider.id == payload.llm_credential_id, + AIProvider.organization_id == organization_id, + ) + .first() + ) + if not cred: + raise HTTPException( + status_code=400, + detail=( + "The provided llm_credential_id does not exist in " + "this organization." + ), + ) + evaluation.llm_credential_id = payload.llm_credential_id + + if payload.llm_config is not None: + evaluation.llm_config = payload.llm_config + + # --- Per-metric LLM overrides --- + # We accept the same dict shape as the create endpoint but + # constrain keys to leaf metrics that are actually in this run. + # Passing an empty dict explicitly clears existing overrides. + if payload.metric_llm_overrides is not None: + valid_leaf_ids = { + str(mid) for mid in (evaluation.selected_metric_ids or []) + } + overrides_payload: Dict[str, Dict[str, Any]] = {} + for metric_id, override in payload.metric_llm_overrides.items(): + if metric_id not in valid_leaf_ids: + raise HTTPException( + status_code=400, + detail=( + "metric_llm_overrides references metric " + f"{metric_id} which is not a leaf metric in " + "this run." + ), + ) + override_dict: Dict[str, Any] = {} + if override.provider is not None: + if not override.model: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} has a " + "provider but no model." + ), + ) + try: + override_dict["provider"] = ModelProvider( + override.provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} uses " + f"unknown provider '{override.provider}'." + ), + ) + override_dict["model"] = override.model.strip() + elif override.model: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} has a model " + "but no provider." + ), + ) + if override.credential_id is not None: + override_dict["credential_id"] = str(override.credential_id) + if override.llm_config is not None: + override_dict["llm_config"] = override.llm_config + if override_dict: + overrides_payload[metric_id] = override_dict + evaluation.metric_llm_overrides = overrides_payload or None + + # --- STT provider + model (must be sent together) --- + if payload.stt_provider is not None or payload.stt_model is not None: + if not (payload.stt_provider and payload.stt_model): + raise HTTPException( + status_code=400, + detail=( + "Both stt_provider and stt_model are required " + "when overriding the run STT on retry." + ), + ) + try: + evaluation.stt_provider = ModelProvider( + payload.stt_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Unknown STT provider '{payload.stt_provider}'.", + ) + new_stt_model = payload.stt_model.strip() or None + if not new_stt_model: + raise HTTPException( + status_code=400, detail="stt_model cannot be empty." + ) + evaluation.stt_model = new_stt_model + + # --- STT credential pin --- + if payload.stt_credential_id is not None: + evaluation.stt_credential_id = payload.stt_credential_id + + # --- LLM diariser provider + model (must be sent together) --- + if ( + payload.diarization_llm_provider is not None + or payload.diarization_llm_model is not None + ): + if not ( + payload.diarization_llm_provider + and payload.diarization_llm_model + ): + raise HTTPException( + status_code=400, + detail=( + "Both diarization_llm_provider and " + "diarization_llm_model are required when overriding " + "the run diariser on retry." + ), + ) + try: + evaluation.diarisation_llm_provider = ModelProvider( + payload.diarization_llm_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + "Unknown diarisation LLM provider " + f"'{payload.diarization_llm_provider}'." + ), + ) + new_diariser_model = ( + payload.diarization_llm_model.strip() or None + ) + if not new_diariser_model: + raise HTTPException( + status_code=400, + detail="diarization_llm_model cannot be empty.", + ) + evaluation.diarisation_llm_model = new_diariser_model + + if payload.diarization_llm_credential_id is not None: + evaluation.diarisation_llm_credential_id = ( + payload.diarization_llm_credential_id + ) + + # ``diarization_prompt`` semantics: None = leave untouched; + # empty string = clear (fall back to the canonical default at + # worker time); anything else = persist verbatim. + if payload.diarization_prompt is not None: + cleaned = payload.diarization_prompt.strip() + evaluation.diarisation_prompt = cleaned or None + + if payload.transcribe_mode is not None: + mode = payload.transcribe_mode.strip().lower() + if mode not in {"stt_llm", "llm_only"}: + raise HTTPException( + status_code=400, + detail=( + f"Unknown transcribe_mode '{payload.transcribe_mode}'. " + "Valid values are 'stt_llm' and 'llm_only'." + ), + ) + evaluation.transcribe_mode = mode + + +def _gather_retry_targets( + db: Session, + evaluation: CallImportEvaluation, + requested_ids: Optional[List[UUID]], + *, + include_completed: bool = False, +) -> Tuple[ + List[Tuple[CallImportEvaluationRow, CallImportRow]], + List[CallImportEvaluationRetrySkippedItem], +]: + """Resolve which rows to retry + reasons for any we refuse. + + When ``requested_ids`` is None we retry every row whose status is + ``failed`` (or every row when ``include_completed`` is also set — + used by the metric-subset retry path which legitimately wants to + recompute a metric on already-successful rows). When the caller + passes ids explicitly we still filter out rows that are currently + in flight; ``include_completed`` controls whether previously- + successful rows are eligible. + """ + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + from app.db_sharding.eval_rows import gather_retry_targets_sharded + + return gather_retry_targets_sharded( + db, + evaluation, + requested_ids, + include_completed=include_completed, + ) + + eval_rows_query = db.query(CallImportEvaluationRow).filter( + CallImportEvaluationRow.evaluation_id == evaluation.id + ) + + targets: List[Tuple[CallImportEvaluationRow, CallImportRow]] = [] + skipped: List[CallImportEvaluationRetrySkippedItem] = [] + + if requested_ids is None: + if include_completed: + # "Retry everything" path used by the metric-subset re-run + # UI. Still skip in-flight rows below so we don't trample + # work the worker is actively doing. + candidate_rows = eval_rows_query.filter( + CallImportEvaluationRow.status.in_(["failed", "completed"]) + ).all() + else: + candidate_rows = eval_rows_query.filter( + CallImportEvaluationRow.status == "failed" + ).all() + else: + requested_set = set(requested_ids) + candidate_rows = eval_rows_query.filter( + CallImportEvaluationRow.id.in_(requested_set) + ).all() + found_ids = {row.id for row in candidate_rows} + for missing in requested_set - found_ids: + skipped.append( + CallImportEvaluationRetrySkippedItem( + eval_row_id=missing, + reason="unknown", + ) + ) + + if not candidate_rows: + return targets, skipped + + source_row_ids = [row.call_import_row_id for row in candidate_rows] + source_rows = ( + db.query(CallImportRow) + .filter(CallImportRow.id.in_(source_row_ids)) + .all() + ) + source_by_id = {row.id: row for row in source_rows} + + for eval_row in candidate_rows: + if eval_row.status in {"pending", "running"}: + skipped.append( + CallImportEvaluationRetrySkippedItem( + eval_row_id=eval_row.id, + reason="in_progress", + ) + ) + continue + if eval_row.status == "completed" and not include_completed: + skipped.append( + CallImportEvaluationRetrySkippedItem( + eval_row_id=eval_row.id, + reason="completed", + ) + ) + continue + source_row = source_by_id.get(eval_row.call_import_row_id) + if source_row is None: + skipped.append( + CallImportEvaluationRetrySkippedItem( + eval_row_id=eval_row.id, + reason="source_row_missing", + ) + ) + continue + targets.append((eval_row, source_row)) + + return targets, skipped + + +@router.post( + "/{eval_id}/retry", + response_model=CallImportEvaluationRetryResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="retryCallImportEvaluation", +) +async def retry_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + payload: Optional[CallImportEvaluationRetryRequest] = Body(default=None), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationRetryResponse: + """Re-enqueue failed rows in an evaluation run. + + Default behavior (no body) is "retry every row that failed". Pass + ``eval_row_ids`` to scope the retry to a specific subset (e.g. the + single row a user clicked in the UI). Rows that are still + in-flight or already completed are returned in ``skipped`` rather + than re-enqueued, so this endpoint is always safe to call. + + When ``metric_ids`` is set in the payload, this is a **metric- + subset retry**: only the listed metrics are recomputed (and merged + into the row's existing ``metric_scores`` — other metrics' values + are preserved). The route auto-flips ``include_completed=True`` in + that case so previously-successful rows are eligible for re- + scoring; without it the call would no-op because every row would + be skipped as ``completed``. + + The worker contract is the same as the create endpoint: + ``evaluate_call_import_row_task(eval_row_id, [restricted_metric_ids])``. + When the run is configured for diarised transcripts and the row's + diarised transcript is missing, we chain through + ``transcribe_call_import_row_task`` first — matching the + auto-transcribe behavior of POST ``/evaluations``. + """ + del api_key + call_import = _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + requested_ids = payload.eval_row_ids if payload else None + # Metric-subset retry: validate that every metric is something this + # run actually scored. Empty list is rejected too — callers that + # want a full re-run should omit the field entirely. + # + # ``selected_metric_ids`` holds the LEAVES only (children for + # hierarchical / category metrics, standalone metrics otherwise) — + # see ``leaf_metric_ids`` in :func:`create_call_import_evaluation`. + # Parent IDs for hierarchical metrics live separately in + # ``selected_metric_groups`` (``{parent_id: [child_ids]}``) so the + # UI can reconstruct the tree without round-tripping through the + # metric table. + # + # The Re-run-metrics modal surfaces PARENTS for hierarchical + # metrics (it suppresses individual children via + # ``childrenInGroups`` in ``CallImportEvaluationDetail.tsx``), so a + # naive ``metric_ids Γèå selected_metric_ids`` check rejects every + # parent-ID request with a misleading "unknown ids" 400. We accept + # both shapes here and then EXPAND any parent IDs into + # ``{parent_id, *child_ids}`` so the downstream helpers see the + # full set of keys that need clearing + the full set of leaves + # that need re-scoring. + metric_ids: Optional[List[UUID]] = ( + payload.metric_ids if payload else None + ) + if metric_ids is not None: + if not metric_ids: + raise HTTPException( + status_code=400, + detail=( + "metric_ids must be a non-empty list. Omit the " + "field to re-run all metrics." + ), + ) + + leaf_set: Set[str] = { + str(item).lower() + for item in (evaluation.selected_metric_ids or []) + } + # ``selected_metric_groups`` is a dict ``{parent_id_str: + # [child_id_str, ...]}`` (see line ~487 in + # ``create_call_import_evaluation``). We tolerate stale data + # (string / UUID / non-dict) without crashing the retry path — + # if it's malformed we just treat it as "no parents" and fall + # back to the leaf-only check. + groups_raw = ( + evaluation.selected_metric_groups + if isinstance(evaluation.selected_metric_groups, dict) + else {} + ) + parent_to_children_str: Dict[str, List[str]] = {} + for parent_key, children_raw in groups_raw.items(): + if not isinstance(children_raw, (list, tuple)): + continue + children_norm = [ + str(c).lower() for c in children_raw if c is not None + ] + parent_to_children_str[str(parent_key).lower()] = children_norm + parent_set = set(parent_to_children_str.keys()) + + unknown = [ + mid for mid in metric_ids + if str(mid).lower() not in leaf_set + and str(mid).lower() not in parent_set + ] + if unknown: + raise HTTPException( + status_code=400, + detail=( + "metric_ids must be a subset of this evaluation's " + f"selected metrics; unknown ids: {[str(u) for u in unknown]}." + ), + ) + + # Expand parent IDs into ``{parent, *children}`` so: + # * ``_reset_eval_row_for_retry`` strips BOTH the parent + # entry (with ``chosen_child_id`` / rationale) AND every + # per-child boolean entry that the LLM evaluator wrote + # under each child's ID (see + # ``app/workers/tasks/helpers/llm_evaluation.py`` lines + # 1584 and 1649). + # * ``_enqueue_eval_rows_with_optional_transcribe`` → + # ``evaluate_call_import_row_task`` filters the work-list + # off ``selected_metric_ids`` (leaves), so we MUST hand it + # the child IDs for the parent to actually get re-scored. + # Leaves pass through unchanged. + expanded: List[UUID] = [] + seen: Set[str] = set() + for mid in metric_ids: + mid_norm = str(mid).lower() + children_str = parent_to_children_str.get(mid_norm) + if children_str is not None: + # Parent: include the parent ID itself (so the parent + # entry in ``metric_scores`` is also cleared) and all + # of its children. + candidates = [mid_norm, *children_str] + else: + candidates = [mid_norm] + for candidate in candidates: + if candidate in seen: + continue + try: + expanded.append(UUID(candidate)) + except (TypeError, ValueError): + # Defensive: skip junk values rather than 500. + continue + seen.add(candidate) + metric_ids = expanded + + # ``include_completed`` is auto-enabled when the caller asked for a + # metric subset (otherwise the metric-subset retry would always + # no-op on a green run, which is the whole reason this feature + # exists). The explicit payload flag wins for full-row retries. + include_completed = bool( + (payload.include_completed if payload else False) + or (metric_ids is not None) + ) + + transcribe_overwrite = bool( + payload.transcribe_overwrite if payload else False + ) + + skipped: List[CallImportEvaluationRetrySkippedItem] = [] + if requested_ids is None: + from app.db_sharding.eval_rows import count_evaluation_rows_for_run + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + statuses = ( + ["failed", "completed"] if include_completed else ["failed"] + ) + target_count = count_evaluation_rows_for_run( + db, eval_id, statuses=statuses + ) + else: + from sqlalchemy import func + + count_query = db.query(func.count(CallImportEvaluationRow.id)).filter( + CallImportEvaluationRow.evaluation_id == eval_id + ) + if include_completed: + count_query = count_query.filter( + CallImportEvaluationRow.status.in_(["failed", "completed"]) + ) + else: + count_query = count_query.filter( + CallImportEvaluationRow.status == "failed" + ) + target_count = int(count_query.scalar() or 0) + if target_count == 0: + return CallImportEvaluationRetryResponse( + requeued=0, + transcribe_requeued=0, + skipped=skipped, + ) + else: + targets, skipped = _gather_retry_targets( + db, + evaluation, + requested_ids, + include_completed=include_completed, + ) + if not targets: + return CallImportEvaluationRetryResponse( + requeued=0, + transcribe_requeued=0, + skipped=skipped, + ) + target_count = len(targets) + + # Apply LLM / STT overrides BEFORE enqueueing so the persisted run + # config is correct by the time the worker reads it. + if payload is not None: + _apply_retry_overrides(db, evaluation, organization_id, payload) + _apply_telephony_retry_overrides( + db, + call_import=call_import, + organization_id=organization_id, + payload=payload, + ) + + evaluation.error_message = None + evaluation.finished_at = None + evaluation.status = "running" + if not evaluation.started_at: + from datetime import datetime, timezone + + evaluation.started_at = datetime.now(timezone.utc) + + _claim_evaluation_bulk_operation(eval_id, "retry") + stamp_evaluation_actor(evaluation, principal) + db.commit() + + from app.workers.tasks.call_import_bulk_ops import ( + retry_call_import_evaluation_task, + ) + + retry_call_import_evaluation_task.delay( + str(eval_id), + { + "eval_row_ids": [str(rid) for rid in requested_ids] + if requested_ids + else None, + "metric_ids": [str(mid) for mid in metric_ids] if metric_ids else None, + "include_completed": include_completed, + "transcribe_overwrite": transcribe_overwrite, + }, + ) + + return CallImportEvaluationRetryResponse( + requeued=target_count, + transcribe_requeued=0, + skipped=skipped, + ) + + +@router.post( + "/{eval_id}/rows/{eval_row_id}/retry", + response_model=CallImportEvaluationRowResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="retryCallImportEvaluationRow", +) +async def retry_call_import_evaluation_row( + call_import_id: UUID, + eval_id: UUID, + eval_row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationRowResponse: + """Re-enqueue a single failed evaluation row. + + Convenience wrapper around ``retry_call_import_evaluation`` for the + "Retry this row" affordance in the row table. Returns the + refreshed row so the UI can update its badge immediately, without + waiting for the next polling tick. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + _require_no_evaluation_bulk_operation(eval_id) + + from app.db_sharding.eval_rows import ( + evaluation_row_session, + find_evaluation_row_in_run, + ) + from app.db_sharding.sessions import is_sharding_enabled + + eval_row, _source_stub = find_evaluation_row_in_run(db, eval_id, eval_row_id) + if eval_row is None: + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) + + if eval_row.status in {"pending", "running"}: + raise HTTPException( + status_code=409, + detail=( + "This row is still in progress — wait for it to finish " + "before retrying." + ), + ) + + targets, _ = _gather_retry_targets(db, evaluation, [eval_row.id]) + if not targets: + raise HTTPException( + status_code=409, + detail=( + "This row cannot be retried in its current state " + f"(status={eval_row.status})." + ), + ) + + if is_sharding_enabled(): + with evaluation_row_session(eval_row_id) as ( + row_db, + _catalog_db, + eval_row, + source_row, + _shard_id, + ): + _prepare_source_row_for_retry(source_row, transcribe_overwrite=False) + _reset_eval_row_for_retry(eval_row) + row_db.commit() + targets = [(eval_row, source_row)] + else: + for er, source_row in targets: + _prepare_source_row_for_retry(source_row, transcribe_overwrite=False) + _reset_eval_row_for_retry(er) + + evaluation.error_message = None + evaluation.finished_at = None + evaluation.status = "running" + if not evaluation.started_at: + from datetime import datetime, timezone + + evaluation.started_at = datetime.now(timezone.utc) + db.flush() + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + + try: + _enqueue_eval_rows_with_optional_transcribe(db, evaluation, targets) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to re-enqueue retry for evaluation row {}", eval_row_id + ) + if is_sharding_enabled(): + with evaluation_row_session(eval_row_id) as ( + row_db, + _catalog_db, + eval_row, + _source_row, + _shard_id, + ): + eval_row.status = "failed" + eval_row.error_message = f"Failed to re-enqueue retry: {exc}" + row_db.commit() + else: + eval_row.status = "failed" + eval_row.error_message = f"Failed to re-enqueue retry: {exc}" + _rollup_evaluation_status(evaluation, db) + db.commit() + raise HTTPException( + status_code=500, + detail=f"Failed to re-enqueue retry: {exc}", + ) + + if is_sharding_enabled(): + with evaluation_row_session(eval_row_id) as ( + _row_db, + _catalog_db, + eval_row, + source_row, + _shard_id, + ): + return _to_evaluation_row_response(eval_row, source_row, evaluation) + + db.refresh(eval_row) + source_row = targets[0][1] + return _to_evaluation_row_response(eval_row, source_row, evaluation) + + +from app.core.auth.capabilities import EVALS_RUN, EVALS_VIEW, REPORTS_GENERATE +from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities + +apply_workspace_route_capabilities( + router, + view_capability=EVALS_VIEW, + manage_capability=EVALS_RUN, + run_capability=EVALS_RUN, + report_capability=REPORTS_GENERATE, +) diff --git a/app/api/v1/routes/call_import_schemas.py b/app/api/v1/routes/call_import_schemas.py index 2a8877b8..86b414b8 100644 --- a/app/api/v1/routes/call_import_schemas.py +++ b/app/api/v1/routes/call_import_schemas.py @@ -7,10 +7,9 @@ parameters are mapped to source columns. Every schema MUST contain exactly one parameter with -``type='conversation_id'`` and exactly one with -``type='recording_url'``. At most one parameter each may use -``type='recording_date'`` or ``type='transcript'``. -``conversation_id`` and ``recording_url`` are forced to +``type='conversation_id'``. At most one parameter each may use +``type='recording_url'``, ``type='recording_date'``, or +``type='transcript'``. Only ``conversation_id`` is forced to ``is_required=True``. The invariant is enforced here on create + update because it spans the parent (`call_import_schemas`) and the children (`call_import_schema_parameters`) which are written in the same @@ -78,11 +77,7 @@ def _materialize_parameters( rows: List[CallImportSchemaParameter] = [] for idx, param in enumerate(payload_params): is_required = ( - param.type - in ( - CallImportParameterType.CONVERSATION_ID, - CallImportParameterType.RECORDING_URL, - ) + param.type == CallImportParameterType.CONVERSATION_ID or bool(param.is_required) ) rows.append( diff --git a/app/api/v1/routes/call_imports.py b/app/api/v1/routes/call_imports.py index 3b7bab55..7cb60d2a 100644 --- a/app/api/v1/routes/call_imports.py +++ b/app/api/v1/routes/call_imports.py @@ -1,4016 +1,4344 @@ -"""CSV-driven call import routes. - -Users upload a CSV plus a per-batch column mapping (CSV header -> system -field). The backend persists a CallImport batch + one CallImportRow per -line, then fans the rows out to the Celery ``imports`` queue where each -row is downloaded using the telephony credential pinned on the batch. -Exotel credentialed imports require a ``recording_url`` on every row; -direct-URL imports (no credential) also require a mapped recording URL. -""" - -from __future__ import annotations - -import csv -import io -import json -import re -from dataclasses import dataclass, field -from datetime import date, datetime, time, timedelta -from typing import Any, Dict, Iterable, List, Optional, Tuple -from uuid import UUID, uuid4 - -from fastapi import APIRouter, Body, BackgroundTasks, Depends, File, Form, HTTPException, Query, Response, UploadFile, status -from loguru import logger -from sqlalchemy import desc, func, or_ -from sqlalchemy.orm import Session - -from app.config import settings -from app.core.auth import Principal, get_principal -from app.core.auth.rbac import require_admin -from app.database import get_db -from app.db_sharding.sessions import is_sharding_enabled -from app.dependencies import ( - get_api_key, - get_organization_id, - get_workspace_id, - require_enterprise_feature, -) -from app.services.billing.flexprice_service import record_call_import_batch_created -from app.services.call_imports.audit import ( - actor_emails_for_call_import, - emails_for_user_ids, - stamp_call_import_actor, - user_ids_from_call_imports, -) -from app.services.call_imports.dispatch_diagnostics import ( - build_call_import_dispatch_diagnostics, -) -from app.models.database import ( - CallImport, - CallImportRow, - CallImportSchema, - CallImportSchemaParameter, - CallImportTag, - TelephonyIntegration, -) -from app.models.enums import ( - CallImportParameterType, - CallImportRowStatus, - CallImportStatus, -) -from app.models.schemas import ( - CallImportCancelDiarisationRequest, - CallImportCancelDiarisationResponse, - CallImportDetailResponse, - CallImportDeleteResponse, - CallImportDiarisationPromptDefaultResponse, - CallImportDispatchDiagnosticsResponse, - CallImportInsightsMetric, - CallImportInsightsResponse, - CallImportInsightsRunPoint, - CallImportListResponse, - CallImportMappingUpdate, - CallImportMetricAggregate, - CallImportPreviewResponse, - CallImportPreviewSheet, - CallImportRetryFailedRowsRequest, - CallImportRetryFailedRowsResponse, - CallImportResponse, - CallImportRowIdsResponse, - CallImportRowBulkDelete, - CallImportRowBulkDeleteResponse, - CallImportRowResponse, - CallImportStartRequest, - CallImportTranscribeRequest, - CallImportTranscribeResponse, - CallImportUpdate, - CallImportUploadResponse, -) - - -router = APIRouter( - prefix="/call-imports", - tags=["Call Imports"], - dependencies=[Depends(require_enterprise_feature("call_imports"))], -) - - -@dataclass(frozen=True) -class CallImportParseSkip: - """One source row excluded during CSV/Excel parse (identity / recording URL).""" - - source_row: int - reason: str - message: str - - -@dataclass -class CallImportParseResult: - rows: List[Dict[str, Any]] = field(default_factory=list) - skipped: List[CallImportParseSkip] = field(default_factory=list) - - -def parse_skips_to_json(skips: List[CallImportParseSkip]) -> List[Dict[str, Any]]: - """Persistable JSON shape for ``CallImport.source_row_skips``.""" - return [ - { - "source_row": item.source_row, - "reason": item.reason, - "message": item.message, - } - for item in skips - ] - - -def _normalize_dataset(raw: Optional[str]) -> Optional[str]: - """Trim and treat empty strings as 'no dataset' (NULL).""" - if raw is None: - return None - cleaned = raw.strip() - return cleaned or None - - -def _serialize_call_import( - db: Session, - call_import: CallImport, - *, - user_emails: Optional[Dict[UUID, str]] = None, -) -> CallImportResponse: - """Catalog parent fields; counters come from SQL rollup (not Redis merge).""" - from app.services.call_imports.bulk_ops import rollup_call_import_batch_status - from app.services.call_imports.progress_counters import ( - clear_import_progress_redis, - read_import_progress, - ) - - redis_completed, redis_failed = read_import_progress(call_import.id) - if ( - redis_completed - or redis_failed - or int(call_import.completed_rows or 0) > int(call_import.total_rows or 0) - or int(call_import.failed_rows or 0) > int(call_import.total_rows or 0) - ): - rollup_call_import_batch_status(db, call_import) - db.flush() - - clear_import_progress_redis(call_import.id) - db.refresh(call_import) - total = int(call_import.total_rows or 0) - completed = min(int(call_import.completed_rows or 0), total) if total else int( - call_import.completed_rows or 0 - ) - failed = min(int(call_import.failed_rows or 0), total) if total else int( - call_import.failed_rows or 0 - ) - if user_emails is None: - user_emails = emails_for_user_ids( - db, user_ids_from_call_imports([call_import]) - ) - created_email, updated_email = actor_emails_for_call_import( - call_import, user_emails - ) - base = CallImportResponse.model_validate(call_import) - return base.model_copy( - update={ - "completed_rows": completed, - "failed_rows": failed, - "created_by_email": created_email, - "last_updated_by_email": updated_email, - } - ) - - -def _resolve_tags( - db: Session, organization_id: UUID, tag_ids: Optional[List[UUID]] -) -> List[CallImportTag]: - """Look up tag rows by id, scoped to the organization. - - Raises HTTPException(400) if any id is unknown for the org. - """ - if not tag_ids: - return [] - rows = ( - db.query(CallImportTag) - .filter( - CallImportTag.organization_id == organization_id, - CallImportTag.id.in_(tag_ids), - ) - .all() - ) - found_ids = {row.id for row in rows} - missing = [str(tag_id) for tag_id in tag_ids if tag_id not in found_ids] - if missing: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Unknown call_import_tag id(s): {missing}", - ) - return rows - - -MAX_UPLOAD_BYTES = 15 * 1024 * 1024 # 15 MB upload cap (CSV or Excel) - -# File extensions accepted by the upload + preview endpoints. Keep in -# lockstep with the frontend ``accept`` attribute on the file picker. -CSV_EXTENSIONS = (".csv",) -XLSX_EXTENSIONS = (".xlsx", ".xlsm") -ALLOWED_EXTENSIONS = CSV_EXTENSIONS + XLSX_EXTENSIONS - -AUDIO_CONTENT_TYPES = { - "wav": "audio/wav", - "mp3": "audio/mpeg", - "flac": "audio/flac", - "m4a": "audio/mp4", -} - - -def _file_format(filename: Optional[str]) -> Optional[str]: - """Classify ``filename`` as ``'csv'`` / ``'xlsx'`` or ``None`` if unsupported.""" - if not filename: - return None - name = filename.lower() - if name.endswith(CSV_EXTENSIONS): - return "csv" - if name.endswith(XLSX_EXTENSIONS): - return "xlsx" - return None - - -def _audio_extension(filename: Optional[str]) -> Optional[str]: - """Return the validated lower-case extension for a manual recording.""" - if not filename or "." not in filename: - return None - ext = filename.rsplit(".", 1)[-1].lower().strip() - allowed = {fmt.lower().lstrip(".") for fmt in settings.ALLOWED_AUDIO_FORMATS} - return ext if ext in allowed else None - - -def _audio_content_type(ext: str, upload_content_type: Optional[str]) -> str: - """Prefer the browser-supplied audio content type, with a safe fallback.""" - supplied = (upload_content_type or "").strip() - if supplied and supplied != "application/octet-stream": - return supplied - return AUDIO_CONTENT_TYPES.get(ext.lower(), "application/octet-stream") - - -def _audio_s3_key( - organization_id: UUID, call_import_id: UUID, row_id: UUID, ext: str -) -> str: - """Build the canonical S3 key for a manually uploaded recording.""" - from app.services.storage.s3_service import s3_service - - return ( - f"{s3_service.prefix}organizations/{organization_id}/call_imports/" - f"{call_import_id}/{row_id}.{ext}" - ) - - -def _filename_stem(filename: Optional[str]) -> str: - """Extract a cross-platform filename stem from an UploadFile name.""" - raw = (filename or "").strip() - basename = re.split(r"[\\/]", raw)[-1] if raw else "" - if "." in basename: - basename = basename.rsplit(".", 1)[0] - return basename.strip() - - -def _sanitize_conversation_id(raw: str) -> str: - """Turn a filename stem into a stable conversation_id.""" - cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", raw.strip()) - cleaned = re.sub(r"_+", "_", cleaned).strip("._-") - return (cleaned or "recording")[:255] - - -def _dedupe_conversation_id( - base: str, counts: Dict[str, int] -) -> str: - """Make conversation ids unique within one manual upload batch.""" - count = counts.get(base, 0) + 1 - counts[base] = count - if count == 1: - return base - suffix = f"-{count}" - return f"{base[: 255 - len(suffix)]}{suffix}" - - -def _normalize_header(name: str) -> str: - return (name or "").strip().lower() - - -def _header_lookup(fieldnames: List[str]) -> Dict[str, str]: - """Map normalized header -> original header for case-insensitive lookup.""" - return {_normalize_header(h): h for h in fieldnames or []} - - -def _resolve_mapped_header( - mapping_value: Optional[str], header_lookup: Dict[str, str] -) -> Optional[str]: - """Translate a user-supplied CSV header into the actual column key. - - The frontend sends headers exactly as they appear in the source file, - but we still normalize on the server so trailing whitespace / casing - doesn't break matching. Returns the canonical fieldname or ``None`` - if not present in the file. - """ - if not mapping_value: - return None - return header_lookup.get(_normalize_header(mapping_value)) - - -def _xlsx_cell_to_str(value: Any) -> str: - """Coerce an openpyxl cell value to the string the rest of the - pipeline expects. - - openpyxl returns native Python types (int, float, datetime, bool, - None). The CSV path always works with strings, so we mirror that: - integers stringify cleanly (no ``.0`` suffix on whole-number floats), - datetimes use ISO-8601, booleans use SQL-style ``TRUE`` / ``FALSE``. - """ - if value is None: - return "" - if isinstance(value, bool): - return "TRUE" if value else "FALSE" - if isinstance(value, int): - return str(value) - if isinstance(value, float): - if value.is_integer(): - return str(int(value)) - return str(value) - if isinstance(value, datetime): - return value.isoformat() - if isinstance(value, date): - return value.isoformat() - if isinstance(value, time): - return value.isoformat() - if isinstance(value, timedelta): - return str(value) - return str(value) - - -def _parse_recording_date_cell(cell: str) -> date: - """Parse day-first dates with one/two digit day-month parts.""" - match = re.fullmatch(r"\s*(\d{1,2})[/-](\d{1,2})[/-](\d{4})\s*", cell) - if match: - day, month, year = (int(part) for part in match.groups()) - return date(year, month, day) - - # Native Excel date cells arrive from ``_xlsx_cell_to_str`` as ISO - # datetimes (e.g. ``2026-01-04T00:00:00``). Accept that resolved date, - # while keeping plain ISO dates rejected for hand-entered text/CSV cells. - if "T" in cell: - return datetime.fromisoformat(cell.replace("Z", "+00:00")).date() - - raise ValueError("expected D/M/YYYY or D-M-YYYY") - - -def _coerce_parameter_value( - raw: str, - param_type: CallImportParameterType, - *, - row_idx: int, - param_name: str, -) -> Any: - """Validate + coerce a single CSV cell against its declared type. - - Returns the typed Python value to surface in ``raw_columns``. Empty - strings are returned as ``None`` regardless of the parameter type so - optional cells stay null end-to-end. Coercion failures raise a - 400 with a row-anchored message. - """ - cell = (raw or "").strip() - if not cell: - return None - - if param_type == CallImportParameterType.CONVERSATION_ID: - return cell - if param_type == CallImportParameterType.RECORDING_URL: - # Recording URLs are exercised by the worker (which downloads - # them); we only do a light "starts with http" check here so a - # paste-error surfaces immediately at upload time. - lower = cell.lower() - if not (lower.startswith("http://") or lower.startswith("https://")): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - "valid recording URL (must start with http:// or https://)." - ), - ) - return cell - if param_type == CallImportParameterType.RECORDING_DATE: - try: - parsed_date = _parse_recording_date_cell(cell) - except ValueError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - f"valid recording date ({cell!r}); expected day-first " - "D/M/YYYY or D-M-YYYY." - ), - ) - return parsed_date.strftime("%d/%m/%Y") - if param_type == CallImportParameterType.TRANSCRIPT: - return cell - if param_type == CallImportParameterType.TEXT: - return cell - if param_type == CallImportParameterType.NUMBER: - try: - value = float(cell) - except ValueError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - f"valid number ({cell!r})." - ), - ) - if value.is_integer(): - return int(value) - return value - if param_type == CallImportParameterType.BOOLEAN: - truthy = {"true", "yes", "y", "1", "t"} - falsy = {"false", "no", "n", "0", "f"} - norm = cell.lower() - if norm in truthy: - return True - if norm in falsy: - return False - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - f"valid boolean ({cell!r})." - ), - ) - if param_type == CallImportParameterType.DATETIME: - try: - parsed = datetime.fromisoformat(cell.replace("Z", "+00:00")) - except ValueError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - f"valid ISO-8601 date/time ({cell!r})." - ), - ) - return parsed.isoformat() - if param_type == CallImportParameterType.URL: - lower = cell.lower() - if not (lower.startswith("http://") or lower.startswith("https://")): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - "valid URL (must start with http:// or https://)." - ), - ) - return cell - # Unknown types: store as text and let the next migration catch up. - return cell - - -def _recording_url_cell_is_valid_http(raw: str) -> bool: - cell = (raw or "").strip() - if not cell: - return False - lower = cell.lower() - return lower.startswith("http://") or lower.startswith("https://") - - -def _parameter_is_required(param: CallImportSchemaParameter) -> bool: - """Return whether a schema parameter must be mapped on every upload.""" - if param.is_required: - return True - try: - param_type = CallImportParameterType(param.type) - except ValueError: - return False - return param_type in ( - CallImportParameterType.CONVERSATION_ID, - CallImportParameterType.RECORDING_URL, - ) - - -def _apply_schema_mapping( - fieldnames: List[str], - rows_iter: Iterable[Dict[str, str]], - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, str], - skipped_columns: List[str], - *, - source_label: str = "CSV", - validate_only: bool = False, -) -> CallImportParseResult: - """Schema-driven row projection: parameter -> CSV header -> typed value. - - Validates that every required schema parameter is mapped to a CSV - header that actually exists in the file, and that every CSV header - is either mapped to a parameter or explicitly listed in - ``skipped_columns``. Returns one dict per non-empty data row with: - - * ``conversation_id`` (str, mandatory) - * ``recording_date`` (Optional[str], DD/MM/YYYY date) - * ``recording_url`` (Optional[str]) - * ``transcript`` (Optional[str]) - * ``parameter_values`` (Dict[str, Any]) of typed values keyed by - parameter name (drives ``raw_columns`` so the export can - reproduce the source). - - ``validate_only=True`` runs the header / mapping / skipped-column - checks (every check that doesn't need to read row data) and then - returns an empty list — used by the MAP stage to validate a - mapping payload against the cached sheet snapshot without - re-fetching the source bytes from S3. - """ - header_lookup = _header_lookup(list(fieldnames)) - - # 1. Look up the conversation_id parameter so we can address it - # directly while building each row. - conv_param = next( - (p for p in parameters if p.type == CallImportParameterType.CONVERSATION_ID), - None, - ) - if conv_param is None: - # The schema invariant should have caught this on create/update, - # but a defensive 400 here keeps us safe against hand-rolled - # API callers that bypassed validation. - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Selected schema is missing the mandatory conversation_id parameter.", - ) - # 2. Resolve every mapped parameter to a canonical fieldname. - # Required parameters MUST resolve; optional ones may resolve to - # None if the user left them blank (no mapping). - canonical_by_param: Dict[str, Optional[str]] = {} - recording_date_param_name: Optional[str] = None - rec_url_param_name: Optional[str] = None - transcript_param_name: Optional[str] = None - for param in parameters: - mapped_header = parameter_mapping.get(param.name) - canonical = ( - _resolve_mapped_header(mapped_header, header_lookup) - if mapped_header - else None - ) - if _parameter_is_required(param) and canonical is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"{source_label} does not contain the column " - f"'{mapped_header or ''}' mapped to required parameter " - f"'{param.name}'." - ), - ) - canonical_by_param[param.name] = canonical - if param.type == CallImportParameterType.RECORDING_DATE: - recording_date_param_name = param.name - elif param.type == CallImportParameterType.RECORDING_URL: - rec_url_param_name = param.name - elif param.type == CallImportParameterType.TRANSCRIPT: - transcript_param_name = param.name - - # 3. Every CSV column must either be mapped to a parameter or - # explicitly skipped. Catches "I forgot to skip the email - # column" gracefully instead of dropping data silently. - mapped_canonicals = {c for c in canonical_by_param.values() if c} - skipped_canonicals = { - _resolve_mapped_header(h, header_lookup) - for h in skipped_columns - } - skipped_canonicals.discard(None) - unhandled = [ - h - for h in fieldnames - if h not in mapped_canonicals and h not in skipped_canonicals - ] - if unhandled: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"{source_label} columns must either be mapped to a schema " - f"parameter or explicitly skipped. Unhandled: {unhandled}." - ), - ) - - conv_canonical = canonical_by_param[conv_param.name] - rec_canonical = ( - canonical_by_param.get(rec_url_param_name) - if rec_url_param_name - else None - ) - recording_date_canonical = ( - canonical_by_param.get(recording_date_param_name) - if recording_date_param_name - else None - ) - transcript_canonical = ( - canonical_by_param.get(transcript_param_name) - if transcript_param_name - else None - ) - - if validate_only: - # MAP-stage validation: every header check above has already - # run; the row loop only matters at IMPORT time. Skip it (and - # the "no data rows" guard at the bottom of the function) so - # the caller gets a clean pass when the mapping is shaped right. - return CallImportParseResult() - - parsed: List[Dict[str, Any]] = [] - skipped: List[CallImportParseSkip] = [] - for idx, row in enumerate(rows_iter): - # Drop fully-blank lines - matches the legacy parser behavior so - # trailing-newline edge cases don't fail an otherwise-good upload. - non_blank = any( - (row.get(c) or "").strip() - for c in mapped_canonicals - if c - ) - if not non_blank: - continue - - source_row = idx + 1 - conv_value = (row.get(conv_canonical) or "").strip() if conv_canonical else "" - if not conv_value: - skipped.append( - CallImportParseSkip( - source_row=source_row, - reason="missing_conversation_id", - message=( - f"Row {source_row} is missing the '{conv_param.name}' " - "(conversation_id) value." - ), - ) - ) - continue - - if rec_canonical and rec_url_param_name: - rec_param = next( - (p for p in parameters if p.name == rec_url_param_name), - None, - ) - if rec_param is not None and _parameter_is_required(rec_param): - rec_raw = (row.get(rec_canonical) or "").strip() - if not rec_raw: - skipped.append( - CallImportParseSkip( - source_row=source_row, - reason="missing_recording_url", - message=( - f"Row {source_row} is missing the required " - f"'{rec_url_param_name}' value." - ), - ) - ) - continue - if not _recording_url_cell_is_valid_http(rec_raw): - skipped.append( - CallImportParseSkip( - source_row=source_row, - reason="invalid_recording_url", - message=( - f"Row {source_row}: value for " - f"'{rec_url_param_name}' is not a valid recording " - "URL (must start with http:// or https://)." - ), - ) - ) - continue - - # Materialize every mapped parameter into the per-row snapshot, - # running per-type coercion so a bad cell aborts the upload - # rather than silently storing garbage. - parameter_values: Dict[str, Any] = {} - row_skipped = False - for param in parameters: - canonical = canonical_by_param[param.name] - if canonical is None: - continue - try: - param_type = CallImportParameterType(param.type) - except ValueError: - param_type = CallImportParameterType.TEXT - coerced = _coerce_parameter_value( - row.get(canonical) or "", - param_type, - row_idx=idx, - param_name=param.name, - ) - if _parameter_is_required(param) and coerced is None: - if param_type == CallImportParameterType.RECORDING_URL: - skipped.append( - CallImportParseSkip( - source_row=source_row, - reason="missing_recording_url", - message=( - f"Row {source_row} is missing the required " - f"'{param.name}' value." - ), - ) - ) - row_skipped = True - break - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {source_row} is missing the required " - f"'{param.name}' value." - ), - ) - parameter_values[param.name] = coerced - if row_skipped: - continue - - rec_value = ( - (row.get(rec_canonical) or "").strip() if rec_canonical else "" - ) - transcript_value = ( - (row.get(transcript_canonical) or "").strip() - if transcript_canonical - else "" - ) - recording_date_value = ( - parameter_values.get(recording_date_param_name) - if recording_date_param_name - else None - ) - - parsed.append( - { - "conversation_id": conv_value, - "recording_date": recording_date_value, - "recording_url": rec_value or None, - "transcript": transcript_value or None, - "parameter_values": parameter_values, - } - ) - - if not parsed and not skipped: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"{source_label} did not contain any data rows.", - ) - - return CallImportParseResult(rows=parsed, skipped=skipped) - - -def _raise_if_no_importable_rows( - result: CallImportParseResult, *, source_label: str = "CSV" -) -> None: - """Sync upload / API callers fail fast when every data row was skipped.""" - if result.rows: - return - if result.skipped: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"No importable rows. {len(result.skipped)} row(s) skipped due " - "to missing or invalid conversation ID or recording URL." - ), - ) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"{source_label} did not contain any data rows.", - ) - - -def _parse_csv( - file_bytes: bytes, - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, str], - skipped_columns: List[str], -) -> CallImportParseResult: - """Parse a CSV file using the resolved schema parameters.""" - if not file_bytes: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Uploaded CSV is empty.", - ) - - try: - text_stream = io.StringIO(file_bytes.decode("utf-8-sig")) - except UnicodeDecodeError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="CSV must be UTF-8 encoded.", - ) - - reader = csv.DictReader(text_stream) - if not reader.fieldnames: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="CSV is missing a header row.", - ) - - return _apply_schema_mapping( - list(reader.fieldnames), - reader, - parameters, - parameter_mapping, - skipped_columns, - source_label="CSV", - ) - - -def _open_xlsx_workbook(file_bytes: bytes): - """Open an xlsx/xlsm workbook from in-memory bytes (read-only stream). - - Imports openpyxl lazily so the module loads even in environments that - haven't installed the optional dep yet (e.g. lightweight tooling - images). Surfaces a clean 400 if openpyxl is missing or the file is - not a valid Office Open XML workbook. - """ - if not file_bytes: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Uploaded Excel file is empty.", - ) - try: - from openpyxl import load_workbook # type: ignore - from openpyxl.utils.exceptions import InvalidFileException # type: ignore - except ImportError as exc: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=( - "Excel uploads require the 'openpyxl' package which is " - "not installed in this environment." - ), - ) from exc - - try: - return load_workbook( - io.BytesIO(file_bytes), - read_only=True, - data_only=True, - ) - except InvalidFileException as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"File is not a valid .xlsx workbook: {exc}", - ) from exc - except Exception as exc: # zipfile.BadZipFile etc. - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Could not open Excel workbook: {exc}", - ) from exc - - -def _xlsx_sheet_headers_and_rows( - worksheet, -) -> Tuple[List[str], List[Dict[str, str]]]: - """Read row 1 as headers and the rest as dicts of stringified cells. - - Empty trailing header cells are dropped. Duplicate headers preserve - the first occurrence (matches ``csv.DictReader`` behavior, which - silently drops duplicates). - """ - iterator = worksheet.iter_rows(values_only=True) - try: - header_row = next(iterator) - except StopIteration: - return [], [] - - headers: List[str] = [] - seen: set[str] = set() - for cell in header_row: - name = _xlsx_cell_to_str(cell).strip() - if not name: - # Stop at the first blank header — treats trailing empty - # columns as not part of the table (matches typical Excel - # workbook conventions). - break - norm = name.lower() - if norm in seen: - continue - seen.add(norm) - headers.append(name) - - rows: List[Dict[str, str]] = [] - for row in iterator: - if row is None: - continue - # Pad / truncate to the header length so dict construction is - # stable even when a row has fewer / extra cells than the header. - cells = list(row[: len(headers)]) - if len(cells) < len(headers): - cells.extend([None] * (len(headers) - len(cells))) - if not any(_xlsx_cell_to_str(c).strip() for c in cells): - # Skip fully-blank rows (openpyxl read_only routinely yields - # trailing empties when the worksheet's used range exceeds - # the actual data). - continue - rows.append( - { - header: _xlsx_cell_to_str(value) - for header, value in zip(headers, cells) - } - ) - - return headers, rows - - -def _parse_xlsx( - file_bytes: bytes, - sheet_name: Optional[str], - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, str], - skipped_columns: List[str], -) -> CallImportParseResult: - """Parse a single worksheet from an xlsx/xlsm workbook. - - ``sheet_name`` must match one of the workbook's sheets (case - insensitive whitespace-trimmed match). Returns the same shape as - :func:`_parse_csv` so the upload handler can persist either format - through the same code path. - """ - if not sheet_name or not sheet_name.strip(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is required when uploading an Excel workbook.", - ) - - workbook = _open_xlsx_workbook(file_bytes) - try: - sheet_names = list(workbook.sheetnames) - target_norm = sheet_name.strip().lower() - match = next( - (s for s in sheet_names if s.strip().lower() == target_norm), - None, - ) - if match is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Sheet '{sheet_name}' not found in workbook. " - f"Available sheets: {sheet_names}" - ), - ) - worksheet = workbook[match] - headers, rows = _xlsx_sheet_headers_and_rows(worksheet) - finally: - workbook.close() - - if not headers: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Sheet '{sheet_name}' is missing a header row.", - ) - - return _apply_schema_mapping( - headers, - rows, - parameters, - parameter_mapping, - skipped_columns, - source_label=f"Sheet '{sheet_name}'", - ) - - -def _csv_preview_sheets( - file_bytes: bytes, filename: Optional[str] -) -> List[CallImportPreviewSheet]: - """Build the synthetic single-sheet preview entry for a CSV upload.""" - if not file_bytes: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Uploaded CSV is empty.", - ) - try: - text_stream = io.StringIO(file_bytes.decode("utf-8-sig")) - except UnicodeDecodeError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="CSV must be UTF-8 encoded.", - ) - reader = csv.DictReader(text_stream) - if not reader.fieldnames: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="CSV is missing a header row.", - ) - headers = list(reader.fieldnames) - row_count = 0 - for row in reader: - # Match the parse-time skip: ignore fully blank rows so the - # count the user sees lines up with what /upload will ingest. - if any((v or "").strip() for v in row.values()): - row_count += 1 - - sheet_label = (filename or "sheet1").rsplit("/", 1)[-1] or "sheet1" - return [ - CallImportPreviewSheet( - name=sheet_label, - headers=headers, - row_count=row_count, - ) - ] - - -def _xlsx_preview_sheets(file_bytes: bytes) -> List[CallImportPreviewSheet]: - """List every worksheet in the workbook with its headers and row count.""" - workbook = _open_xlsx_workbook(file_bytes) - sheets: List[CallImportPreviewSheet] = [] - try: - for name in workbook.sheetnames: - worksheet = workbook[name] - headers, rows = _xlsx_sheet_headers_and_rows(worksheet) - sheets.append( - CallImportPreviewSheet( - name=name, - headers=headers, - row_count=len(rows), - ) - ) - finally: - workbook.close() - return sheets - - -def _parse_json_form_field(name: str, raw: Optional[str], default): - """Decode a JSON-encoded form field with a friendly 400 on bad JSON.""" - if raw is None or raw == "": - return default - try: - return json.loads(raw) - except json.JSONDecodeError as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"{name} must be valid JSON: {exc}", - ) - - -# --------------------------------------------------------------------------- -# Shared helpers used by the staged endpoints (UPLOAD / MAP / IMPORT) and the -# legacy one-shot ``POST /upload`` shim. Extracted here so each stage and the -# back-compat path operate on the exact same validation + persistence code. -# --------------------------------------------------------------------------- - - -def _source_content_type(fmt: str) -> str: - """Return the canonical ``Content-Type`` for a parsed file format.""" - if fmt == "csv": - return "text/csv" - if fmt == "xlsx": - return ( - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ) - return "application/octet-stream" - - -def _source_s3_key( - organization_id: UUID, call_import_id: UUID, fmt: str -) -> str: - """Build the canonical S3 key for an upload's source file. - - Mirrors the per-row recording key convention used by - ``process_call_import_row`` so a single prefix sweep on delete still - cleans up both the source artefact and every fetched recording. - """ - from app.services.storage.s3_service import s3_service - - ext = "xlsx" if fmt == "xlsx" else "csv" - return ( - f"{s3_service.prefix}organizations/{organization_id}/call_imports/" - f"{call_import_id}/source.{ext}" - ) - - -def _build_available_sheets( - file_bytes: bytes, fmt: str, filename: Optional[str] -) -> List[CallImportPreviewSheet]: - """Snapshot of sheets + headers cached on the batch at UPLOAD time.""" - if fmt == "csv": - return _csv_preview_sheets(file_bytes, filename) - return _xlsx_preview_sheets(file_bytes) - - -def _resolve_schema( - db: Session, - organization_id: UUID, - workspace_id: UUID, - schema_id: UUID, -) -> CallImportSchema: - """Fetch + validate a schema row in the active workspace. - - Eager-loads ``parameters`` so callers can iterate without re-querying. - """ - from sqlalchemy.orm import selectinload as _selectinload - - schema = ( - db.query(CallImportSchema) - .options(_selectinload(CallImportSchema.parameters)) - .filter( - CallImportSchema.id == schema_id, - CallImportSchema.organization_id == organization_id, - CallImportSchema.workspace_id == workspace_id, - ) - .first() - ) - if not schema: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Call import schema not found in the active workspace.", - ) - if not list(schema.parameters): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Selected schema has no parameters defined.", - ) - return schema - - -def _validate_direct_url_import_ready( - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, Any], -) -> None: - """Ensure direct-URL import has a mapped recording_url column.""" - rec_url_param = next( - ( - p - for p in parameters - if p.type == CallImportParameterType.RECORDING_URL.value - ), - None, - ) - if rec_url_param is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Direct URL import requires a schema parameter of type " - "'recording_url'." - ), - ) - mapped_header = (parameter_mapping or {}).get(rec_url_param.name) - if not (mapped_header or "").strip(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Direct URL import requires the 'recording_url' parameter to " - "be mapped to a source column." - ), - ) - - -def _validate_exotel_import_ready( - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, Any], -) -> None: - """Ensure Exotel credentialed import has a mapped recording_url column.""" - rec_url_param = next( - ( - p - for p in parameters - if p.type == CallImportParameterType.RECORDING_URL.value - ), - None, - ) - if rec_url_param is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Exotel import requires a schema parameter of type " - "'recording_url'." - ), - ) - mapped_header = (parameter_mapping or {}).get(rec_url_param.name) - if not (mapped_header or "").strip(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Exotel import requires the 'recording_url' parameter to " - "be mapped to a source column." - ), - ) - - -def _resolve_telephony_integration( - db: Session, - organization_id: UUID, - telephony_integration_id: UUID, - provider: str, -) -> TelephonyIntegration: - """Fetch + validate a telephony credential against the requested provider.""" - integration = ( - db.query(TelephonyIntegration) - .filter( - TelephonyIntegration.id == telephony_integration_id, - TelephonyIntegration.organization_id == organization_id, - ) - .first() - ) - if not integration: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Telephony credential not found for this organization.", - ) - if (integration.provider or "").lower() != provider.lower(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Selected credential is for provider '{integration.provider}', " - f"but request specified '{provider}'." - ), - ) - if not integration.is_active: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Selected telephony credential is inactive.", - ) - return integration - - -def _clean_parameter_mapping( - mapping_payload: Any, - parameters: List[CallImportSchemaParameter], - schema_name: str, -) -> Dict[str, str]: - """Trim values and drop empties; reject unknown parameter names. - - Accepts an already-decoded value (dict-shaped) so the same helper - works for the JSON-form upload path and the JSON-body PATCH path. - """ - if not isinstance(mapping_payload, dict) or not all( - isinstance(k, str) and (v is None or isinstance(v, str)) - for k, v in mapping_payload.items() - ): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "parameter_mapping must be an object of " - "{parameter_name: csv_header}." - ), - ) - - valid_param_names = {p.name for p in parameters} - cleaned: Dict[str, str] = {} - for raw_name, raw_header in mapping_payload.items(): - if raw_name not in valid_param_names: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"parameter_mapping references unknown parameter " - f"'{raw_name}' on schema '{schema_name}'." - ), - ) - header = (raw_header or "").strip() - if header: - cleaned[raw_name] = header - return cleaned - - -def _clean_skipped_columns(skipped_payload: Any) -> List[str]: - """Dedupe (case-insensitively) and drop blanks; preserve original casing.""" - if not isinstance(skipped_payload, list) or not all( - isinstance(item, str) for item in skipped_payload - ): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="skipped_columns must be a list of header strings.", - ) - cleaned: List[str] = [] - seen: set[str] = set() - for item in skipped_payload: - norm = _normalize_header(item) - if not norm or norm in seen: - continue - seen.add(norm) - cleaned.append(item) - return cleaned - - -def _parse_source_file( - file_bytes: bytes, - fmt: str, - sheet_name: Optional[str], - parameters: List[CallImportSchemaParameter], - cleaned_mapping: Dict[str, str], - cleaned_skipped: List[str], -) -> CallImportParseResult: - """Run the format-appropriate parser against a buffer of file bytes.""" - if fmt == "csv": - return _parse_csv(file_bytes, parameters, cleaned_mapping, cleaned_skipped) - return _parse_xlsx( - file_bytes, sheet_name, parameters, cleaned_mapping, cleaned_skipped - ) - - -def _materialize_rows( - db: Session, - call_import: CallImport, - parsed_rows: List[Dict[str, Any]], - organization_id: UUID, -) -> List[CallImportRow]: - """Insert one ``CallImportRow`` per parsed row, returning the new models.""" - row_models: List[CallImportRow] = [] - for idx, row in enumerate(parsed_rows): - # Stamp ``transcript_source='csv'`` when the upload actually - # provided a transcript so the UI badge ("From CSV") works from - # day one. Blank cells stay NULL so the row reads as "no - # production transcript yet". - csv_transcript = row["transcript"] - row_model = CallImportRow( - call_import_id=call_import.id, - organization_id=organization_id, - workspace_id=call_import.workspace_id, - row_index=idx, - conversation_id=row["conversation_id"], - recording_date=( - _parse_recording_date_cell(row["recording_date"]) - if row.get("recording_date") - else None - ), - recording_url=row["recording_url"], - transcript=csv_transcript, - transcript_source=( - "csv" if csv_transcript and csv_transcript.strip() else None - ), - raw_columns=row["parameter_values"] or None, - status=CallImportRowStatus.PENDING, - ) - db.add(row_model) - row_models.append(row_model) - return row_models - - -def _enqueue_row_tasks( - db: Session, - call_import: CallImport, - row_models: List[CallImportRow], -) -> None: - """Schedule fair round-robin dispatch for pending import rows.""" - del db, call_import, row_models - from app.workers.concurrency.fair_import_dispatch import ( - schedule_fair_import_dispatch, - ) - - schedule_fair_import_dispatch(max_workspace_turns=999) - - -def _ensure_blob_storage_enabled() -> None: - """Hard-fail UPLOAD if cloud blob storage isn't configured (no local fallback).""" - from app.services.storage.s3_service import s3_service - - if not s3_service.is_enabled(): - err = ( - s3_service.get_status_message() - or "Cloud blob storage is not enabled or not configured" - ) - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail=( - "Call uploads require cloud blob storage so the file can be " - f"persisted between stages: {err}" - ), - ) - - -def _validate_sheet_choice( - fmt: str, - sheet_name: Optional[str], - available_sheets: Optional[List[Dict[str, Any]]], -) -> Optional[str]: - """Normalize / validate ``sheet_name`` against the persisted snapshot. - - Returns the canonical sheet name (matching the workbook's casing) - so downstream parsing addresses the right worksheet. - """ - if fmt == "csv": - if sheet_name and sheet_name.strip(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is not applicable to CSV uploads.", - ) - return None - - cleaned = (sheet_name or "").strip() or None - if cleaned is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is required when the source is an Excel workbook.", - ) - - if not available_sheets: - # Nothing to validate against (e.g. legacy batch without snapshot); - # let downstream parsing error out instead of silently importing. - return cleaned - - target = cleaned.strip().lower() - for entry in available_sheets: - name = entry.get("name") if isinstance(entry, dict) else None - if isinstance(name, str) and name.strip().lower() == target: - return name - sheet_names = [ - entry.get("name") for entry in available_sheets if isinstance(entry, dict) - ] - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Sheet '{cleaned}' not found in the staged file. " - f"Available sheets: {sheet_names}" - ), - ) - - -def _tag_response_payload(tags: Optional[List[CallImportTag]]) -> List[Dict[str, Any]]: - """Shape a CallImport's tag relationship for the upload response.""" - return [ - { - "id": tag.id, - "name": tag.name, - "color": tag.color, - "created_at": tag.created_at, - "updated_at": tag.updated_at, - } - for tag in (tags or []) - ] - - -@router.post( - "/preview", - response_model=CallImportPreviewResponse, - operation_id="previewCallImportFile", -) -async def preview_call_import_file( - file: UploadFile = File(...), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportPreviewResponse: - """Inspect an uploaded CSV / Excel file and return its sheets + headers. - - Drives the column-mapping UI without forcing the frontend to parse - CSV / xlsx itself — keeps client and server in lockstep on quoted - fields, encodings, and Excel cell coercion. CSVs return a single - synthetic sheet named after the filename; Excel workbooks return one - entry per worksheet (in workbook order). - """ - del api_key, organization_id, workspace_id, db # auth only - - fmt = _file_format(file.filename) - if fmt is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Unsupported file format. Allowed extensions: " - f"{', '.join(ALLOWED_EXTENSIONS)}." - ), - ) - - file_bytes = await file.read() - if len(file_bytes) > MAX_UPLOAD_BYTES: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", - ) - - if fmt == "csv": - sheets = _csv_preview_sheets(file_bytes, file.filename) - else: - sheets = _xlsx_preview_sheets(file_bytes) - - return CallImportPreviewResponse(format=fmt, sheets=sheets) - - -@router.post( - "", - response_model=CallImportResponse, - status_code=status.HTTP_201_CREATED, - operation_id="createCallImport", -) -async def create_call_import( - file: UploadFile = File( - ..., - description="CSV / Excel file to stage. Persisted to S3 between stages.", - ), - dataset: str = Form( - ..., - description=( - "Required free-text dataset label. Collected up-front so the " - "batch is filterable from the moment it lands." - ), - ), - tag_ids: Optional[List[UUID]] = Form( - None, - description="Optional list of CallImportTag ids to attach to the new batch.", - ), - schema_id: Optional[UUID] = Form( - None, - description=( - "Optional schema pre-pick. The user can still change it during " - "the MAP stage; provided here only so the detail page can pre-" - "select the schema dropdown." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportResponse: - """UPLOAD stage of the staged call-import flow. - - Persists the source file to S3 and creates a ``CallImport`` row with - ``status='uploaded'``. No mapping, no provider, no rows yet — the - user moves through MAP and IMPORT as separate idempotent steps. - - Dataset is collected here (rather than at IMPORT) so the batch is - filterable from the moment it appears in the list view. - """ - del api_key - - fmt = _file_format(file.filename) - if fmt is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Unsupported file format. Allowed extensions: " - f"{', '.join(ALLOWED_EXTENSIONS)}." - ), - ) - - normalized_dataset = _normalize_dataset(dataset) - if not normalized_dataset: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="dataset is required and must be a non-empty string.", - ) - - file_bytes = await file.read() - if len(file_bytes) > MAX_UPLOAD_BYTES: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", - ) - - # Parse-now so we (a) reject garbage uploads up-front instead of - # later in the MAP step, and (b) capture the sheets snapshot the - # MAP UI needs without having to re-fetch the file from S3. - sheets = _build_available_sheets(file_bytes, fmt, file.filename) - - # Optional schema pre-pick: validated only if supplied (the user is - # allowed to set it for the first time during MAP). - if schema_id is not None: - _resolve_schema(db, organization_id, workspace_id, schema_id) - - tag_rows = _resolve_tags(db, organization_id, tag_ids) - - _ensure_blob_storage_enabled() - - # Pre-generate the id so we can compute a deterministic S3 key - # before the row is persisted, keeping ``source_s3_key`` consistent - # with the prefix sweep used at delete-time. - import uuid as _uuid - - call_import_id = _uuid.uuid4() - s3_key = _source_s3_key(organization_id, call_import_id, fmt) - content_type = _source_content_type(fmt) - - from app.services.storage.s3_service import s3_service, StorageError - - try: - s3_service.upload_file_by_key(file_bytes, s3_key, content_type=content_type) - except StorageError as exc: - logger.exception( - "Failed to upload source file to S3 for new call import {}", - call_import_id, - ) - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail=f"Failed to persist upload to S3: {exc}", - ) - - call_import = CallImport( - id=call_import_id, - organization_id=organization_id, - workspace_id=workspace_id, - # Provider + credential aren't known until the IMPORT stage; leave - # them NULL so the staged-vs-legacy distinction is visible at a - # glance from the DB. - provider=None, - telephony_integration_id=None, - original_filename=file.filename, - sheet_name=None, - dataset=normalized_dataset, - schema_id=schema_id, - parameter_mapping={}, - skipped_columns=[], - column_mapping={}, - extra_columns=[], - custom_column_mapping={}, - source_s3_key=s3_key, - source_format=fmt, - source_size_bytes=len(file_bytes), - source_content_type=content_type, - available_sheets=[sheet.model_dump() for sheet in sheets], - total_rows=0, - completed_rows=0, - failed_rows=0, - status=CallImportStatus.UPLOADED, - ) - if tag_rows: - call_import.tags = tag_rows - - stamp_call_import_actor(call_import, principal, creating=True) - db.add(call_import) - try: - db.commit() - except Exception: - db.rollback() - # Best-effort cleanup of the uploaded S3 object so a failed - # commit doesn't leak storage. - try: - s3_service.delete_file_by_key(s3_key) - except Exception as cleanup_exc: # noqa: BLE001 - logger.warning( - "Failed to clean up orphaned S3 object {} after DB rollback: {}", - s3_key, - cleanup_exc, - ) - raise - - db.refresh(call_import) - return _serialize_call_import(db, call_import) - - -@router.patch( - "/{call_import_id}/mapping", - response_model=CallImportResponse, - operation_id="updateCallImportMapping", -) -async def update_call_import_mapping( - call_import_id: UUID, - payload: CallImportMappingUpdate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportResponse: - """MAP stage of the staged call-import flow. - - Validates ``parameter_mapping`` + ``skipped_columns`` against the - sheet headers captured at UPLOAD time and persists them on the - batch. Idempotent: callers may submit this multiple times while - the batch is in ``uploaded`` or ``mapped`` state. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if call_import.status not in ( - CallImportStatus.UPLOADED, - CallImportStatus.MAPPED, - ): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - f"Cannot edit mapping on a batch in status " - f"'{call_import.status.value}'. Mapping can only be edited " - "before the IMPORT stage." - ), - ) - - if not call_import.source_s3_key or not call_import.source_format: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This batch was not uploaded through the staged flow and " - "cannot have its mapping edited." - ), - ) - - schema = _resolve_schema( - db, organization_id, workspace_id, payload.schema_id - ) - parameters = list(schema.parameters) - - canonical_sheet = _validate_sheet_choice( - call_import.source_format, - payload.sheet_name, - call_import.available_sheets, - ) - - # Pull the headers for the selected sheet straight out of the - # snapshot so we don't have to re-download the file from S3 just to - # validate the mapping. - headers: List[str] = [] - if call_import.available_sheets: - if canonical_sheet is None: - # CSV: single synthetic sheet. - entry = call_import.available_sheets[0] - headers = list(entry.get("headers") or []) - else: - for entry in call_import.available_sheets: - if not isinstance(entry, dict): - continue - name = entry.get("name") - if isinstance(name, str) and name == canonical_sheet: - headers = list(entry.get("headers") or []) - break - - cleaned_mapping = _clean_parameter_mapping( - payload.parameter_mapping, parameters, schema.name - ) - cleaned_skipped = _clean_skipped_columns(payload.skipped_columns) - - # Run the same per-column validation as the parse path so the user - # gets an immediate 400 if a required parameter is left unmapped or - # a header is neither mapped nor skipped — without needing to read - # the file. ``validate_only`` skips the row loop (and the empty-rows - # guard) since the row data lives in S3, not in this request. - if headers: - _apply_schema_mapping( - headers, - iter(()), - parameters, - cleaned_mapping, - cleaned_skipped, - source_label=( - f"Sheet '{canonical_sheet}'" - if canonical_sheet is not None - else "CSV" - ), - validate_only=True, - ) - - call_import.schema_id = schema.id - call_import.parameter_mapping = dict(cleaned_mapping) - call_import.skipped_columns = list(cleaned_skipped) - call_import.sheet_name = canonical_sheet - call_import.status = CallImportStatus.MAPPED - stamp_call_import_actor(call_import, principal) - db.commit() - db.refresh(call_import) - return _serialize_call_import(db, call_import) - - -@router.post( - "/{call_import_id}/import", - response_model=CallImportUploadResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="startCallImport", -) -async def start_call_import( - call_import_id: UUID, - payload: CallImportStartRequest, - background_tasks: BackgroundTasks, - legacy: bool = Query( - False, - description=( - "Deprecated escape hatch for import-only processing. " - "New batches should use Run Evaluation instead." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportUploadResponse: - """Deprecated IMPORT stage — use Run Evaluation for new batches. - - Recording fetch is part of the unified evaluation pipeline. This - endpoint remains available only with ``?legacy=true`` for backward - compatibility. - """ - del api_key, background_tasks - - if not legacy: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "Standalone import is deprecated. Use Run Evaluation — " - "recording fetch is part of the evaluation pipeline. " - "Append ?legacy=true to use the import-only path." - ), - ) - - from sqlalchemy.orm import selectinload as _selectinload - - call_import = ( - db.query(CallImport) - .options(_selectinload(CallImport.tags)) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if call_import.status != CallImportStatus.MAPPED: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - f"Cannot start import for a batch in status " - f"'{call_import.status.value}'. Map the columns first." - ), - ) - - if not call_import.source_s3_key or not call_import.source_format: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This batch has no staged source file and cannot be imported " - "through the staged flow." - ), - ) - - if not call_import.schema_id: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="Cannot start import without a mapped schema.", - ) - - schema = _resolve_schema( - db, organization_id, workspace_id, call_import.schema_id - ) - parameters = list(schema.parameters) - - if payload.telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, - organization_id, - payload.telephony_integration_id, - payload.provider or "", - ) - if (integration.provider or "").lower() == "exotel": - _validate_exotel_import_ready( - parameters, dict(call_import.parameter_mapping or {}) - ) - else: - _validate_direct_url_import_ready( - parameters, dict(call_import.parameter_mapping or {}) - ) - integration = None - - _ensure_blob_storage_enabled() - - if integration is not None: - call_import.provider = integration.provider - call_import.telephony_integration_id = integration.id - else: - call_import.provider = None - call_import.telephony_integration_id = None - - call_import.total_rows = 0 - call_import.completed_rows = 0 - call_import.failed_rows = 0 - call_import.error_message = None - call_import.status = CallImportStatus.PROCESSING - stamp_call_import_actor(call_import, principal) - db.commit() - db.refresh(call_import) - - from app.workers.tasks.call_import_bulk_ops import ( - materialize_call_import_rows_task, - ) - - materialize_call_import_rows_task.delay( - str(call_import_id), - str(organization_id), - str(workspace_id), - schedule_import_dispatch=True, - ) - - return CallImportUploadResponse( - id=call_import.id, - total_rows=0, - status=call_import.status, - dataset=call_import.dataset, - tags=_tag_response_payload(call_import.tags), - message=( - "Import accepted. Rows are being materialized in the background; " - "recordings will be fetched asynchronously." - ), - ) - - -@router.post( - "/upload", - response_model=CallImportUploadResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="uploadCallImportCsv", - deprecated=True, -) -async def upload_call_import_csv( - background_tasks: BackgroundTasks, - file: UploadFile = File(...), - provider: Optional[str] = Form( - None, - description=( - "Telephony provider key (e.g. 'exotel', 'plivo'). Must match the " - "selected telephony_integration_id's provider. Omit together " - "with telephony_integration_id for direct-URL import." - ), - ), - telephony_integration_id: Optional[UUID] = Form( - None, - description=( - "Specific TelephonyIntegration credential row to use when " - "downloading recordings for this batch. Omit together with " - "provider for direct-URL import." - ), - ), - schema_id: UUID = Form( - ..., - description=( - "Reusable Input Parameter schema this upload is mapped against. " - "Must belong to the active workspace." - ), - ), - parameter_mapping: str = Form( - ..., - description=( - "JSON-encoded ``{schema_parameter_name: source_header}`` map " - "covering every required schema parameter. Optional parameters " - "may be omitted or set to an empty string." - ), - ), - skipped_columns: Optional[str] = Form( - None, - description=( - "JSON-encoded list of source header strings the uploader has " - "explicitly skipped. Every header in the file must either be " - "mapped or appear here; otherwise the upload is rejected so a " - "forgotten column never silently drops." - ), - ), - dataset: Optional[str] = Form( - None, - description=( - "Optional free-text dataset label for high-level segregation. " - "Empty strings are stored as NULL." - ), - ), - tag_ids: Optional[List[UUID]] = Form( - None, - description="Optional list of CallImportTag ids to attach to the new batch.", - ), - sheet_name: Optional[str] = Form( - None, - description=( - "Worksheet to import when the file is an Excel workbook " - "(.xlsx / .xlsm). REQUIRED for Excel uploads. Ignored for CSV " - "uploads (rejected with 400 if non-empty so typos surface " - "instead of silently importing the wrong source)." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportUploadResponse: - """Legacy one-shot upload kept for backward compatibility. - - DEPRECATED: prefer the staged flow - (``POST /`` → ``PATCH /{id}/mapping`` → ``POST /{id}/import``) so - each step is idempotent and resumable. This endpoint runs all three - stages inline in a single transaction so existing scripts / - integrations keep working unchanged. - """ - del api_key - - fmt = _file_format(file.filename) - if fmt is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Unsupported file format. Allowed extensions: " - f"{', '.join(ALLOWED_EXTENSIONS)}." - ), - ) - - sheet_name_clean = (sheet_name or "").strip() or None - if fmt == "csv" and sheet_name_clean is not None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is not applicable to CSV uploads.", - ) - if fmt == "xlsx" and sheet_name_clean is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is required when uploading an Excel workbook.", - ) - - file_bytes = await file.read() - if len(file_bytes) > MAX_UPLOAD_BYTES: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", - ) - - schema = _resolve_schema(db, organization_id, workspace_id, schema_id) - parameters = list(schema.parameters) - - mapping_payload = _parse_json_form_field( - "parameter_mapping", parameter_mapping, {} - ) - cleaned_mapping = _clean_parameter_mapping( - mapping_payload, parameters, schema.name - ) - - skipped_payload = _parse_json_form_field("skipped_columns", skipped_columns, []) - cleaned_skipped = _clean_skipped_columns(skipped_payload) - - has_provider = bool((provider or "").strip()) - has_integration = telephony_integration_id is not None - if has_provider != has_integration: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "provider and telephony_integration_id must both be provided " - "or both omitted for direct-URL import." - ), - ) - - if telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, organization_id, telephony_integration_id, provider or "" - ) - if (integration.provider or "").lower() == "exotel": - _validate_exotel_import_ready(parameters, cleaned_mapping) - else: - _validate_direct_url_import_ready(parameters, cleaned_mapping) - integration = None - - parsed_rows = _parse_source_file( - file_bytes, fmt, sheet_name_clean, parameters, cleaned_mapping, cleaned_skipped - ) - _raise_if_no_importable_rows(parsed_rows, source_label=fmt) - - tag_rows = _resolve_tags(db, organization_id, tag_ids) - - call_import = CallImport( - organization_id=organization_id, - workspace_id=workspace_id, - provider=integration.provider if integration is not None else None, - telephony_integration_id=integration.id if integration is not None else None, - original_filename=file.filename, - sheet_name=sheet_name_clean, - dataset=_normalize_dataset(dataset), - schema_id=schema.id, - parameter_mapping=dict(cleaned_mapping), - skipped_columns=list(cleaned_skipped), - # Legacy columns are left empty on new uploads; the detail page - # falls back to ``parameter_mapping`` when ``schema_id`` is set. - column_mapping={}, - extra_columns=[], - custom_column_mapping={}, - total_rows=len(parsed_rows.rows), - completed_rows=0, - failed_rows=0, - status=CallImportStatus.PENDING, - source_row_skips=parse_skips_to_json(parsed_rows.skipped), - ) - if tag_rows: - call_import.tags = tag_rows - stamp_call_import_actor(call_import, principal, creating=True) - db.add(call_import) - db.flush() # populate call_import.id - if integration is None: - # The model's historical Python default is "exotel"; direct-URL - # imports intentionally have no telephony provider. - call_import.provider = None - - row_models = _materialize_rows( - db, call_import, parsed_rows.rows, organization_id - ) - - call_import.status = CallImportStatus.PROCESSING - stamp_call_import_actor(call_import, principal) - db.commit() - db.refresh(call_import) - - background_tasks.add_task( - record_call_import_batch_created, - organization_id, - call_import.id, - workspace_id=workspace_id, - total_rows=call_import.total_rows, - source="csv", - provider=call_import.provider, - ) - - _enqueue_row_tasks(db, call_import, row_models) - - return CallImportUploadResponse( - id=call_import.id, - total_rows=call_import.total_rows, - status=call_import.status, - dataset=call_import.dataset, - tags=_tag_response_payload(call_import.tags), - message=( - f"Accepted {call_import.total_rows} rows for import. " - "Recordings will be fetched asynchronously." - ), - ) - - -@router.post( - "/audio-upload", - response_model=CallImportUploadResponse, - status_code=status.HTTP_201_CREATED, - operation_id="uploadCallImportAudio", -) -async def upload_call_import_audio( - background_tasks: BackgroundTasks, - files: List[UploadFile] = File( - ..., - description="One or more manual call recording audio files.", - ), - dataset: str = Form( - ..., - description="Required free-text dataset label for the manual upload batch.", - ), - tag_ids: Optional[List[UUID]] = Form( - None, - description="Optional list of CallImportTag ids to attach to the new batch.", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportUploadResponse: - """Persist manually uploaded recordings as completed CallImport rows. - - The rows skip the provider-download worker entirely because the audio - bytes are already in hand. From this point onward they behave exactly - like completed CSV-import rows: playback reads ``recording_s3_key`` and - the existing diarisation/evaluation endpoints can operate on them. - """ - - normalized_dataset = _normalize_dataset(dataset) - if not normalized_dataset: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="dataset is required and must be a non-empty string.", - ) - if not files: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="At least one audio file is required.", - ) - - _ensure_blob_storage_enabled() - tag_rows = _resolve_tags(db, organization_id, tag_ids) - - max_bytes = int(settings.MAX_FILE_SIZE_MB) * 1024 * 1024 - prepared: List[Dict[str, Any]] = [] - conversation_counts: Dict[str, int] = {} - - for idx, upload in enumerate(files): - filename = upload.filename or f"recording-{idx + 1}" - ext = _audio_extension(filename) - if not ext: - allowed = ", ".join(settings.ALLOWED_AUDIO_FORMATS) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Unsupported audio file '{filename}'. Allowed formats: {allowed}.", - ) - - contents = await upload.read() - if not contents: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Audio file '{filename}' is empty.", - ) - if len(contents) > max_bytes: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=( - f"Audio file '{filename}' exceeds " - f"{settings.MAX_FILE_SIZE_MB} MB." - ), - ) - - base_conversation_id = _sanitize_conversation_id(_filename_stem(filename)) - conversation_id = _dedupe_conversation_id( - base_conversation_id, - conversation_counts, - ) - prepared.append( - { - "filename": filename, - "extension": ext, - "content_type": _audio_content_type(ext, upload.content_type), - "contents": contents, - "conversation_id": conversation_id, - } - ) - - original_filename = ( - prepared[0]["filename"] - if len(prepared) == 1 - else f"{len(prepared)} manual recordings" - ) - total_size = sum(len(item["contents"]) for item in prepared) - uploaded_keys: List[str] = [] - - from app.services.storage.s3_service import s3_service - - call_import = CallImport( - organization_id=organization_id, - workspace_id=workspace_id, - provider=None, - telephony_integration_id=None, - original_filename=original_filename, - source_format="audio", - source_size_bytes=total_size, - source_content_type="audio/*", - dataset=normalized_dataset, - total_rows=len(prepared), - completed_rows=len(prepared), - failed_rows=0, - status=CallImportStatus.COMPLETED, - ) - if tag_rows: - call_import.tags = tag_rows - - stamp_call_import_actor(call_import, principal, creating=True) - try: - db.add(call_import) - db.flush() - # The model's historical Python default is "exotel"; manual uploads - # intentionally have no telephony provider. - call_import.provider = None - - row_mappings: List[Dict[str, Any]] = [] - for idx, item in enumerate(prepared): - row_id = uuid4() - key = _audio_s3_key( - organization_id, - call_import.id, - row_id, - item["extension"], - ) - s3_service.upload_file_by_key( - item["contents"], - key, - content_type=item["content_type"], - ) - uploaded_keys.append(key) - - row_mappings.append( - { - "id": row_id, - "call_import_id": call_import.id, - "organization_id": organization_id, - "workspace_id": workspace_id, - "row_index": idx, - "conversation_id": item["conversation_id"], - "recording_url": None, - "transcript": None, - "transcript_source": None, - "raw_columns": {"conversation_id": item["conversation_id"]}, - "status": CallImportRowStatus.COMPLETED, - "recording_s3_key": key, - "recording_content_type": item["content_type"], - "recording_size_bytes": len(item["contents"]), - } - ) - - if is_sharding_enabled(): - from app.db_sharding.row_ops import ( - bulk_insert_mappings_on_shards, - register_shard_slices, - ) - - bulk_insert_mappings_on_shards(db, call_import.id, row_mappings) - register_shard_slices(db, call_import.id, len(row_mappings)) - else: - for mapping in row_mappings: - db.add(CallImportRow(**mapping)) - - db.commit() - except Exception as exc: - db.rollback() - if uploaded_keys and s3_service.is_enabled(): - try: - s3_service.delete_keys(uploaded_keys) - except Exception: - logger.exception( - "Failed to clean up manual audio upload keys after error" - ) - logger.exception("Failed to persist manual call recording upload") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to upload manual recordings: {exc}", - ) from exc - - db.refresh(call_import) - background_tasks.add_task( - record_call_import_batch_created, - organization_id, - call_import.id, - workspace_id=workspace_id, - total_rows=call_import.total_rows, - source="audio", - provider=None, - ) - return CallImportUploadResponse( - id=call_import.id, - total_rows=call_import.total_rows, - status=call_import.status, - dataset=call_import.dataset, - tags=_tag_response_payload(call_import.tags), - message=( - f"Uploaded {call_import.total_rows} manual recording" - f"{'' if call_import.total_rows == 1 else 's'}." - ), - ) - - -@router.get( - "", - response_model=CallImportListResponse, - operation_id="listCallImports", -) -async def list_call_imports( - page: int = Query(1, ge=1), - page_size: int = Query(20, ge=1, le=100), - status_filter: Optional[CallImportStatus] = Query(None, alias="status"), - dataset: Optional[str] = Query( - None, - description=( - "Filter by exact dataset string (case-insensitive). Pass the " - "literal value '__none__' to filter to imports with no dataset." - ), - ), - tag_id: Optional[List[UUID]] = Query( - None, - description="Filter to imports tagged with ALL of the given tag ids.", - ), - source_format: Optional[str] = Query( - None, - description=( - "Filter by source format. Use 'audio' for manual recordings or " - "'__non_audio__' for CSV/Excel/legacy imports." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportListResponse: - """List call-import batches for the active workspace, newest first. - - Scoped to (organization_id, workspace_id) so users only see imports - for the workspace they're currently in. Supports a high-level - ``dataset`` filter (powers the segregation dropdown at the top of - the imports page) plus an AND-style multi-tag filter via repeated - ``tag_id`` parameters. - """ - - query = ( - db.query(CallImport) - .filter( - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - ) - ) - if status_filter is not None: - query = query.filter(CallImport.status == status_filter) - - source_filter = (source_format or "").strip().lower() - if source_filter == "__non_audio__": - query = query.filter( - or_(CallImport.source_format.is_(None), CallImport.source_format != "audio") - ) - elif source_filter: - query = query.filter(func.lower(CallImport.source_format) == source_filter) - - if dataset is not None: - if dataset == "__none__": - query = query.filter(CallImport.dataset.is_(None)) - elif dataset.strip(): - query = query.filter( - func.lower(CallImport.dataset) == dataset.strip().lower() - ) - - if tag_id: - from app.models.database import CallImportTagAssignment - - for single_tag_id in tag_id: - sub = ( - db.query(CallImportTagAssignment.call_import_id) - .filter(CallImportTagAssignment.tag_id == single_tag_id) - .subquery() - ) - query = query.filter(CallImport.id.in_(sub)) - - total = query.count() - items = ( - query.order_by(desc(CallImport.created_at)) - .offset((page - 1) * page_size) - .limit(page_size) - .all() - ) - - email_map = emails_for_user_ids(db, user_ids_from_call_imports(items)) - return CallImportListResponse( - items=[ - _serialize_call_import(db, item, user_emails=email_map) - for item in items - ], - total=total, - page=page, - page_size=page_size, - ) - - -@router.get( - "/dispatch-diagnostics", - response_model=CallImportDispatchDiagnosticsResponse, - operation_id="getCallImportDispatchDiagnostics", - dependencies=[Depends(require_admin)], -) -async def get_call_import_dispatch_diagnostics( - workspace_id: Optional[UUID] = Query( - None, - description=( - "Optional workspace filter. When omitted, returns every workspace " - "in the organization with active eval dispatch state." - ), - ), - include_idle_workspaces: bool = Query( - False, - description=( - "When true, include org workspaces with zero pending rows and " - "zero in-flight slots." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportDispatchDiagnosticsResponse: - """Live eval slot usage and fair-dispatch state for operators. - - Org admins use this to diagnose cross-workspace starvation (e.g. one - workspace's 10k run blocking another's pending eval rows) by inspecting - Redis in-flight counters, pending dispatch rows, and scheduler cursors. - """ - del api_key - payload = build_call_import_dispatch_diagnostics( - db, - organization_id, - workspace_id=workspace_id, - include_idle_workspaces=include_idle_workspaces, - ) - return CallImportDispatchDiagnosticsResponse.model_validate(payload) - - -@router.get( - "/datasets", - response_model=List[str], - operation_id="listCallImportDatasets", -) -async def list_call_import_datasets( - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> List[str]: - """Return the distinct, non-null dataset labels in use for the active - workspace. - - Scoped per-workspace so each workspace's Dataset dropdown only shows - its own segregation labels. - """ - rows = ( - db.query(CallImport.dataset) - .filter( - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - CallImport.dataset.isnot(None), - CallImport.dataset != "", - ) - .distinct() - .order_by(CallImport.dataset.asc()) - .all() - ) - return [row[0] for row in rows if row[0]] - - -@router.get( - "/diarisation-prompt-default", - response_model=CallImportDiarisationPromptDefaultResponse, - operation_id="getCallImportDiarisationPromptDefault", -) -async def get_call_import_diarisation_prompt_default( - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), -) -> CallImportDiarisationPromptDefaultResponse: - """Return the canonical LLM diariser prompt. - - The Transcribe / Run Evaluation modals call this on open so they - can pre-fill the prompt textarea. Returning the constant from the - backend (rather than hard-coding it in the frontend) keeps the - fallback used by the worker and the placeholder shown in the UI - in lock-step — operators always see the *actual* default they'd - get if they leave the field blank. - - Registered before ``GET /{call_import_id}`` so the static path is - not mistaken for a UUID import id (which would 422). - """ - del api_key, organization_id - from app.workers.tasks.helpers.llm_diarisation import ( - DEFAULT_DIARIZATION_PROMPT, - ) - - return CallImportDiarisationPromptDefaultResponse( - prompt=DEFAULT_DIARIZATION_PROMPT - ) - - -@router.patch( - "/{call_import_id}", - response_model=CallImportResponse, - operation_id="updateCallImport", -) -async def update_call_import( - call_import_id: UUID, - payload: CallImportUpdate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportResponse: - """Edit dataset / tag assignments (and schema, pre-import) on a batch. - - ``dataset = ""`` clears the label; ``tag_ids = []`` removes all tag - assignments. Fields omitted from the body are left untouched. - - ``schema_id`` is only honoured while the batch is in - ``uploaded`` / ``mapped`` state — once rows have been materialised - the schema is locked. Changing the schema resets any persisted - mapping (the user must re-MAP) and rewinds status to ``uploaded``. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - body = payload.model_dump(exclude_unset=True) - if "dataset" in body: - call_import.dataset = _normalize_dataset(body["dataset"]) - - if "tag_ids" in body: - tag_ids = body["tag_ids"] or [] - call_import.tags = _resolve_tags(db, organization_id, tag_ids) - - if "schema_id" in body and body["schema_id"] is not None: - if call_import.status not in ( - CallImportStatus.UPLOADED, - CallImportStatus.MAPPED, - ): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - f"Cannot reassign schema on a batch in status " - f"'{call_import.status.value}'." - ), - ) - new_schema = _resolve_schema( - db, organization_id, workspace_id, body["schema_id"] - ) - if call_import.schema_id != new_schema.id: - # Switching schemas invalidates the persisted mapping — - # parameter names won't line up with the new schema, so - # reset to UPLOADED and force a fresh MAP. - call_import.schema_id = new_schema.id - call_import.parameter_mapping = {} - call_import.skipped_columns = [] - call_import.sheet_name = None - call_import.status = CallImportStatus.UPLOADED - - stamp_call_import_actor(call_import, principal) - db.commit() - db.refresh(call_import) - return _serialize_call_import(db, call_import) - - -@router.get( - "/{call_import_id}", - response_model=CallImportDetailResponse, - operation_id="getCallImportDetail", -) -async def get_call_import_detail( - call_import_id: UUID, - row_limit: int = Query(500, ge=0, le=5000), - row_offset: int = Query(0, ge=0), - q: Optional[str] = Query( - None, - description=( - "Optional case-insensitive substring filter on " - "``conversation_id``. When set, ``filtered_total_rows`` in " - "the response reflects the post-filter row count so the UI " - "can paginate against the filtered slice." - ), - ), - diarised_status: Optional[str] = Query( - None, - description=( - "Optional filter on ``CallImportRow.diarised_transcript_status``. " - "Accepts one of ``pending``, ``running``, ``completed``, " - "``failed``. When set, ``filtered_total_rows`` reflects the " - "post-filter row count (combined with the ``q`` filter when " - "both are supplied) so the UI can paginate against the same " - "slice it's displaying." - ), - pattern="^(pending|running|completed|failed)$", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportDetailResponse: - """Fetch a single import batch with a slice of its rows. - - ``row_limit=0`` is intentionally allowed so callers that only need the - batch metadata (e.g. the evaluation-detail page rendering the parent's - column mapping) can skip the rows payload entirely. - """ - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if call_import.status == CallImportStatus.PROCESSING: - from app.services.call_imports.bulk_ops import rollup_call_import_batch_status - - prior_status = call_import.status - rollup_call_import_batch_status(db, call_import) - if call_import.status != prior_status: - db.commit() - db.refresh(call_import) - - search_term = (q or "").strip() - diarised_status_filter = (diarised_status or "").strip() or None - filtered_total_rows: Optional[int] = None - has_row_filters = bool(search_term or diarised_status_filter) - - if has_row_filters: - if is_sharding_enabled(): - from app.db_sharding.scatter_gather import count_call_import_rows_filtered - - filtered_total_rows = count_call_import_rows_filtered( - db, - call_import.id, - search_term=search_term, - diarised_status_filter=diarised_status_filter, - ) - else: - rows_query = db.query(CallImportRow).filter( - CallImportRow.call_import_id == call_import.id - ) - if search_term: - rows_query = rows_query.filter( - CallImportRow.conversation_id.ilike(f"%{search_term}%") - ) - if diarised_status_filter: - rows_query = rows_query.filter( - CallImportRow.diarised_transcript_status == diarised_status_filter - ) - filtered_total_rows = rows_query.count() - - if row_limit == 0: - rows: List[CallImportRow] = [] - elif is_sharding_enabled(): - from app.db_sharding.scatter_gather import ( - fetch_call_import_rows_filtered_page, - fetch_call_import_rows_page, - ) - - if has_row_filters: - rows = fetch_call_import_rows_filtered_page( - db, - call_import.id, - search_term=search_term, - diarised_status_filter=diarised_status_filter, - offset=row_offset, - limit=row_limit, - ) - else: - rows = fetch_call_import_rows_page( - db, - call_import.id, - offset=row_offset, - limit=row_limit, - ) - else: - rows_query = db.query(CallImportRow).filter( - CallImportRow.call_import_id == call_import.id - ) - if search_term: - rows_query = rows_query.filter( - CallImportRow.conversation_id.ilike(f"%{search_term}%") - ) - if diarised_status_filter: - rows_query = rows_query.filter( - CallImportRow.diarised_transcript_status == diarised_status_filter - ) - rows = ( - rows_query.order_by(CallImportRow.row_index) - .offset(row_offset) - .limit(row_limit) - .all() - ) - - # Batch-wide diarisation status aggregate. One ``GROUP BY`` query - # across the whole batch — much cheaper than paging through every - # row to recount on the client and lets the UI render a - # transcribe/diarise progress bar without a separate roundtrip. - if is_sharding_enabled(): - from app.db_sharding.scatter_gather import aggregate_diarised_transcript_counts - - diarised_status_counts = aggregate_diarised_transcript_counts( - db, call_import.id - ) - else: - diarised_status_counts: Dict[str, int] = {} - for status_value, count in ( - db.query(CallImportRow.diarised_transcript_status, func.count()) - .filter(CallImportRow.call_import_id == call_import.id) - .group_by(CallImportRow.diarised_transcript_status) - .all() - ): - if isinstance(status_value, str): - diarised_status_counts[status_value] = int(count or 0) - - detail = CallImportDetailResponse.model_validate( - _serialize_call_import(db, call_import).model_dump() - ) - detail.rows = [CallImportRowResponse.model_validate(r) for r in rows] - detail.filtered_total_rows = filtered_total_rows - detail.diarised_pending_rows = diarised_status_counts.get("pending", 0) - detail.diarised_running_rows = diarised_status_counts.get("running", 0) - detail.diarised_completed_rows = diarised_status_counts.get("completed", 0) - detail.diarised_failed_rows = diarised_status_counts.get("failed", 0) - return detail - - -@router.get( - "/{call_import_id}/row-ids", - response_model=CallImportRowIdsResponse, - operation_id="listCallImportRowIds", -) -async def list_call_import_row_ids( - call_import_id: UUID, - q: Optional[str] = Query( - None, - description=( - "Optional case-insensitive substring filter on " - "``conversation_id``. Same semantics as the detail endpoint." - ), - ), - diarised_status: Optional[str] = Query( - None, - description=( - "Optional filter on ``CallImportRow.diarised_transcript_status``. " - "Accepts ``pending`` / ``running`` / ``completed`` / ``failed``." - ), - pattern="^(pending|running|completed|failed)$", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportRowIdsResponse: - """Return every matching ``CallImportRow.id`` for cross-page bulk select. - - Lightweight companion to ``GET /{call_import_id}`` — the detail - endpoint caps ``row_limit`` at 5000 and ships the entire row body - on each page, so harvesting ids that way is wasteful when the - user just wants to bulk-delete or bulk-transcribe everything that - matches the current filters. This endpoint applies the same ``q`` - and ``diarised_status`` filters and returns only the ids. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - search_term = (q or "").strip() - status_filter = (diarised_status or "").strip() or None - - if is_sharding_enabled(): - from app.db_sharding.scatter_gather import list_call_import_row_ids_filtered - - ids = list_call_import_row_ids_filtered( - db, - call_import.id, - search_term=search_term, - diarised_status_filter=status_filter, - ) - return CallImportRowIdsResponse(ids=ids, total=len(ids)) - - rows_query = db.query(CallImportRow.id).filter( - CallImportRow.call_import_id == call_import.id - ) - if search_term: - rows_query = rows_query.filter( - CallImportRow.conversation_id.ilike(f"%{search_term}%") - ) - if status_filter: - rows_query = rows_query.filter( - CallImportRow.diarised_transcript_status == status_filter - ) - - ids = [ - row_id - for (row_id,) in rows_query.order_by(CallImportRow.row_index).all() - ] - return CallImportRowIdsResponse(ids=ids, total=len(ids)) - - -def _revoke_pending_tasks(rows: List[CallImportRow]) -> None: - """Best-effort revoke of in-flight Celery tasks for the given rows. - - Failures are logged and swallowed — Celery's control plane is async and - best-effort by design, and we always do an idempotent S3 cleanup - afterwards so a missed revoke can't leak storage. - """ - task_ids = [ - r.celery_task_id - for r in rows - if r.celery_task_id - and r.status in (CallImportRowStatus.PENDING, CallImportRowStatus.PROCESSING) - ] - if not task_ids: - return - - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke(task_ids, terminate=False) - logger.info("Revoked {} pending call-import tasks", len(task_ids)) - except Exception as exc: # noqa: BLE001 - logger.warning("Failed to revoke pending call-import tasks: {}", exc) - - -def _delete_s3_objects( - organization_id: UUID, - call_import_id: UUID, - rows: List[CallImportRow], -) -> tuple[int, int]: - """Delete every recording associated with ``rows`` plus a prefix sweep. - - The prefix sweep also cleans up the staged source file written at - UPLOAD time (``…/call_imports/{id}/source.{csv,xlsx}``) — both the - per-row recording keys and the source artefact share the same - organization-scoped prefix, so a single sweep covers them all. - - Returns ``(deleted_count, error_count)``. Never raises — callers proceed - with the DB delete regardless; orphans, if any, can be cleaned up by - re-running the same delete (it's idempotent). - """ - from app.services.storage.s3_service import s3_service - - if not s3_service.is_enabled(): - return 0, 0 - - keys = [r.recording_s3_key for r in rows if r.recording_s3_key] - deleted = 0 - errors = 0 - - if keys: - try: - d, errs = s3_service.delete_keys(keys) - deleted += d - errors += len(errs) - if errs: - logger.warning( - "S3 bulk-delete reported {} errors for call_import {}", - len(errs), - call_import_id, - ) - except Exception as exc: # noqa: BLE001 - logger.exception( - "Bulk S3 delete failed for call_import {}: {}", call_import_id, exc - ) - errors += len(keys) - - # Belt-and-braces sweep: catch anything that landed under the import's - # prefix but never made it into a row's recording_s3_key (narrow - # window where the S3 upload succeeded but the DB commit didn't). - sweep_prefix = ( - f"{s3_service.prefix}organizations/{organization_id}/" - f"call_imports/{call_import_id}/" - ) - try: - d, errs = s3_service.delete_keys_by_prefix(sweep_prefix) - deleted += d - errors += len(errs) - except Exception as exc: # noqa: BLE001 - logger.exception( - "S3 prefix sweep failed for {}: {}", sweep_prefix, exc - ) - - return deleted, errors - - -@router.delete( - "/{call_import_id}", - response_model=CallImportDeleteResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="deleteCallImport", -) -async def delete_call_import( - call_import_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportDeleteResponse: - """Delete a call-import batch asynchronously. - - Flips the batch to ``deleting`` and enqueues background teardown so - large imports (thousands of rows + S3 objects) do not block the API. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - return CallImportDeleteResponse( - id=call_import_id, - status="completed", - ) - - if call_import.status == CallImportStatus.DELETING: - return CallImportDeleteResponse( - id=call_import.id, - status="accepted", - ) - - call_import.status = CallImportStatus.DELETING - call_import.error_message = None - stamp_call_import_actor(call_import, principal) - db.commit() - - from app.workers.tasks.call_import_bulk_ops import delete_call_import_task - - delete_call_import_task.delay( - str(call_import_id), - str(organization_id), - ) - - return CallImportDeleteResponse( - id=call_import.id, - status="accepted", - ) - - -def _locate_call_import_row_or_404( - catalog_db: Session, - *, - call_import_id: UUID, - row_id: UUID, - organization_id: UUID, -) -> Tuple[Session, CallImportRow, Optional[Session]]: - """Find a call import row on the correct DB session for mutation. - - When sharding is enabled rows live on shard databases; ``get_db`` only - opens the catalog. Returns ``(row_db, row, extra_catalog_to_close)`` - where ``extra_catalog_to_close`` is the catalog session opened by - :func:`locate_call_import_row` (distinct from the route's catalog - session) and must be closed via :func:`close_row_sessions`. - """ - from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row - - try: - row_db, located_catalog, row, _shard_id = locate_call_import_row(row_id) - except LookupError: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import row not found", - ) from None - if ( - row.call_import_id != call_import_id - or row.organization_id != organization_id - ): - close_row_sessions( - row_db, - located_catalog if located_catalog is not row_db else None, - ) - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import row not found", - ) - extra_catalog = located_catalog if located_catalog is not row_db else None - return row_db, row, extra_catalog - - -@router.delete( - "/{call_import_id}/rows/{row_id}", - status_code=status.HTTP_204_NO_CONTENT, - operation_id="deleteCallImportRow", -) -async def delete_call_import_row( - call_import_id: UUID, - row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> Response: - """Delete a single CallImportRow and its S3 recording. - - The parent ``CallImport`` is left in place. After deletion we recompute - its ``total_rows`` / ``completed_rows`` / ``failed_rows`` / ``status`` - so the UI's progress bar stays consistent with reality. - """ - from app.services.storage.s3_service import s3_service - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.db_sharding.row_ops import close_row_sessions - - row_db, row, extra_catalog = _locate_call_import_row_or_404( - db, - call_import_id=call_import.id, - row_id=row_id, - organization_id=organization_id, - ) - try: - _revoke_pending_tasks([row]) - - if row.recording_s3_key and s3_service.is_enabled(): - try: - s3_service.delete_file_by_key(row.recording_s3_key) - except Exception as exc: # noqa: BLE001 — best-effort, DB is source of truth - logger.warning( - "Failed to delete S3 object {} for row {}: {}", - row.recording_s3_key, - row.id, - exc, - ) - - row_db.delete(row) - row_db.commit() - - _recompute_call_import_counters(db, call_import) - stamp_call_import_actor(call_import, principal) - db.commit() - finally: - close_row_sessions(row_db, extra_catalog) - - logger.info( - "Deleted call_import_row {} (call_import={}, org={})", - row_id, - call_import.id, - organization_id, - ) - - return Response(status_code=status.HTTP_204_NO_CONTENT) - - -def _recompute_call_import_counters( - db: Session, call_import: CallImport -) -> None: - """Resync ``total/completed/failed_rows`` + status on the parent batch. - - Called after row-level mutations (single delete, bulk delete) so the - UI's progress bar stays consistent with the actual row set. The - rules mirror :func:`delete_call_import_row` so behavior doesn't - diverge between the per-row and bulk paths. - """ - - from app.services.call_imports.bulk_ops import rollup_call_import_batch_status - - rollup_call_import_batch_status(db, call_import) - - -@router.post( - "/{call_import_id}/retry-failed", - response_model=CallImportRetryFailedRowsResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="retryFailedCallImportRows", -) -async def retry_failed_call_import_rows( - call_import_id: UUID, - payload: Optional[CallImportRetryFailedRowsRequest] = Body(None), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportRetryFailedRowsResponse: - """Re-enqueue every failed import row in this batch. - - Useful when transient provider issues are resolved and the operator wants - a one-click "try failed downloads again" pass without re-uploading the CSV. - - Pass ``provider`` + ``telephony_integration_id`` (or both omitted for - direct-URL retry) to change how recordings are fetched on this pass. - When the body is omitted entirely, the batch keeps its existing pinned - credentials. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if payload is not None: - if payload.telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, - organization_id, - payload.telephony_integration_id, - payload.provider or "", - ) - call_import.provider = integration.provider - call_import.telephony_integration_id = integration.id - if (integration.provider or "").lower() == "exotel": - schema = _resolve_schema( - db, - organization_id, - call_import.workspace_id, - call_import.schema_id, - ) - _validate_exotel_import_ready( - list(schema.parameters), - dict(call_import.parameter_mapping or {}), - ) - else: - call_import.provider = None - call_import.telephony_integration_id = None - db.flush() - - failed_rows = ( - db.query(CallImportRow) - .filter( - CallImportRow.call_import_id == call_import.id, - CallImportRow.status == CallImportRowStatus.FAILED, - ) - .order_by(CallImportRow.row_index.asc()) - .all() - ) - if not failed_rows: - return CallImportRetryFailedRowsResponse( - requeued=0, - enqueue_failed=0, - skipped=0, - ) - - from app.workers.concurrency.fair_import_dispatch import ( - schedule_fair_import_dispatch, - ) - - # Reset rows to pending BEFORE enqueue so the UI reflects "retry in - # progress" immediately even if the worker queue is backlogged. - for row in failed_rows: - row.status = CallImportRowStatus.PENDING - row.error_message = None - row.celery_task_id = None - - db.flush() - _recompute_call_import_counters(db, call_import) - stamp_call_import_actor(call_import, principal) - db.commit() - - try: - schedule_fair_import_dispatch(max_workspace_turns=999) - requeued = len(failed_rows) - enqueue_failed = 0 - skipped = 0 - except Exception as exc: # noqa: BLE001 - logger.exception( - "Failed to schedule fair import dispatch for import {}", - call_import.id, - ) - requeued = 0 - enqueue_failed = len(failed_rows) - skipped = 0 - for row in failed_rows: - db.refresh(row) - if row.status != CallImportRowStatus.PENDING: - skipped += 1 - enqueue_failed -= 1 - continue - row.status = CallImportRowStatus.FAILED - row.error_message = f"Failed to enqueue retry: {exc}" - db.flush() - _recompute_call_import_counters(db, call_import) - db.commit() - - return CallImportRetryFailedRowsResponse( - requeued=requeued, - enqueue_failed=enqueue_failed, - skipped=skipped, - ) - - -@router.post( - "/{call_import_id}/rows/bulk-delete", - response_model=CallImportRowBulkDeleteResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="bulkDeleteCallImportRows", -) -async def bulk_delete_call_import_rows( - call_import_id: UUID, - payload: CallImportRowBulkDelete, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportRowBulkDeleteResponse: - """Delete multiple ``CallImportRow`` rows in one request. - - Unknown / cross-tenant row ids are silently skipped — the response - reports how many actually went away so a UI that holds onto stale - ids (e.g. after another tab already deleted a row) doesn't 404 - the entire bulk action. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if not payload.row_ids: - return CallImportRowBulkDeleteResponse(deleted=0, status="completed") - - from app.workers.tasks.call_import_bulk_ops import bulk_delete_call_import_rows_task - - row_id_strs = [str(rid) for rid in payload.row_ids] - - stamp_call_import_actor(call_import, principal) - db.commit() - - bulk_delete_call_import_rows_task.delay( - str(call_import_id), - str(organization_id), - row_id_strs, - ) - - return CallImportRowBulkDeleteResponse(deleted=0, status="accepted") - - -# --------------------------------------------------------------------------- -# Diarization / transcription endpoints -# --------------------------------------------------------------------------- - - -def _select_rows_for_transcription( - db: Session, - call_import: CallImport, - payload: CallImportTranscribeRequest, - requested_row_ids: Optional[List[UUID]] = None, -) -> tuple[List[CallImportRow], Dict[str, int]]: - """Pick which rows to enqueue for diarisation (delegates to bulk_ops).""" - from app.services.call_imports.bulk_ops import select_rows_for_transcription - - try: - return select_rows_for_transcription( - db, call_import, payload, requested_row_ids=requested_row_ids - ) - except ValueError as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=str(exc), - ) from exc - - -@router.post( - "/{call_import_id}/transcribe", - response_model=CallImportTranscribeResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="transcribeCallImport", -) -async def transcribe_call_import( - call_import_id: UUID, - payload: CallImportTranscribeRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportTranscribeResponse: - """Fan out diarization tasks for many rows in a single call. - - Returns a summary with how many rows were queued and how many were - skipped (broken down by reason) so the UI can show a meaningful - toast even when nothing actually got enqueued (e.g. "All 12 rows - already have transcripts"). - """ - - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.workers.tasks.call_import_bulk_ops import bulk_diarize_call_import_task - - stamp_call_import_actor(call_import, principal) - db.commit() - - bulk_diarize_call_import_task.delay( - str(call_import_id), - str(organization_id), - payload.model_dump(mode="json"), - [str(rid) for rid in payload.row_ids] if payload.row_ids else None, - ) - - return CallImportTranscribeResponse( - queued=0, - skipped_rows=0, - skipped_reason_counts={}, - accepted=True, - ) - - -@router.post( - "/{call_import_id}/rows/{row_id}/transcribe", - response_model=CallImportTranscribeResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="transcribeCallImportRow", -) -async def transcribe_call_import_row( - call_import_id: UUID, - row_id: UUID, - payload: CallImportTranscribeRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportTranscribeResponse: - """Diarize / transcribe a single row. - - Thin wrapper over the batch endpoint that hard-codes a single - ``row_ids`` filter. Skip counts still surface so the UI can render - "Skipped — transcript present" diagnostics consistently. - """ - - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.services.call_imports.bulk_ops import execute_bulk_diarization - - try: - result = execute_bulk_diarization( - db, - call_import, - payload, - requested_row_ids=[row_id], - ) - except ValueError as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=str(exc), - ) from exc - - stamp_call_import_actor(call_import, principal) - db.commit() - - return CallImportTranscribeResponse( - queued=result.queued, - skipped_rows=result.skipped_rows, - skipped_reason_counts=result.skipped_reason_counts, - ) - - -# --------------------------------------------------------------------------- -# Cancel-in-flight diarisation -# --------------------------------------------------------------------------- -# -# Long-running multimodal LLM diarisation calls (especially LLM-only mode on -# slow audio) can sit in ``pending`` / ``running`` for tens of minutes when an -# upstream provider stalls. Without an abort affordance the operator's only -# recourse is to wait for Celery's ``time_limit`` to fire — which can be -# several minutes — or to manually mutate the DB. These helpers + the two -# endpoints below give the UI a first-class "Stop diarisation" button. -# -# Why ``terminate=True``: the legacy ``_revoke_pending_tasks`` helper uses -# ``terminate=False`` because it's called from delete-flow paths where the -# task may simply not get to run (a worker pulls it off the queue and drops -# it). For a user-initiated cancel we want SIGTERM to interrupt the worker -# mid-LLM call so the audio HTTP request actually aborts. ``terminate=True`` -# routes SIGTERM to the executing process; ``signal="SIGTERM"`` is the -# default but we spell it out so the intent is obvious to reviewers. - -# Sentinel error message stamped on cancelled rows. Read by the transcribe -# worker's finaliser (see ``app/workers/tasks/transcribe_call_import_row.py``) -# to detect a row that was cancelled mid-flight and AVOID overwriting it -# with whatever partial result the worker had managed to compute before the -# SIGTERM landed. -CANCELLED_BY_USER_ERROR: str = "Diarisation cancelled by user" - - -def _cancellable_diarisation_states() -> Tuple[str, ...]: - """States that a diarisation row can be cancelled from. - - Kept as a tiny helper so adding a future ``"queued"`` / ``"retrying"`` - state only needs one edit. - """ - return ("pending", "running") - - -def _revoke_diarisation_task(row: CallImportRow) -> None: - """Best-effort revoke of a single row's diarisation Celery task. - - Always swallows control-plane exceptions — Celery's control bus is - inherently best-effort and a missed revoke is not catastrophic - because the DB row is already flipped to ``failed`` by the caller - before this runs (so the UI immediately reflects the cancel; if - the task happens to finish anyway, the worker's finaliser skips - over the row via :data:`CANCELLED_BY_USER_ERROR`). - """ - task_id = (row.celery_task_id or "").strip() - if not task_id: - return - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke( - task_id, terminate=True, signal="SIGTERM" - ) - logger.info( - "Revoked diarisation task {} for call-import row {}", - task_id, - row.id, - ) - except Exception as exc: # noqa: BLE001 — revoke is best-effort - logger.warning( - "Failed to revoke diarisation task {} for row {}: {}", - task_id, - row.id, - exc, - ) - - -def _apply_diarisation_cancel(rows: List[CallImportRow]) -> Tuple[int, int]: - """Cancel diarisation on every cancellable row in ``rows``. - - Returns ``(cancelled, skipped)`` so the caller can build a typed - response without re-querying the DB. The caller is responsible for - ``db.commit()`` after this returns — we deliberately don't commit - here so a batch endpoint can flush all rows in one transaction. - """ - cancellable_states = _cancellable_diarisation_states() - cancelled = 0 - skipped = 0 - for row in rows: - if (row.diarised_transcript_status or "").lower() not in cancellable_states: - skipped += 1 - continue - # Flip the row state BEFORE we revoke so the UI's next poll - # already shows the cancel, even if Celery's control plane is - # slow to ack. - row.diarised_transcript_status = "failed" - row.diarised_transcript_error = CANCELLED_BY_USER_ERROR - _revoke_diarisation_task(row) - # Drop the task id so a follow-up retry (or a stale poll) can't - # accidentally re-revoke or get confused. - row.celery_task_id = None - cancelled += 1 - return cancelled, skipped - - -@router.post( - "/{call_import_id}/rows/{row_id}/cancel-diarisation", - response_model=CallImportRowResponse, - status_code=status.HTTP_200_OK, - operation_id="cancelCallImportRowDiarisation", -) -async def cancel_call_import_row_diarisation( - call_import_id: UUID, - row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportRowResponse: - """Abort an in-flight (or queued) diarisation for a single row. - - Idempotent: calling on a row that's already terminal (``completed`` - / ``failed`` / ``idle``) returns the row unchanged with a 200, so - the UI can fire this from a "Stop" button without having to - pre-check the state. - - Race notes: - - * The row's ``diarised_transcript_status`` is flipped to ``failed`` - with :data:`CANCELLED_BY_USER_ERROR` BEFORE the Celery revoke, - so the polling UI sees the cancel immediately. - * If the worker happens to finish between our DB flip and the - SIGTERM landing, its finaliser will detect the cancelled - sentinel on the row and skip its own status / score writes - (see :mod:`app.workers.tasks.transcribe_call_import_row`). - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.db_sharding.row_ops import close_row_sessions - - row_db, row, extra_catalog = _locate_call_import_row_or_404( - db, - call_import_id=call_import_id, - row_id=row_id, - organization_id=organization_id, - ) - try: - _apply_diarisation_cancel([row]) - row_db.commit() - stamp_call_import_actor(call_import, principal) - db.commit() - row_db.refresh(row) - return CallImportRowResponse.model_validate(row) - finally: - close_row_sessions(row_db, extra_catalog) - - -@router.post( - "/{call_import_id}/cancel-diarisation", - response_model=CallImportCancelDiarisationResponse, - status_code=status.HTTP_200_OK, - operation_id="cancelCallImportDiarisation", -) -async def cancel_call_import_diarisation( - call_import_id: UUID, - payload: Optional[CallImportCancelDiarisationRequest] = None, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportCancelDiarisationResponse: - """Abort in-flight diarisation for many rows in a single call. - - Default body (no ``row_ids``) cancels every row in this import - whose ``diarised_transcript_status`` is ``pending`` or - ``running`` — the "stop everything" button. Pass ``row_ids`` to - scope the cancel to the rows the operator has selected. - - Returns ``(cancelled, skipped)`` so the UI can render a tight - toast ("Cancelled 3 rows · 1 skipped (already completed)"). - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - base_query = db.query(CallImportRow).filter( - CallImportRow.call_import_id == call_import_id - ) - - requested_ids = ( - payload.row_ids if payload and payload.row_ids is not None else None - ) - if requested_ids is not None: - if not requested_ids: - # Empty list is "no rows requested" — treat as a no-op - # 200 rather than 400 so the UI can pass through an empty - # selection without a special-case. - return CallImportCancelDiarisationResponse(cancelled=0, skipped=0) - rows = base_query.filter(CallImportRow.id.in_(requested_ids)).all() - found_ids = {r.id for r in rows} - # Treat requested-but-not-found ids as ``skipped`` so the UI's - # numbers reconcile (a stale selection that includes deleted - # rows shouldn't 404 the whole call). - missing = [rid for rid in requested_ids if rid not in found_ids] - skipped_missing = len(missing) - else: - # Implicit "cancel every cancellable row in this import" path. - rows = base_query.filter( - CallImportRow.diarised_transcript_status.in_( - list(_cancellable_diarisation_states()) - ) - ).all() - skipped_missing = 0 - - cancelled, skipped = _apply_diarisation_cancel(rows) - stamp_call_import_actor(call_import, principal) - db.commit() - return CallImportCancelDiarisationResponse( - cancelled=cancelled, - skipped=skipped + skipped_missing, - ) - - -def _render_diarised_segments_text( - segments: Optional[List[Dict[str, Any]]], - *, - swap: bool = False, -) -> str: - """Render ``CallImportRow.diarised_segments`` as ``: `` lines. - - Mirrors the worker's ``_render_turns_as_text`` (kept duplicated so - the route doesn't need to import a Celery task module just to - rebuild the rendered transcript). Only ``agent`` and ``user`` are - swapped — multi-party calls keep their ``speaker_N`` labels through - a swap so we don't silently collapse a third speaker into the user - side. - """ - if not segments: - return "" - out: List[str] = [] - for turn in segments: - if not isinstance(turn, dict): - continue - speaker = (turn.get("speaker") or "").strip() - text = (turn.get("text") or "").strip() - if not speaker or not text: - continue - if swap: - if speaker == "agent": - speaker = "user" - elif speaker == "user": - speaker = "agent" - out.append(f"{speaker}: {text}") - return "\n".join(out) - - -@router.post( - "/{call_import_id}/rows/{row_id}/diarised-speaker-swap", - response_model=CallImportRowResponse, - operation_id="toggleCallImportRowSpeakerSwap", -) -async def toggle_call_import_row_speaker_swap( - call_import_id: UUID, - row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - principal: Principal = Depends(get_principal), - db: Session = Depends(get_db), -) -> CallImportRowResponse: - """Flip the user <-> agent mapping on a diarised row. - - The worker's "first speaker is the agent" heuristic is right most of - the time but does fail on inbound recordings where the customer - greets first, on recordings where the agent stays silent for the - intro, etc. Rather than rerun the (expensive) STT + pyannote - pipeline for those cases, we let reviewers flip the mapping in - place: the structured ``diarised_segments`` are the source of truth - and we re-render the plain-text ``diarised_transcript`` from them - with the swap applied. The next CSV export will then show the - corrected labels. - - Returns the updated row so the frontend can refresh without an - extra round-trip. - """ - - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.db_sharding.row_ops import close_row_sessions - - row_db, row, extra_catalog = _locate_call_import_row_or_404( - db, - call_import_id=call_import_id, - row_id=row_id, - organization_id=organization_id, - ) - try: - segments = ( - row.diarised_segments if isinstance(row.diarised_segments, list) else None - ) - if not segments: - # Without structured turns the swap toggle would have nothing to - # re-render — surface a clear error rather than silently - # flipping a flag the UI never read. - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This row has no structured diarised segments to swap. " - "Re-run diarisation to generate per-speaker turns first." - ), - ) - - new_swap = not bool(row.diarised_speaker_swap) - row.diarised_speaker_swap = new_swap - row.diarised_transcript = ( - _render_diarised_segments_text(segments, swap=new_swap) or None - ) - row_db.commit() - stamp_call_import_actor(call_import, principal) - db.commit() - row_db.refresh(row) - return CallImportRowResponse.model_validate(row) - finally: - close_row_sessions(row_db, extra_catalog) - - -# --------------------------------------------------------------------------- -# Cross-run insights for the import detail page -# --------------------------------------------------------------------------- - - -@router.get( - "/{call_import_id}/insights", - response_model=CallImportInsightsResponse, - operation_id="getCallImportInsights", -) -async def get_call_import_insights( - call_import_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportInsightsResponse: - """Aggregate signals across every evaluation run on this import. - - Powers the Insights tab on the call-import detail page: returns - per-metric "latest run" summaries plus a trend series of mean values - across runs so the UI can render a small line chart per metric. Also - bundles transcript coverage stats since those are the cheapest - pre-eval health-check (e.g. "30 of 50 rows still missing - transcripts"). - """ - - del api_key - - from app.models.database import ( - CallImportEvaluation, - CallImportEvaluationRow, - Metric, - ) - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - rows = ( - db.query(CallImportRow) - .filter(CallImportRow.call_import_id == call_import_id) - .all() - ) - # A row "has a transcript" if EITHER the production (CSV) or the - # diarised (worker) column is populated — the insights tile reports - # the union so users see total coverage regardless of which source - # produced the value. - rows_with_transcript = sum( - 1 - for r in rows - if (r.transcript or "").strip() - or (r.diarised_transcript or "").strip() - ) - rows_without_transcript = len(rows) - rows_with_transcript - source_counts: Dict[str, int] = {} - for r in rows: - has_production = bool((r.transcript or "").strip()) - has_diarised = bool((r.diarised_transcript or "").strip()) - if has_production: - key = r.transcript_source or "csv" - source_counts[key] = source_counts.get(key, 0) + 1 - if has_diarised: - source_counts["diarised"] = source_counts.get("diarised", 0) + 1 - - evaluations = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .order_by(CallImportEvaluation.created_at.asc()) - .all() - ) - - # Defer heavy lifting to the aggregation helper so this endpoint and - # the per-run aggregate endpoint share the exact same metric - # bucketing math (no chance of "trend" disagreeing with "latest" on - # the same data set). - from app.api.v1.routes.call_import_evaluations import ( - _compute_metric_aggregates, - ) - - metric_history: Dict[str, List[CallImportInsightsRunPoint]] = {} - metric_meta: Dict[str, Metric] = {} - metric_latest: Dict[str, CallImportMetricAggregate] = {} - - for evaluation in evaluations: - eval_rows = ( - db.query(CallImportEvaluationRow) - .filter(CallImportEvaluationRow.evaluation_id == evaluation.id) - .all() - ) - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - for agg in aggregates: - if agg.metric_id not in metric_meta: - # ``agg.metric_id`` is normally a UUID string, but the - # aggregator also emits ids that surface in row scores - # without a matching ``Metric`` row (e.g. a metric the - # user deleted mid-run, or LLM-discovered slugs). Those - # are not valid UUIDs, so coerce defensively and skip - # the metric registry lookup when the cast fails — the - # ``meta is None`` branch below already handles the - # display via the values stored on ``agg`` itself. - try: - metric_uuid = UUID(agg.metric_id) - except (ValueError, AttributeError, TypeError): - metric_uuid = None - if metric_uuid is not None: - metric_obj = ( - db.query(Metric) - .filter( - Metric.id == metric_uuid, - Metric.organization_id == organization_id, - ) - .first() - ) - if metric_obj is not None: - metric_meta[agg.metric_id] = metric_obj - history = metric_history.setdefault(agg.metric_id, []) - history.append( - CallImportInsightsRunPoint( - evaluation_id=evaluation.id, - name=evaluation.name, - created_at=evaluation.created_at, - mean=agg.mean, - completed_rows=agg.count, - ) - ) - metric_latest[agg.metric_id] = agg - - metrics_payload: List[CallImportInsightsMetric] = [] - for metric_id, latest in metric_latest.items(): - meta = metric_meta.get(metric_id) - metrics_payload.append( - CallImportInsightsMetric( - metric_id=metric_id, - metric_name=(meta.name if meta else latest.metric_name), - metric_type=(meta.metric_type if meta else latest.metric_type), - latest=latest, - trend=metric_history.get(metric_id, []), - ) - ) - - return CallImportInsightsResponse( - call_import_id=call_import_id, - total_rows=len(rows), - rows_with_transcript=rows_with_transcript, - rows_without_transcript=rows_without_transcript, - transcript_source_counts=source_counts, - evaluation_count=len(evaluations), - metrics=metrics_payload, - ) - - -from app.core.auth.capabilities import CALLS_DELETE, CALLS_IMPORT, CALLS_VIEW -from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities - -apply_workspace_route_capabilities( - router, - view_capability=CALLS_VIEW, - manage_capability=CALLS_IMPORT, - delete_capability=CALLS_DELETE, -) +"""CSV-driven call import routes. + +Users upload a CSV plus a per-batch column mapping (CSV header -> system +field). The backend persists a CallImport batch + one CallImportRow per +line, then fans the rows out to the Celery ``imports`` queue where each +row is downloaded using the telephony credential pinned on the batch. +Exotel credentialed imports require a ``recording_url`` on every row; +direct-URL imports (no credential) also require a mapped recording URL. +""" + +from __future__ import annotations + +import csv +import io +import json +import re +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field +from datetime import date, datetime, time, timedelta +from typing import Any, Dict, Iterable, List, Optional, Tuple +from uuid import UUID, uuid4 + +from fastapi import APIRouter, Body, BackgroundTasks, Depends, File, Form, HTTPException, Query, Response, UploadFile, status +from loguru import logger +from sqlalchemy import desc, func, or_ +from sqlalchemy.orm import Session + +from app.config import settings +from app.core.auth import Principal, get_principal +from app.core.auth.rbac import require_admin +from app.database import get_db +from app.db_sharding.sessions import is_sharding_enabled +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.billing.flexprice_service import record_call_import_batch_created +from app.services.call_imports.audit import ( + actor_emails_for_call_import, + emails_for_user_ids, + stamp_call_import_actor, + user_ids_from_call_imports, +) +from app.services.call_imports.dispatch_diagnostics import ( + build_call_import_dispatch_diagnostics, +) +from app.models.database import ( + CallImport, + CallImportRow, + CallImportSchema, + CallImportSchemaParameter, + CallImportTag, + TelephonyIntegration, +) +from app.models.enums import ( + CallImportParameterType, + CallImportRowStatus, + CallImportStatus, +) +from app.models.schemas import ( + CallImportCancelDiarisationRequest, + CallImportCancelDiarisationResponse, + CallImportDetailResponse, + CallImportDeleteResponse, + CallImportDiarisationPromptDefaultResponse, + CallImportDispatchDiagnosticsResponse, + CallImportInsightsMetric, + CallImportInsightsResponse, + CallImportInsightsRunPoint, + CallImportListResponse, + CallImportMappingUpdate, + CallImportMetricAggregate, + CallImportPreviewResponse, + CallImportPreviewSheet, + CallImportRetryFailedRowsRequest, + CallImportRetryFailedRowsResponse, + CallImportResponse, + CallImportRowIdsResponse, + CallImportRowBulkDelete, + CallImportRowBulkDeleteResponse, + CallImportRowResponse, + CallImportStartRequest, + CallImportTranscribeRequest, + CallImportTranscribeResponse, + CallImportUpdate, + CallImportUploadResponse, +) + + +router = APIRouter( + prefix="/call-imports", + tags=["Call Imports"], + dependencies=[Depends(require_enterprise_feature("call_imports"))], +) + + +@dataclass(frozen=True) +class CallImportParseSkip: + """One source row excluded during CSV/Excel parse (identity / recording URL).""" + + source_row: int + reason: str + message: str + + +@dataclass +class CallImportParseResult: + rows: List[Dict[str, Any]] = field(default_factory=list) + skipped: List[CallImportParseSkip] = field(default_factory=list) + + +def parse_skips_to_json(skips: List[CallImportParseSkip]) -> List[Dict[str, Any]]: + """Persistable JSON shape for ``CallImport.source_row_skips``.""" + return [ + { + "source_row": item.source_row, + "reason": item.reason, + "message": item.message, + } + for item in skips + ] + + +def _normalize_dataset(raw: Optional[str]) -> Optional[str]: + """Trim and treat empty strings as 'no dataset' (NULL).""" + if raw is None: + return None + cleaned = raw.strip() + return cleaned or None + + +def _normalize_import_display_name(raw: Optional[str]) -> Optional[str]: + """Trim a user-facing import/batch label; empty clears to NULL.""" + if raw is None: + return None + cleaned = raw.strip() + if not cleaned: + return None + return cleaned[:512] + + +def _serialize_call_import( + db: Session, + call_import: CallImport, + *, + user_emails: Optional[Dict[UUID, str]] = None, +) -> CallImportResponse: + """Catalog parent fields; counters come from SQL rollup (not Redis merge).""" + from app.services.call_imports.bulk_ops import rollup_call_import_batch_status + from app.services.call_imports.progress_counters import ( + clear_import_progress_redis, + read_import_progress, + ) + + redis_completed, redis_failed = read_import_progress(call_import.id) + if ( + redis_completed + or redis_failed + or int(call_import.completed_rows or 0) > int(call_import.total_rows or 0) + or int(call_import.failed_rows or 0) > int(call_import.total_rows or 0) + ): + rollup_call_import_batch_status(db, call_import) + db.flush() + + clear_import_progress_redis(call_import.id) + db.refresh(call_import) + total = int(call_import.total_rows or 0) + completed = min(int(call_import.completed_rows or 0), total) if total else int( + call_import.completed_rows or 0 + ) + failed = min(int(call_import.failed_rows or 0), total) if total else int( + call_import.failed_rows or 0 + ) + if user_emails is None: + user_emails = emails_for_user_ids( + db, user_ids_from_call_imports([call_import]) + ) + created_email, updated_email = actor_emails_for_call_import( + call_import, user_emails + ) + base = CallImportResponse.model_validate(call_import) + return base.model_copy( + update={ + "completed_rows": completed, + "failed_rows": failed, + "created_by_email": created_email, + "last_updated_by_email": updated_email, + } + ) + + +def _resolve_tags( + db: Session, organization_id: UUID, tag_ids: Optional[List[UUID]] +) -> List[CallImportTag]: + """Look up tag rows by id, scoped to the organization. + + Raises HTTPException(400) if any id is unknown for the org. + """ + if not tag_ids: + return [] + rows = ( + db.query(CallImportTag) + .filter( + CallImportTag.organization_id == organization_id, + CallImportTag.id.in_(tag_ids), + ) + .all() + ) + found_ids = {row.id for row in rows} + missing = [str(tag_id) for tag_id in tag_ids if tag_id not in found_ids] + if missing: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unknown call_import_tag id(s): {missing}", + ) + return rows + + +MAX_UPLOAD_BYTES = 15 * 1024 * 1024 # 15 MB upload cap (CSV or Excel) +MAX_AUDIO_FILES_PER_REQUEST = 100 +MANUAL_AUDIO_BLOB_UPLOAD_WORKERS = 10 + +# File extensions accepted by the upload + preview endpoints. Keep in +# lockstep with the frontend ``accept`` attribute on the file picker. +CSV_EXTENSIONS = (".csv",) +XLSX_EXTENSIONS = (".xlsx", ".xlsm") +ALLOWED_EXTENSIONS = CSV_EXTENSIONS + XLSX_EXTENSIONS + +AUDIO_CONTENT_TYPES = { + "wav": "audio/wav", + "mp3": "audio/mpeg", + "flac": "audio/flac", + "m4a": "audio/mp4", +} + + +def _file_format(filename: Optional[str]) -> Optional[str]: + """Classify ``filename`` as ``'csv'`` / ``'xlsx'`` or ``None`` if unsupported.""" + if not filename: + return None + name = filename.lower() + if name.endswith(CSV_EXTENSIONS): + return "csv" + if name.endswith(XLSX_EXTENSIONS): + return "xlsx" + return None + + +def _audio_extension(filename: Optional[str]) -> Optional[str]: + """Return the validated lower-case extension for a manual recording.""" + if not filename or "." not in filename: + return None + ext = filename.rsplit(".", 1)[-1].lower().strip() + allowed = {fmt.lower().lstrip(".") for fmt in settings.ALLOWED_AUDIO_FORMATS} + return ext if ext in allowed else None + + +def _audio_content_type(ext: str, upload_content_type: Optional[str]) -> str: + """Prefer the browser-supplied audio content type, with a safe fallback.""" + supplied = (upload_content_type or "").strip() + if supplied and supplied != "application/octet-stream": + return supplied + return AUDIO_CONTENT_TYPES.get(ext.lower(), "application/octet-stream") + + +def _audio_s3_key( + organization_id: UUID, call_import_id: UUID, row_id: UUID, ext: str +) -> str: + """Build the canonical S3 key for a manually uploaded recording.""" + from app.services.storage.s3_service import s3_service + + return ( + f"{s3_service.prefix}organizations/{organization_id}/call_imports/" + f"{call_import_id}/{row_id}.{ext}" + ) + + +def _filename_stem(filename: Optional[str]) -> str: + """Extract a cross-platform filename stem from an UploadFile name.""" + raw = (filename or "").strip() + basename = re.split(r"[\\/]", raw)[-1] if raw else "" + if "." in basename: + basename = basename.rsplit(".", 1)[0] + return basename.strip() + + +def _sanitize_conversation_id(raw: str) -> str: + """Turn a filename stem into a stable conversation_id.""" + cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", raw.strip()) + cleaned = re.sub(r"_+", "_", cleaned).strip("._-") + return (cleaned or "recording")[:255] + + +def _dedupe_conversation_id( + base: str, counts: Dict[str, int] +) -> str: + """Make conversation ids unique within one manual upload batch.""" + count = counts.get(base, 0) + 1 + counts[base] = count + if count == 1: + return base + suffix = f"-{count}" + return f"{base[: 255 - len(suffix)]}{suffix}" + + +def _conversation_id_base(conversation_id: str) -> str: + match = re.match(r"^(.*)-(\d+)$", (conversation_id or "").strip()) + if match: + return match.group(1) + return (conversation_id or "").strip() + + +def _manual_audio_append_start_state( + db: Session, + call_import: CallImport, +) -> tuple[int, Dict[str, int]]: + """Next row_index and conversation-id suffix counts for audio append.""" + from app.db_sharding.scatter_gather import ( + load_call_import_conversation_ids, + max_call_import_row_index, + ) + + max_index = max_call_import_row_index(db, call_import.id) + start_row_index = max_index + 1 + if start_row_index == 0 and (call_import.total_rows or 0) > 0: + start_row_index = call_import.total_rows + conversation_counts = _seed_conversation_counts( + cid for cid in load_call_import_conversation_ids(db, call_import.id) if cid + ) + return start_row_index, conversation_counts + + +def _seed_conversation_counts(existing_conversation_ids: Iterable[str]) -> Dict[str, int]: + counts: Dict[str, int] = {} + for conversation_id in existing_conversation_ids: + base = _conversation_id_base(conversation_id) + if not base: + continue + counts[base] = counts.get(base, 0) + 1 + return counts + + +def _enforce_audio_upload_file_limit(file_count: int) -> None: + if file_count > MAX_AUDIO_FILES_PER_REQUEST: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"At most {MAX_AUDIO_FILES_PER_REQUEST} audio files may be uploaded " + "per request. Split large batches into multiple requests." + ), + ) + + +async def _prepare_audio_upload_files( + files: List[UploadFile], + conversation_counts: Dict[str, int], +) -> List[Dict[str, Any]]: + if not files: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="At least one audio file is required.", + ) + _enforce_audio_upload_file_limit(len(files)) + + max_bytes = int(settings.MAX_FILE_SIZE_MB) * 1024 * 1024 + prepared: List[Dict[str, Any]] = [] + + for idx, upload in enumerate(files): + filename = upload.filename or f"recording-{idx + 1}" + ext = _audio_extension(filename) + if not ext: + allowed = ", ".join(settings.ALLOWED_AUDIO_FORMATS) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unsupported audio file '{filename}'. Allowed formats: {allowed}.", + ) + + contents = await upload.read() + if not contents: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Audio file '{filename}' is empty.", + ) + if len(contents) > max_bytes: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=( + f"Audio file '{filename}' exceeds " + f"{settings.MAX_FILE_SIZE_MB} MB." + ), + ) + + base_conversation_id = _sanitize_conversation_id(_filename_stem(filename)) + conversation_id = _dedupe_conversation_id( + base_conversation_id, + conversation_counts, + ) + prepared.append( + { + "filename": filename, + "extension": ext, + "content_type": _audio_content_type(ext, upload.content_type), + "contents": contents, + "conversation_id": conversation_id, + } + ) + return prepared + + +def _upload_manual_audio_blobs_parallel( + upload_specs: List[Tuple[str, bytes, str]], +) -> None: + """Upload prepared manual-audio blobs concurrently (key, body, content_type).""" + if not upload_specs: + return + + from app.services.storage.s3_service import s3_service + + def _upload_one(spec: Tuple[str, bytes, str]) -> None: + key, contents, content_type = spec + s3_service.upload_file_by_key( + contents, + key, + content_type=content_type, + ) + + if len(upload_specs) == 1: + _upload_one(upload_specs[0]) + return + + workers = min(MANUAL_AUDIO_BLOB_UPLOAD_WORKERS, len(upload_specs)) + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = [pool.submit(_upload_one, spec) for spec in upload_specs] + for future in as_completed(futures): + future.result() + + +def _persist_prepared_audio_rows( + db: Session, + *, + call_import: CallImport, + organization_id: UUID, + workspace_id: UUID, + prepared: List[Dict[str, Any]], + start_row_index: int, +) -> List[str]: + uploaded_keys: List[str] = [] + row_mappings: List[Dict[str, Any]] = [] + upload_specs: List[Tuple[str, bytes, str]] = [] + + for offset, item in enumerate(prepared): + row_id = uuid4() + key = _audio_s3_key( + organization_id, + call_import.id, + row_id, + item["extension"], + ) + uploaded_keys.append(key) + upload_specs.append((key, item["contents"], item["content_type"])) + row_mappings.append( + { + "id": row_id, + "call_import_id": call_import.id, + "organization_id": organization_id, + "workspace_id": workspace_id, + "row_index": start_row_index + offset, + "conversation_id": item["conversation_id"], + "recording_url": None, + "transcript": None, + "transcript_source": None, + "raw_columns": {"conversation_id": item["conversation_id"]}, + "status": CallImportRowStatus.COMPLETED, + "recording_s3_key": key, + "recording_content_type": item["content_type"], + "recording_size_bytes": len(item["contents"]), + } + ) + + _upload_manual_audio_blobs_parallel(upload_specs) + + added_size = sum(len(item["contents"]) for item in prepared) + new_total_rows = (call_import.total_rows or 0) + len(prepared) + + if is_sharding_enabled(): + from app.db_sharding.row_ops import ( + bulk_insert_mappings_on_shards, + register_shard_slices, + ) + + bulk_insert_mappings_on_shards(db, call_import.id, row_mappings) + register_shard_slices(db, call_import.id, new_total_rows) + else: + for mapping in row_mappings: + db.add(CallImportRow(**mapping)) + + call_import.total_rows = new_total_rows + call_import.completed_rows = (call_import.completed_rows or 0) + len(prepared) + call_import.source_size_bytes = (call_import.source_size_bytes or 0) + added_size + return uploaded_keys + + +def _normalize_header(name: str) -> str: + return (name or "").strip().lower() + + +def _header_lookup(fieldnames: List[str]) -> Dict[str, str]: + """Map normalized header -> original header for case-insensitive lookup.""" + return {_normalize_header(h): h for h in fieldnames or []} + + +def _resolve_mapped_header( + mapping_value: Optional[str], header_lookup: Dict[str, str] +) -> Optional[str]: + """Translate a user-supplied CSV header into the actual column key. + + The frontend sends headers exactly as they appear in the source file, + but we still normalize on the server so trailing whitespace / casing + doesn't break matching. Returns the canonical fieldname or ``None`` + if not present in the file. + """ + if not mapping_value: + return None + return header_lookup.get(_normalize_header(mapping_value)) + + +def _xlsx_cell_to_str(value: Any) -> str: + """Coerce an openpyxl cell value to the string the rest of the + pipeline expects. + + openpyxl returns native Python types (int, float, datetime, bool, + None). The CSV path always works with strings, so we mirror that: + integers stringify cleanly (no ``.0`` suffix on whole-number floats), + datetimes use ISO-8601, booleans use SQL-style ``TRUE`` / ``FALSE``. + """ + if value is None: + return "" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + if isinstance(value, int): + return str(value) + if isinstance(value, float): + if value.is_integer(): + return str(int(value)) + return str(value) + if isinstance(value, datetime): + return value.isoformat() + if isinstance(value, date): + return value.isoformat() + if isinstance(value, time): + return value.isoformat() + if isinstance(value, timedelta): + return str(value) + return str(value) + + +def _parse_recording_date_cell(cell: str) -> date: + """Parse day-first dates with one/two digit day-month parts.""" + match = re.fullmatch(r"\s*(\d{1,2})[/-](\d{1,2})[/-](\d{4})\s*", cell) + if match: + day, month, year = (int(part) for part in match.groups()) + return date(year, month, day) + + # Native Excel date cells arrive from ``_xlsx_cell_to_str`` as ISO + # datetimes (e.g. ``2026-01-04T00:00:00``). Accept that resolved date, + # while keeping plain ISO dates rejected for hand-entered text/CSV cells. + if "T" in cell: + return datetime.fromisoformat(cell.replace("Z", "+00:00")).date() + + raise ValueError("expected D/M/YYYY or D-M-YYYY") + + +def _coerce_parameter_value( + raw: str, + param_type: CallImportParameterType, + *, + row_idx: int, + param_name: str, +) -> Any: + """Validate + coerce a single CSV cell against its declared type. + + Returns the typed Python value to surface in ``raw_columns``. Empty + strings are returned as ``None`` regardless of the parameter type so + optional cells stay null end-to-end. Coercion failures raise a + 400 with a row-anchored message. + """ + cell = (raw or "").strip() + if not cell: + return None + + if param_type == CallImportParameterType.CONVERSATION_ID: + return cell + if param_type == CallImportParameterType.RECORDING_URL: + # Recording URLs are exercised by the worker (which downloads + # them); we only do a light "starts with http" check here so a + # paste-error surfaces immediately at upload time. + lower = cell.lower() + if not (lower.startswith("http://") or lower.startswith("https://")): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + "valid recording URL (must start with http:// or https://)." + ), + ) + return cell + if param_type == CallImportParameterType.RECORDING_DATE: + try: + parsed_date = _parse_recording_date_cell(cell) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + f"valid recording date ({cell!r}); expected day-first " + "D/M/YYYY or D-M-YYYY." + ), + ) + return parsed_date.strftime("%d/%m/%Y") + if param_type == CallImportParameterType.TRANSCRIPT: + return cell + if param_type == CallImportParameterType.TEXT: + return cell + if param_type == CallImportParameterType.NUMBER: + try: + value = float(cell) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + f"valid number ({cell!r})." + ), + ) + if value.is_integer(): + return int(value) + return value + if param_type == CallImportParameterType.BOOLEAN: + truthy = {"true", "yes", "y", "1", "t"} + falsy = {"false", "no", "n", "0", "f"} + norm = cell.lower() + if norm in truthy: + return True + if norm in falsy: + return False + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + f"valid boolean ({cell!r})." + ), + ) + if param_type == CallImportParameterType.DATETIME: + try: + parsed = datetime.fromisoformat(cell.replace("Z", "+00:00")) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + f"valid ISO-8601 date/time ({cell!r})." + ), + ) + return parsed.isoformat() + if param_type == CallImportParameterType.URL: + lower = cell.lower() + if not (lower.startswith("http://") or lower.startswith("https://")): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + "valid URL (must start with http:// or https://)." + ), + ) + return cell + # Unknown types: store as text and let the next migration catch up. + return cell + + +def _recording_url_cell_is_valid_http(raw: str) -> bool: + cell = (raw or "").strip() + if not cell: + return False + lower = cell.lower() + return lower.startswith("http://") or lower.startswith("https://") + + +def _parameter_is_required(param: CallImportSchemaParameter) -> bool: + """Return whether a schema parameter must be mapped on every upload.""" + if param.is_required: + return True + try: + param_type = CallImportParameterType(param.type) + except ValueError: + return False + return param_type == CallImportParameterType.CONVERSATION_ID + + +def _apply_schema_mapping( + fieldnames: List[str], + rows_iter: Iterable[Dict[str, str]], + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, str], + skipped_columns: List[str], + *, + source_label: str = "CSV", + validate_only: bool = False, +) -> CallImportParseResult: + """Schema-driven row projection: parameter -> CSV header -> typed value. + + Validates that every required schema parameter is mapped to a CSV + header that actually exists in the file, and that every CSV header + is either mapped to a parameter or explicitly listed in + ``skipped_columns``. Returns one dict per non-empty data row with: + + * ``conversation_id`` (str, mandatory) + * ``recording_date`` (Optional[str], DD/MM/YYYY date) + * ``recording_url`` (Optional[str]) + * ``transcript`` (Optional[str]) + * ``parameter_values`` (Dict[str, Any]) of typed values keyed by + parameter name (drives ``raw_columns`` so the export can + reproduce the source). + + ``validate_only=True`` runs the header / mapping / skipped-column + checks (every check that doesn't need to read row data) and then + returns an empty list — used by the MAP stage to validate a + mapping payload against the cached sheet snapshot without + re-fetching the source bytes from S3. + """ + header_lookup = _header_lookup(list(fieldnames)) + + # 1. Look up the conversation_id parameter so we can address it + # directly while building each row. + conv_param = next( + (p for p in parameters if p.type == CallImportParameterType.CONVERSATION_ID), + None, + ) + if conv_param is None: + # The schema invariant should have caught this on create/update, + # but a defensive 400 here keeps us safe against hand-rolled + # API callers that bypassed validation. + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Selected schema is missing the mandatory conversation_id parameter.", + ) + # 2. Resolve every mapped parameter to a canonical fieldname. + # Required parameters MUST resolve; optional ones may resolve to + # None if the user left them blank (no mapping). + canonical_by_param: Dict[str, Optional[str]] = {} + recording_date_param_name: Optional[str] = None + rec_url_param_name: Optional[str] = None + transcript_param_name: Optional[str] = None + for param in parameters: + mapped_header = parameter_mapping.get(param.name) + canonical = ( + _resolve_mapped_header(mapped_header, header_lookup) + if mapped_header + else None + ) + if _parameter_is_required(param) and canonical is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"{source_label} does not contain the column " + f"'{mapped_header or ''}' mapped to required parameter " + f"'{param.name}'." + ), + ) + canonical_by_param[param.name] = canonical + if param.type == CallImportParameterType.RECORDING_DATE: + recording_date_param_name = param.name + elif param.type == CallImportParameterType.RECORDING_URL: + rec_url_param_name = param.name + elif param.type == CallImportParameterType.TRANSCRIPT: + transcript_param_name = param.name + + # 3. Every CSV column must either be mapped to a parameter or + # explicitly skipped. Catches "I forgot to skip the email + # column" gracefully instead of dropping data silently. + mapped_canonicals = {c for c in canonical_by_param.values() if c} + skipped_canonicals = { + _resolve_mapped_header(h, header_lookup) + for h in skipped_columns + } + skipped_canonicals.discard(None) + unhandled = [ + h + for h in fieldnames + if h not in mapped_canonicals and h not in skipped_canonicals + ] + if unhandled: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"{source_label} columns must either be mapped to a schema " + f"parameter or explicitly skipped. Unhandled: {unhandled}." + ), + ) + + conv_canonical = canonical_by_param[conv_param.name] + rec_canonical = ( + canonical_by_param.get(rec_url_param_name) + if rec_url_param_name + else None + ) + recording_date_canonical = ( + canonical_by_param.get(recording_date_param_name) + if recording_date_param_name + else None + ) + transcript_canonical = ( + canonical_by_param.get(transcript_param_name) + if transcript_param_name + else None + ) + + if validate_only: + # MAP-stage validation: every header check above has already + # run; the row loop only matters at IMPORT time. Skip it (and + # the "no data rows" guard at the bottom of the function) so + # the caller gets a clean pass when the mapping is shaped right. + return CallImportParseResult() + + parsed: List[Dict[str, Any]] = [] + skipped: List[CallImportParseSkip] = [] + for idx, row in enumerate(rows_iter): + # Drop fully-blank lines - matches the legacy parser behavior so + # trailing-newline edge cases don't fail an otherwise-good upload. + non_blank = any( + (row.get(c) or "").strip() + for c in mapped_canonicals + if c + ) + if not non_blank: + continue + + source_row = idx + 1 + conv_value = (row.get(conv_canonical) or "").strip() if conv_canonical else "" + if not conv_value: + skipped.append( + CallImportParseSkip( + source_row=source_row, + reason="missing_conversation_id", + message=( + f"Row {source_row} is missing the '{conv_param.name}' " + "(conversation_id) value." + ), + ) + ) + continue + + if rec_canonical and rec_url_param_name: + rec_param = next( + (p for p in parameters if p.name == rec_url_param_name), + None, + ) + if rec_param is not None and _parameter_is_required(rec_param): + rec_raw = (row.get(rec_canonical) or "").strip() + if not rec_raw: + skipped.append( + CallImportParseSkip( + source_row=source_row, + reason="missing_recording_url", + message=( + f"Row {source_row} is missing the required " + f"'{rec_url_param_name}' value." + ), + ) + ) + continue + if not _recording_url_cell_is_valid_http(rec_raw): + skipped.append( + CallImportParseSkip( + source_row=source_row, + reason="invalid_recording_url", + message=( + f"Row {source_row}: value for " + f"'{rec_url_param_name}' is not a valid recording " + "URL (must start with http:// or https://)." + ), + ) + ) + continue + + # Materialize every mapped parameter into the per-row snapshot, + # running per-type coercion so a bad cell aborts the upload + # rather than silently storing garbage. + parameter_values: Dict[str, Any] = {} + row_skipped = False + for param in parameters: + canonical = canonical_by_param[param.name] + if canonical is None: + continue + try: + param_type = CallImportParameterType(param.type) + except ValueError: + param_type = CallImportParameterType.TEXT + coerced = _coerce_parameter_value( + row.get(canonical) or "", + param_type, + row_idx=idx, + param_name=param.name, + ) + if _parameter_is_required(param) and coerced is None: + if param_type == CallImportParameterType.RECORDING_URL: + skipped.append( + CallImportParseSkip( + source_row=source_row, + reason="missing_recording_url", + message=( + f"Row {source_row} is missing the required " + f"'{param.name}' value." + ), + ) + ) + row_skipped = True + break + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {source_row} is missing the required " + f"'{param.name}' value." + ), + ) + parameter_values[param.name] = coerced + if row_skipped: + continue + + rec_value = ( + (row.get(rec_canonical) or "").strip() if rec_canonical else "" + ) + transcript_value = ( + (row.get(transcript_canonical) or "").strip() + if transcript_canonical + else "" + ) + recording_date_value = ( + parameter_values.get(recording_date_param_name) + if recording_date_param_name + else None + ) + + parsed.append( + { + "conversation_id": conv_value, + "recording_date": recording_date_value, + "recording_url": rec_value or None, + "transcript": transcript_value or None, + "parameter_values": parameter_values, + } + ) + + if not parsed and not skipped: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"{source_label} did not contain any data rows.", + ) + + return CallImportParseResult(rows=parsed, skipped=skipped) + + +def _raise_if_no_importable_rows( + result: CallImportParseResult, *, source_label: str = "CSV" +) -> None: + """Sync upload / API callers fail fast when every data row was skipped.""" + if result.rows: + return + if result.skipped: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"No importable rows. {len(result.skipped)} row(s) skipped due " + "to missing or invalid conversation ID or recording URL." + ), + ) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"{source_label} did not contain any data rows.", + ) + + +def _parse_csv( + file_bytes: bytes, + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, str], + skipped_columns: List[str], +) -> CallImportParseResult: + """Parse a CSV file using the resolved schema parameters.""" + if not file_bytes: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Uploaded CSV is empty.", + ) + + try: + text_stream = io.StringIO(file_bytes.decode("utf-8-sig")) + except UnicodeDecodeError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="CSV must be UTF-8 encoded.", + ) + + reader = csv.DictReader(text_stream) + if not reader.fieldnames: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="CSV is missing a header row.", + ) + + return _apply_schema_mapping( + list(reader.fieldnames), + reader, + parameters, + parameter_mapping, + skipped_columns, + source_label="CSV", + ) + + +def _open_xlsx_workbook(file_bytes: bytes): + """Open an xlsx/xlsm workbook from in-memory bytes (read-only stream). + + Imports openpyxl lazily so the module loads even in environments that + haven't installed the optional dep yet (e.g. lightweight tooling + images). Surfaces a clean 400 if openpyxl is missing or the file is + not a valid Office Open XML workbook. + """ + if not file_bytes: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Uploaded Excel file is empty.", + ) + try: + from openpyxl import load_workbook # type: ignore + from openpyxl.utils.exceptions import InvalidFileException # type: ignore + except ImportError as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=( + "Excel uploads require the 'openpyxl' package which is " + "not installed in this environment." + ), + ) from exc + + try: + return load_workbook( + io.BytesIO(file_bytes), + read_only=True, + data_only=True, + ) + except InvalidFileException as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"File is not a valid .xlsx workbook: {exc}", + ) from exc + except Exception as exc: # zipfile.BadZipFile etc. + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Could not open Excel workbook: {exc}", + ) from exc + + +def _xlsx_sheet_headers_and_rows( + worksheet, +) -> Tuple[List[str], List[Dict[str, str]]]: + """Read row 1 as headers and the rest as dicts of stringified cells. + + Empty trailing header cells are dropped. Duplicate headers preserve + the first occurrence (matches ``csv.DictReader`` behavior, which + silently drops duplicates). + """ + iterator = worksheet.iter_rows(values_only=True) + try: + header_row = next(iterator) + except StopIteration: + return [], [] + + headers: List[str] = [] + seen: set[str] = set() + for cell in header_row: + name = _xlsx_cell_to_str(cell).strip() + if not name: + # Stop at the first blank header — treats trailing empty + # columns as not part of the table (matches typical Excel + # workbook conventions). + break + norm = name.lower() + if norm in seen: + continue + seen.add(norm) + headers.append(name) + + rows: List[Dict[str, str]] = [] + for row in iterator: + if row is None: + continue + # Pad / truncate to the header length so dict construction is + # stable even when a row has fewer / extra cells than the header. + cells = list(row[: len(headers)]) + if len(cells) < len(headers): + cells.extend([None] * (len(headers) - len(cells))) + if not any(_xlsx_cell_to_str(c).strip() for c in cells): + # Skip fully-blank rows (openpyxl read_only routinely yields + # trailing empties when the worksheet's used range exceeds + # the actual data). + continue + rows.append( + { + header: _xlsx_cell_to_str(value) + for header, value in zip(headers, cells) + } + ) + + return headers, rows + + +def _parse_xlsx( + file_bytes: bytes, + sheet_name: Optional[str], + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, str], + skipped_columns: List[str], +) -> CallImportParseResult: + """Parse a single worksheet from an xlsx/xlsm workbook. + + ``sheet_name`` must match one of the workbook's sheets (case + insensitive whitespace-trimmed match). Returns the same shape as + :func:`_parse_csv` so the upload handler can persist either format + through the same code path. + """ + if not sheet_name or not sheet_name.strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is required when uploading an Excel workbook.", + ) + + workbook = _open_xlsx_workbook(file_bytes) + try: + sheet_names = list(workbook.sheetnames) + target_norm = sheet_name.strip().lower() + match = next( + (s for s in sheet_names if s.strip().lower() == target_norm), + None, + ) + if match is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Sheet '{sheet_name}' not found in workbook. " + f"Available sheets: {sheet_names}" + ), + ) + worksheet = workbook[match] + headers, rows = _xlsx_sheet_headers_and_rows(worksheet) + finally: + workbook.close() + + if not headers: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Sheet '{sheet_name}' is missing a header row.", + ) + + return _apply_schema_mapping( + headers, + rows, + parameters, + parameter_mapping, + skipped_columns, + source_label=f"Sheet '{sheet_name}'", + ) + + +def _csv_preview_sheets( + file_bytes: bytes, filename: Optional[str] +) -> List[CallImportPreviewSheet]: + """Build the synthetic single-sheet preview entry for a CSV upload.""" + if not file_bytes: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Uploaded CSV is empty.", + ) + try: + text_stream = io.StringIO(file_bytes.decode("utf-8-sig")) + except UnicodeDecodeError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="CSV must be UTF-8 encoded.", + ) + reader = csv.DictReader(text_stream) + if not reader.fieldnames: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="CSV is missing a header row.", + ) + headers = list(reader.fieldnames) + row_count = 0 + for row in reader: + # Match the parse-time skip: ignore fully blank rows so the + # count the user sees lines up with what /upload will ingest. + if any((v or "").strip() for v in row.values()): + row_count += 1 + + sheet_label = (filename or "sheet1").rsplit("/", 1)[-1] or "sheet1" + return [ + CallImportPreviewSheet( + name=sheet_label, + headers=headers, + row_count=row_count, + ) + ] + + +def _xlsx_preview_sheets(file_bytes: bytes) -> List[CallImportPreviewSheet]: + """List every worksheet in the workbook with its headers and row count.""" + workbook = _open_xlsx_workbook(file_bytes) + sheets: List[CallImportPreviewSheet] = [] + try: + for name in workbook.sheetnames: + worksheet = workbook[name] + headers, rows = _xlsx_sheet_headers_and_rows(worksheet) + sheets.append( + CallImportPreviewSheet( + name=name, + headers=headers, + row_count=len(rows), + ) + ) + finally: + workbook.close() + return sheets + + +def _parse_json_form_field(name: str, raw: Optional[str], default): + """Decode a JSON-encoded form field with a friendly 400 on bad JSON.""" + if raw is None or raw == "": + return default + try: + return json.loads(raw) + except json.JSONDecodeError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"{name} must be valid JSON: {exc}", + ) + + +# --------------------------------------------------------------------------- +# Shared helpers used by the staged endpoints (UPLOAD / MAP / IMPORT) and the +# legacy one-shot ``POST /upload`` shim. Extracted here so each stage and the +# back-compat path operate on the exact same validation + persistence code. +# --------------------------------------------------------------------------- + + +def _source_content_type(fmt: str) -> str: + """Return the canonical ``Content-Type`` for a parsed file format.""" + if fmt == "csv": + return "text/csv" + if fmt == "xlsx": + return ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ) + return "application/octet-stream" + + +def _source_s3_key( + organization_id: UUID, call_import_id: UUID, fmt: str +) -> str: + """Build the canonical S3 key for an upload's source file. + + Mirrors the per-row recording key convention used by + ``process_call_import_row`` so a single prefix sweep on delete still + cleans up both the source artefact and every fetched recording. + """ + from app.services.storage.s3_service import s3_service + + ext = "xlsx" if fmt == "xlsx" else "csv" + return ( + f"{s3_service.prefix}organizations/{organization_id}/call_imports/" + f"{call_import_id}/source.{ext}" + ) + + +def _build_available_sheets( + file_bytes: bytes, fmt: str, filename: Optional[str] +) -> List[CallImportPreviewSheet]: + """Snapshot of sheets + headers cached on the batch at UPLOAD time.""" + if fmt == "csv": + return _csv_preview_sheets(file_bytes, filename) + return _xlsx_preview_sheets(file_bytes) + + +def _resolve_schema( + db: Session, + organization_id: UUID, + workspace_id: UUID, + schema_id: UUID, +) -> CallImportSchema: + """Fetch + validate a schema row in the active workspace. + + Eager-loads ``parameters`` so callers can iterate without re-querying. + """ + from sqlalchemy.orm import selectinload as _selectinload + + schema = ( + db.query(CallImportSchema) + .options(_selectinload(CallImportSchema.parameters)) + .filter( + CallImportSchema.id == schema_id, + CallImportSchema.organization_id == organization_id, + CallImportSchema.workspace_id == workspace_id, + ) + .first() + ) + if not schema: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Call import schema not found in the active workspace.", + ) + if not list(schema.parameters): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Selected schema has no parameters defined.", + ) + return schema + + +def _validate_direct_url_import_ready( + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, Any], +) -> None: + """Ensure direct-URL import has a mapped recording_url when required.""" + rec_url_param = next( + ( + p + for p in parameters + if p.type == CallImportParameterType.RECORDING_URL.value + ), + None, + ) + if rec_url_param is None: + return + if not _parameter_is_required(rec_url_param): + return + mapped_header = (parameter_mapping or {}).get(rec_url_param.name) + if not (mapped_header or "").strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Direct URL import requires the 'recording_url' parameter to " + "be mapped to a source column." + ), + ) + + +def _validate_exotel_import_ready( + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, Any], +) -> None: + """Ensure Exotel credentialed import has a mapped recording_url column.""" + rec_url_param = next( + ( + p + for p in parameters + if p.type == CallImportParameterType.RECORDING_URL.value + ), + None, + ) + if rec_url_param is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Exotel import requires a schema parameter of type " + "'recording_url'." + ), + ) + mapped_header = (parameter_mapping or {}).get(rec_url_param.name) + if not (mapped_header or "").strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Exotel import requires the 'recording_url' parameter to " + "be mapped to a source column." + ), + ) + + +def _is_manual_audio_call_import(call_import: CallImport) -> bool: + """True for batches created via manual audio upload (recordings already in S3).""" + return (call_import.source_format or "").lower() == "audio" + + +def _validate_diarised_eval_recording_ready( + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, Any], +) -> None: + """Ensure diarised evaluation can fetch recordings from the mapped batch.""" + rec_url_param = next( + ( + p + for p in parameters + if p.type == CallImportParameterType.RECORDING_URL.value + ), + None, + ) + if rec_url_param is None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "Diarize then evaluate requires a schema parameter of type " + "'recording_url'. Add one to the schema and map it to a " + "source column." + ), + ) + mapped_header = (parameter_mapping or {}).get(rec_url_param.name) + if not (mapped_header or "").strip(): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "Diarize then evaluate requires the " + f"'{rec_url_param.name}' recording URL parameter to be " + "mapped to a source column." + ), + ) + + +def _resolve_telephony_integration( + db: Session, + organization_id: UUID, + telephony_integration_id: UUID, + provider: str, +) -> TelephonyIntegration: + """Fetch + validate a telephony credential against the requested provider.""" + integration = ( + db.query(TelephonyIntegration) + .filter( + TelephonyIntegration.id == telephony_integration_id, + TelephonyIntegration.organization_id == organization_id, + ) + .first() + ) + if not integration: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Telephony credential not found for this organization.", + ) + if (integration.provider or "").lower() != provider.lower(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Selected credential is for provider '{integration.provider}', " + f"but request specified '{provider}'." + ), + ) + if not integration.is_active: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Selected telephony credential is inactive.", + ) + return integration + + +def _is_stuck_pending_import_row(row: CallImportRow) -> bool: + """True when a pending row exhausted retries or hit credential errors.""" + from app.models.enums import CallImportRowStatus + + if row.status != CallImportRowStatus.PENDING: + return False + if (row.attempts or 0) > 0: + return True + message = (row.error_message or "").lower() + auth_indicators = ( + "rejected credentials", + "telephony credentials rejected", + "auth failed", + "recording fetch failed after", + ) + return any(indicator in message for indicator in auth_indicators) + + +def _validate_telephony_credentials_live( + db: Session, + organization_id: UUID, + integration: TelephonyIntegration, +) -> None: + """Verify telephony credentials against the provider API before import work.""" + from app.services.telephony.telephony_service import telephony_service + + provider_label = (integration.provider or "telephony").title() + try: + client = telephony_service.get_provider_client( + organization_id, + db, + provider=integration.provider, + credential_id=integration.id, + ) + client.test_connection() + except HTTPException: + raise + except Exception as exc: + detail = str(exc).strip() or "Unknown error" + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"{provider_label} credentials could not be verified: {detail}. " + "Update credentials in Integrations and retry." + ), + ) from exc + + +def _clean_parameter_mapping( + mapping_payload: Any, + parameters: List[CallImportSchemaParameter], + schema_name: str, +) -> Dict[str, str]: + """Trim values and drop empties; reject unknown parameter names. + + Accepts an already-decoded value (dict-shaped) so the same helper + works for the JSON-form upload path and the JSON-body PATCH path. + """ + if not isinstance(mapping_payload, dict) or not all( + isinstance(k, str) and (v is None or isinstance(v, str)) + for k, v in mapping_payload.items() + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "parameter_mapping must be an object of " + "{parameter_name: csv_header}." + ), + ) + + valid_param_names = {p.name for p in parameters} + cleaned: Dict[str, str] = {} + for raw_name, raw_header in mapping_payload.items(): + if raw_name not in valid_param_names: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"parameter_mapping references unknown parameter " + f"'{raw_name}' on schema '{schema_name}'." + ), + ) + header = (raw_header or "").strip() + if header: + cleaned[raw_name] = header + return cleaned + + +def _clean_skipped_columns(skipped_payload: Any) -> List[str]: + """Dedupe (case-insensitively) and drop blanks; preserve original casing.""" + if not isinstance(skipped_payload, list) or not all( + isinstance(item, str) for item in skipped_payload + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="skipped_columns must be a list of header strings.", + ) + cleaned: List[str] = [] + seen: set[str] = set() + for item in skipped_payload: + norm = _normalize_header(item) + if not norm or norm in seen: + continue + seen.add(norm) + cleaned.append(item) + return cleaned + + +def _parse_source_file( + file_bytes: bytes, + fmt: str, + sheet_name: Optional[str], + parameters: List[CallImportSchemaParameter], + cleaned_mapping: Dict[str, str], + cleaned_skipped: List[str], +) -> CallImportParseResult: + """Run the format-appropriate parser against a buffer of file bytes.""" + if fmt == "csv": + return _parse_csv(file_bytes, parameters, cleaned_mapping, cleaned_skipped) + return _parse_xlsx( + file_bytes, sheet_name, parameters, cleaned_mapping, cleaned_skipped + ) + + +def _materialize_rows( + db: Session, + call_import: CallImport, + parsed_rows: List[Dict[str, Any]], + organization_id: UUID, +) -> List[CallImportRow]: + """Insert one ``CallImportRow`` per parsed row, returning the new models.""" + row_models: List[CallImportRow] = [] + for idx, row in enumerate(parsed_rows): + # Stamp ``transcript_source='csv'`` when the upload actually + # provided a transcript so the UI badge ("From CSV") works from + # day one. Blank cells stay NULL so the row reads as "no + # production transcript yet". + csv_transcript = row["transcript"] + row_model = CallImportRow( + call_import_id=call_import.id, + organization_id=organization_id, + workspace_id=call_import.workspace_id, + row_index=idx, + conversation_id=row["conversation_id"], + recording_date=( + _parse_recording_date_cell(row["recording_date"]) + if row.get("recording_date") + else None + ), + recording_url=row["recording_url"], + transcript=csv_transcript, + transcript_source=( + "csv" if csv_transcript and csv_transcript.strip() else None + ), + raw_columns=row["parameter_values"] or None, + status=CallImportRowStatus.PENDING, + ) + db.add(row_model) + row_models.append(row_model) + return row_models + + +def _enqueue_row_tasks( + db: Session, + call_import: CallImport, + row_models: List[CallImportRow], +) -> None: + """Schedule fair round-robin dispatch for pending import rows.""" + del db, call_import, row_models + from app.workers.concurrency.fair_import_dispatch import ( + schedule_fair_import_dispatch, + ) + + schedule_fair_import_dispatch(max_workspace_turns=999) + + +def _ensure_blob_storage_enabled() -> None: + """Hard-fail UPLOAD if cloud blob storage isn't configured (no local fallback).""" + from app.services.storage.s3_service import s3_service + + if not s3_service.is_enabled(): + err = ( + s3_service.get_status_message() + or "Cloud blob storage is not enabled or not configured" + ) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=( + "Call uploads require cloud blob storage so the file can be " + f"persisted between stages: {err}" + ), + ) + + +def _validate_sheet_choice( + fmt: str, + sheet_name: Optional[str], + available_sheets: Optional[List[Dict[str, Any]]], +) -> Optional[str]: + """Normalize / validate ``sheet_name`` against the persisted snapshot. + + Returns the canonical sheet name (matching the workbook's casing) + so downstream parsing addresses the right worksheet. + """ + if fmt == "csv": + if sheet_name and sheet_name.strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is not applicable to CSV uploads.", + ) + return None + + cleaned = (sheet_name or "").strip() or None + if cleaned is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is required when the source is an Excel workbook.", + ) + + if not available_sheets: + # Nothing to validate against (e.g. legacy batch without snapshot); + # let downstream parsing error out instead of silently importing. + return cleaned + + target = cleaned.strip().lower() + for entry in available_sheets: + name = entry.get("name") if isinstance(entry, dict) else None + if isinstance(name, str) and name.strip().lower() == target: + return name + sheet_names = [ + entry.get("name") for entry in available_sheets if isinstance(entry, dict) + ] + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Sheet '{cleaned}' not found in the staged file. " + f"Available sheets: {sheet_names}" + ), + ) + + +def _tag_response_payload(tags: Optional[List[CallImportTag]]) -> List[Dict[str, Any]]: + """Shape a CallImport's tag relationship for the upload response.""" + return [ + { + "id": tag.id, + "name": tag.name, + "color": tag.color, + "created_at": tag.created_at, + "updated_at": tag.updated_at, + } + for tag in (tags or []) + ] + + +@router.post( + "/preview", + response_model=CallImportPreviewResponse, + operation_id="previewCallImportFile", +) +async def preview_call_import_file( + file: UploadFile = File(...), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> CallImportPreviewResponse: + """Inspect an uploaded CSV / Excel file and return its sheets + headers. + + Drives the column-mapping UI without forcing the frontend to parse + CSV / xlsx itself — keeps client and server in lockstep on quoted + fields, encodings, and Excel cell coercion. CSVs return a single + synthetic sheet named after the filename; Excel workbooks return one + entry per worksheet (in workbook order). + """ + del api_key, organization_id, workspace_id, db # auth only + + fmt = _file_format(file.filename) + if fmt is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Unsupported file format. Allowed extensions: " + f"{', '.join(ALLOWED_EXTENSIONS)}." + ), + ) + + file_bytes = await file.read() + if len(file_bytes) > MAX_UPLOAD_BYTES: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", + ) + + if fmt == "csv": + sheets = _csv_preview_sheets(file_bytes, file.filename) + else: + sheets = _xlsx_preview_sheets(file_bytes) + + return CallImportPreviewResponse(format=fmt, sheets=sheets) + + +@router.post( + "", + response_model=CallImportResponse, + status_code=status.HTTP_201_CREATED, + operation_id="createCallImport", +) +async def create_call_import( + file: UploadFile = File( + ..., + description="CSV / Excel file to stage. Persisted to S3 between stages.", + ), + dataset: str = Form( + ..., + description=( + "Required free-text dataset label. Collected up-front so the " + "batch is filterable from the moment it lands." + ), + ), + tag_ids: Optional[List[UUID]] = Form( + None, + description="Optional list of CallImportTag ids to attach to the new batch.", + ), + schema_id: Optional[UUID] = Form( + None, + description=( + "Optional schema pre-pick. The user can still change it during " + "the MAP stage; provided here only so the detail page can pre-" + "select the schema dropdown." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportResponse: + """UPLOAD stage of the staged call-import flow. + + Persists the source file to S3 and creates a ``CallImport`` row with + ``status='uploaded'``. No mapping, no provider, no rows yet — the + user moves through MAP and IMPORT as separate idempotent steps. + + Dataset is collected here (rather than at IMPORT) so the batch is + filterable from the moment it appears in the list view. + """ + del api_key + + fmt = _file_format(file.filename) + if fmt is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Unsupported file format. Allowed extensions: " + f"{', '.join(ALLOWED_EXTENSIONS)}." + ), + ) + + normalized_dataset = _normalize_dataset(dataset) + if not normalized_dataset: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="dataset is required and must be a non-empty string.", + ) + + file_bytes = await file.read() + if len(file_bytes) > MAX_UPLOAD_BYTES: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", + ) + + # Parse-now so we (a) reject garbage uploads up-front instead of + # later in the MAP step, and (b) capture the sheets snapshot the + # MAP UI needs without having to re-fetch the file from S3. + sheets = _build_available_sheets(file_bytes, fmt, file.filename) + + # Optional schema pre-pick: validated only if supplied (the user is + # allowed to set it for the first time during MAP). + if schema_id is not None: + _resolve_schema(db, organization_id, workspace_id, schema_id) + + tag_rows = _resolve_tags(db, organization_id, tag_ids) + + _ensure_blob_storage_enabled() + + # Pre-generate the id so we can compute a deterministic S3 key + # before the row is persisted, keeping ``source_s3_key`` consistent + # with the prefix sweep used at delete-time. + import uuid as _uuid + + call_import_id = _uuid.uuid4() + s3_key = _source_s3_key(organization_id, call_import_id, fmt) + content_type = _source_content_type(fmt) + + from app.services.storage.s3_service import s3_service, StorageError + + try: + s3_service.upload_file_by_key(file_bytes, s3_key, content_type=content_type) + except StorageError as exc: + logger.exception( + "Failed to upload source file to S3 for new call import {}", + call_import_id, + ) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=f"Failed to persist upload to S3: {exc}", + ) + + call_import = CallImport( + id=call_import_id, + organization_id=organization_id, + workspace_id=workspace_id, + # Provider + credential aren't known until the IMPORT stage; leave + # them NULL so the staged-vs-legacy distinction is visible at a + # glance from the DB. + provider=None, + telephony_integration_id=None, + original_filename=file.filename, + sheet_name=None, + dataset=normalized_dataset, + schema_id=schema_id, + parameter_mapping={}, + skipped_columns=[], + column_mapping={}, + extra_columns=[], + custom_column_mapping={}, + source_s3_key=s3_key, + source_format=fmt, + source_size_bytes=len(file_bytes), + source_content_type=content_type, + available_sheets=[sheet.model_dump() for sheet in sheets], + total_rows=0, + completed_rows=0, + failed_rows=0, + status=CallImportStatus.UPLOADED, + ) + if tag_rows: + call_import.tags = tag_rows + + stamp_call_import_actor(call_import, principal, creating=True) + db.add(call_import) + try: + db.commit() + except Exception: + db.rollback() + # Best-effort cleanup of the uploaded S3 object so a failed + # commit doesn't leak storage. + try: + s3_service.delete_file_by_key(s3_key) + except Exception as cleanup_exc: # noqa: BLE001 + logger.warning( + "Failed to clean up orphaned S3 object {} after DB rollback: {}", + s3_key, + cleanup_exc, + ) + raise + + db.refresh(call_import) + return _serialize_call_import(db, call_import) + + +@router.patch( + "/{call_import_id}/mapping", + response_model=CallImportResponse, + operation_id="updateCallImportMapping", +) +async def update_call_import_mapping( + call_import_id: UUID, + payload: CallImportMappingUpdate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportResponse: + """MAP stage of the staged call-import flow. + + Validates ``parameter_mapping`` + ``skipped_columns`` against the + sheet headers captured at UPLOAD time and persists them on the + batch. Idempotent: callers may submit this multiple times while + the batch is in ``uploaded`` or ``mapped`` state. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + if call_import.status not in ( + CallImportStatus.UPLOADED, + CallImportStatus.MAPPED, + ): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Cannot edit mapping on a batch in status " + f"'{call_import.status.value}'. Mapping can only be edited " + "before the IMPORT stage." + ), + ) + + if not call_import.source_s3_key or not call_import.source_format: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This batch was not uploaded through the staged flow and " + "cannot have its mapping edited." + ), + ) + + schema = _resolve_schema( + db, organization_id, workspace_id, payload.schema_id + ) + parameters = list(schema.parameters) + + canonical_sheet = _validate_sheet_choice( + call_import.source_format, + payload.sheet_name, + call_import.available_sheets, + ) + + # Pull the headers for the selected sheet straight out of the + # snapshot so we don't have to re-download the file from S3 just to + # validate the mapping. + headers: List[str] = [] + if call_import.available_sheets: + if canonical_sheet is None: + # CSV: single synthetic sheet. + entry = call_import.available_sheets[0] + headers = list(entry.get("headers") or []) + else: + for entry in call_import.available_sheets: + if not isinstance(entry, dict): + continue + name = entry.get("name") + if isinstance(name, str) and name == canonical_sheet: + headers = list(entry.get("headers") or []) + break + + cleaned_mapping = _clean_parameter_mapping( + payload.parameter_mapping, parameters, schema.name + ) + cleaned_skipped = _clean_skipped_columns(payload.skipped_columns) + + # Run the same per-column validation as the parse path so the user + # gets an immediate 400 if a required parameter is left unmapped or + # a header is neither mapped nor skipped — without needing to read + # the file. ``validate_only`` skips the row loop (and the empty-rows + # guard) since the row data lives in S3, not in this request. + if headers: + _apply_schema_mapping( + headers, + iter(()), + parameters, + cleaned_mapping, + cleaned_skipped, + source_label=( + f"Sheet '{canonical_sheet}'" + if canonical_sheet is not None + else "CSV" + ), + validate_only=True, + ) + + call_import.schema_id = schema.id + call_import.parameter_mapping = dict(cleaned_mapping) + call_import.skipped_columns = list(cleaned_skipped) + call_import.sheet_name = canonical_sheet + call_import.status = CallImportStatus.MAPPED + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + return _serialize_call_import(db, call_import) + + +@router.post( + "/{call_import_id}/import", + response_model=CallImportUploadResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="startCallImport", +) +async def start_call_import( + call_import_id: UUID, + payload: CallImportStartRequest, + background_tasks: BackgroundTasks, + legacy: bool = Query( + False, + description=( + "Deprecated escape hatch for import-only processing. " + "New batches should use Run Evaluation instead." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportUploadResponse: + """Deprecated IMPORT stage — use Run Evaluation for new batches. + + Recording fetch is part of the unified evaluation pipeline. This + endpoint remains available only with ``?legacy=true`` for backward + compatibility. + """ + del api_key, background_tasks + + if not legacy: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "Standalone import is deprecated. Use Run Evaluation — " + "recording fetch is part of the evaluation pipeline. " + "Append ?legacy=true to use the import-only path." + ), + ) + + from sqlalchemy.orm import selectinload as _selectinload + + call_import = ( + db.query(CallImport) + .options(_selectinload(CallImport.tags)) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + if call_import.status != CallImportStatus.MAPPED: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Cannot start import for a batch in status " + f"'{call_import.status.value}'. Map the columns first." + ), + ) + + if not call_import.source_s3_key or not call_import.source_format: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This batch has no staged source file and cannot be imported " + "through the staged flow." + ), + ) + + if not call_import.schema_id: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Cannot start import without a mapped schema.", + ) + + schema = _resolve_schema( + db, organization_id, workspace_id, call_import.schema_id + ) + parameters = list(schema.parameters) + + if payload.telephony_integration_id is not None: + integration = _resolve_telephony_integration( + db, + organization_id, + payload.telephony_integration_id, + payload.provider or "", + ) + _validate_telephony_credentials_live(db, organization_id, integration) + if (integration.provider or "").lower() == "exotel": + _validate_exotel_import_ready( + parameters, dict(call_import.parameter_mapping or {}) + ) + else: + _validate_direct_url_import_ready( + parameters, dict(call_import.parameter_mapping or {}) + ) + integration = None + + _ensure_blob_storage_enabled() + + if integration is not None: + call_import.provider = integration.provider + call_import.telephony_integration_id = integration.id + else: + call_import.provider = None + call_import.telephony_integration_id = None + + call_import.total_rows = 0 + call_import.completed_rows = 0 + call_import.failed_rows = 0 + call_import.error_message = None + call_import.status = CallImportStatus.PROCESSING + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + + from app.workers.tasks.call_import_bulk_ops import ( + materialize_call_import_rows_task, + ) + + materialize_call_import_rows_task.delay( + str(call_import_id), + str(organization_id), + str(workspace_id), + schedule_import_dispatch=True, + ) + + return CallImportUploadResponse( + id=call_import.id, + total_rows=0, + status=call_import.status, + dataset=call_import.dataset, + tags=_tag_response_payload(call_import.tags), + message=( + "Import accepted. Rows are being materialized in the background; " + "recordings will be fetched asynchronously." + ), + ) + + +@router.post( + "/upload", + response_model=CallImportUploadResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="uploadCallImportCsv", + deprecated=True, +) +async def upload_call_import_csv( + background_tasks: BackgroundTasks, + file: UploadFile = File(...), + provider: Optional[str] = Form( + None, + description=( + "Telephony provider key (e.g. 'exotel', 'plivo'). Must match the " + "selected telephony_integration_id's provider. Omit together " + "with telephony_integration_id for direct-URL import." + ), + ), + telephony_integration_id: Optional[UUID] = Form( + None, + description=( + "Specific TelephonyIntegration credential row to use when " + "downloading recordings for this batch. Omit together with " + "provider for direct-URL import." + ), + ), + schema_id: UUID = Form( + ..., + description=( + "Reusable Input Parameter schema this upload is mapped against. " + "Must belong to the active workspace." + ), + ), + parameter_mapping: str = Form( + ..., + description=( + "JSON-encoded ``{schema_parameter_name: source_header}`` map " + "covering every required schema parameter. Optional parameters " + "may be omitted or set to an empty string." + ), + ), + skipped_columns: Optional[str] = Form( + None, + description=( + "JSON-encoded list of source header strings the uploader has " + "explicitly skipped. Every header in the file must either be " + "mapped or appear here; otherwise the upload is rejected so a " + "forgotten column never silently drops." + ), + ), + dataset: Optional[str] = Form( + None, + description=( + "Optional free-text dataset label for high-level segregation. " + "Empty strings are stored as NULL." + ), + ), + tag_ids: Optional[List[UUID]] = Form( + None, + description="Optional list of CallImportTag ids to attach to the new batch.", + ), + sheet_name: Optional[str] = Form( + None, + description=( + "Worksheet to import when the file is an Excel workbook " + "(.xlsx / .xlsm). REQUIRED for Excel uploads. Ignored for CSV " + "uploads (rejected with 400 if non-empty so typos surface " + "instead of silently importing the wrong source)." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportUploadResponse: + """Legacy one-shot upload kept for backward compatibility. + + DEPRECATED: prefer the staged flow + (``POST /`` → ``PATCH /{id}/mapping`` → ``POST /{id}/import``) so + each step is idempotent and resumable. This endpoint runs all three + stages inline in a single transaction so existing scripts / + integrations keep working unchanged. + """ + del api_key + + fmt = _file_format(file.filename) + if fmt is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Unsupported file format. Allowed extensions: " + f"{', '.join(ALLOWED_EXTENSIONS)}." + ), + ) + + sheet_name_clean = (sheet_name or "").strip() or None + if fmt == "csv" and sheet_name_clean is not None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is not applicable to CSV uploads.", + ) + if fmt == "xlsx" and sheet_name_clean is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is required when uploading an Excel workbook.", + ) + + file_bytes = await file.read() + if len(file_bytes) > MAX_UPLOAD_BYTES: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", + ) + + schema = _resolve_schema(db, organization_id, workspace_id, schema_id) + parameters = list(schema.parameters) + + mapping_payload = _parse_json_form_field( + "parameter_mapping", parameter_mapping, {} + ) + cleaned_mapping = _clean_parameter_mapping( + mapping_payload, parameters, schema.name + ) + + skipped_payload = _parse_json_form_field("skipped_columns", skipped_columns, []) + cleaned_skipped = _clean_skipped_columns(skipped_payload) + + has_provider = bool((provider or "").strip()) + has_integration = telephony_integration_id is not None + if has_provider != has_integration: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "provider and telephony_integration_id must both be provided " + "or both omitted for direct-URL import." + ), + ) + + if telephony_integration_id is not None: + integration = _resolve_telephony_integration( + db, organization_id, telephony_integration_id, provider or "" + ) + if (integration.provider or "").lower() == "exotel": + _validate_exotel_import_ready(parameters, cleaned_mapping) + else: + _validate_direct_url_import_ready(parameters, cleaned_mapping) + integration = None + + parsed_rows = _parse_source_file( + file_bytes, fmt, sheet_name_clean, parameters, cleaned_mapping, cleaned_skipped + ) + _raise_if_no_importable_rows(parsed_rows, source_label=fmt) + + tag_rows = _resolve_tags(db, organization_id, tag_ids) + + call_import = CallImport( + organization_id=organization_id, + workspace_id=workspace_id, + provider=integration.provider if integration is not None else None, + telephony_integration_id=integration.id if integration is not None else None, + original_filename=file.filename, + sheet_name=sheet_name_clean, + dataset=_normalize_dataset(dataset), + schema_id=schema.id, + parameter_mapping=dict(cleaned_mapping), + skipped_columns=list(cleaned_skipped), + # Legacy columns are left empty on new uploads; the detail page + # falls back to ``parameter_mapping`` when ``schema_id`` is set. + column_mapping={}, + extra_columns=[], + custom_column_mapping={}, + total_rows=len(parsed_rows.rows), + completed_rows=0, + failed_rows=0, + status=CallImportStatus.PENDING, + source_row_skips=parse_skips_to_json(parsed_rows.skipped), + ) + if tag_rows: + call_import.tags = tag_rows + stamp_call_import_actor(call_import, principal, creating=True) + db.add(call_import) + db.flush() # populate call_import.id + if integration is None: + # The model's historical Python default is "exotel"; direct-URL + # imports intentionally have no telephony provider. + call_import.provider = None + + row_models = _materialize_rows( + db, call_import, parsed_rows.rows, organization_id + ) + + call_import.status = CallImportStatus.PROCESSING + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + + background_tasks.add_task( + record_call_import_batch_created, + organization_id, + call_import.id, + workspace_id=workspace_id, + total_rows=call_import.total_rows, + source="csv", + provider=call_import.provider, + ) + + _enqueue_row_tasks(db, call_import, row_models) + + return CallImportUploadResponse( + id=call_import.id, + total_rows=call_import.total_rows, + status=call_import.status, + dataset=call_import.dataset, + tags=_tag_response_payload(call_import.tags), + message=( + f"Accepted {call_import.total_rows} rows for import. " + "Recordings will be fetched asynchronously." + ), + ) + + +@router.post( + "/audio-upload", + response_model=CallImportUploadResponse, + status_code=status.HTTP_201_CREATED, + operation_id="uploadCallImportAudio", +) +async def upload_call_import_audio( + background_tasks: BackgroundTasks, + files: List[UploadFile] = File( + ..., + description="One or more manual call recording audio files.", + ), + dataset: str = Form( + ..., + description="Required free-text dataset label for the manual upload batch.", + ), + tag_ids: Optional[List[UUID]] = Form( + None, + description="Optional list of CallImportTag ids to attach to the new batch.", + ), + batch_name: Optional[str] = Form( + None, + description=( + "Optional display name for this manual upload batch. " + "When omitted, a generic label is used for multi-file uploads." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportUploadResponse: + """Persist manually uploaded recordings as completed CallImport rows. + + The rows skip the provider-download worker entirely because the audio + bytes are already in hand. From this point onward they behave exactly + like completed CSV-import rows: playback reads ``recording_s3_key`` and + the existing diarisation/evaluation endpoints can operate on them. + """ + + normalized_dataset = _normalize_dataset(dataset) + if not normalized_dataset: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="dataset is required and must be a non-empty string.", + ) + + _ensure_blob_storage_enabled() + tag_rows = _resolve_tags(db, organization_id, tag_ids) + prepared = await _prepare_audio_upload_files(files, conversation_counts={}) + + normalized_batch_name = _normalize_import_display_name(batch_name) + if normalized_batch_name: + original_filename = normalized_batch_name + elif len(prepared) == 1: + original_filename = prepared[0]["filename"] + else: + original_filename = "Manual recordings" + uploaded_keys: List[str] = [] + + from app.services.storage.s3_service import s3_service + + call_import = CallImport( + organization_id=organization_id, + workspace_id=workspace_id, + provider=None, + telephony_integration_id=None, + original_filename=original_filename, + source_format="audio", + source_size_bytes=0, + source_content_type="audio/*", + dataset=normalized_dataset, + total_rows=0, + completed_rows=0, + failed_rows=0, + status=CallImportStatus.COMPLETED, + ) + if tag_rows: + call_import.tags = tag_rows + + stamp_call_import_actor(call_import, principal, creating=True) + try: + db.add(call_import) + db.flush() + # The model's historical Python default is "exotel"; manual uploads + # intentionally have no telephony provider. + call_import.provider = None + + uploaded_keys = _persist_prepared_audio_rows( + db, + call_import=call_import, + organization_id=organization_id, + workspace_id=workspace_id, + prepared=prepared, + start_row_index=0, + ) + db.commit() + except Exception as exc: + db.rollback() + if uploaded_keys and s3_service.is_enabled(): + try: + s3_service.delete_keys(uploaded_keys) + except Exception: + logger.exception( + "Failed to clean up manual audio upload keys after error" + ) + logger.exception("Failed to persist manual call recording upload") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to upload manual recordings: {exc}", + ) from exc + + db.refresh(call_import) + background_tasks.add_task( + record_call_import_batch_created, + organization_id, + call_import.id, + workspace_id=workspace_id, + total_rows=call_import.total_rows, + source="audio", + provider=None, + ) + return CallImportUploadResponse( + id=call_import.id, + total_rows=call_import.total_rows, + status=call_import.status, + dataset=call_import.dataset, + tags=_tag_response_payload(call_import.tags), + message=( + f"Uploaded {call_import.total_rows} manual recording" + f"{'' if call_import.total_rows == 1 else 's'}." + ), + ) + + +@router.post( + "/{call_import_id}/audio-append", + response_model=CallImportUploadResponse, + status_code=status.HTTP_200_OK, + operation_id="appendCallImportAudio", +) +async def append_call_import_audio( + call_import_id: UUID, + files: List[UploadFile] = File( + ..., + description="Additional manual call recording audio files for an existing batch.", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> CallImportUploadResponse: + """Append manually uploaded recordings to an existing audio batch.""" + + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + if (call_import.source_format or "").lower() != "audio": + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Audio append is only supported for manual audio upload batches.", + ) + if call_import.status != CallImportStatus.COMPLETED: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Cannot append audio while batch is in status " + f"'{call_import.status.value}'." + ), + ) + + _ensure_blob_storage_enabled() + + start_row_index, conversation_counts = _manual_audio_append_start_state( + db, + call_import, + ) + + prepared = await _prepare_audio_upload_files(files, conversation_counts) + uploaded_keys: List[str] = [] + + from app.services.storage.s3_service import s3_service + + try: + uploaded_keys = _persist_prepared_audio_rows( + db, + call_import=call_import, + organization_id=organization_id, + workspace_id=workspace_id, + prepared=prepared, + start_row_index=start_row_index, + ) + db.commit() + except Exception as exc: + db.rollback() + if uploaded_keys and s3_service.is_enabled(): + try: + s3_service.delete_keys(uploaded_keys) + except Exception: + logger.exception( + "Failed to clean up manual audio append keys after error" + ) + logger.exception("Failed to append manual call recording upload") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to append manual recordings: {exc}", + ) from exc + + db.refresh(call_import) + return CallImportUploadResponse( + id=call_import.id, + total_rows=call_import.total_rows, + status=call_import.status, + dataset=call_import.dataset, + tags=_tag_response_payload(call_import.tags), + message=( + f"Uploaded {len(prepared)} additional manual recording" + f"{'' if len(prepared) == 1 else 's'} " + f"({call_import.total_rows} total)." + ), + ) + + +@router.get( + "", + response_model=CallImportListResponse, + operation_id="listCallImports", +) +async def list_call_imports( + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + status_filter: Optional[CallImportStatus] = Query(None, alias="status"), + dataset: Optional[str] = Query( + None, + description=( + "Filter by exact dataset string (case-insensitive). Pass the " + "literal value '__none__' to filter to imports with no dataset." + ), + ), + tag_id: Optional[List[UUID]] = Query( + None, + description="Filter to imports tagged with ALL of the given tag ids.", + ), + source_format: Optional[str] = Query( + None, + description=( + "Filter by source format. Use 'audio' for manual recordings or " + "'__non_audio__' for CSV/Excel/legacy imports." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> CallImportListResponse: + """List call-import batches for the active workspace, newest first. + + Scoped to (organization_id, workspace_id) so users only see imports + for the workspace they're currently in. Supports a high-level + ``dataset`` filter (powers the segregation dropdown at the top of + the imports page) plus an AND-style multi-tag filter via repeated + ``tag_id`` parameters. + """ + + query = ( + db.query(CallImport) + .filter( + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + ) + if status_filter is not None: + query = query.filter(CallImport.status == status_filter) + + source_filter = (source_format or "").strip().lower() + if source_filter == "__non_audio__": + query = query.filter( + or_(CallImport.source_format.is_(None), CallImport.source_format != "audio") + ) + elif source_filter: + query = query.filter(func.lower(CallImport.source_format) == source_filter) + + if dataset is not None: + if dataset == "__none__": + query = query.filter(CallImport.dataset.is_(None)) + elif dataset.strip(): + query = query.filter( + func.lower(CallImport.dataset) == dataset.strip().lower() + ) + + if tag_id: + from app.models.database import CallImportTagAssignment + + for single_tag_id in tag_id: + sub = ( + db.query(CallImportTagAssignment.call_import_id) + .filter(CallImportTagAssignment.tag_id == single_tag_id) + .subquery() + ) + query = query.filter(CallImport.id.in_(sub)) + + total = query.count() + items = ( + query.order_by(desc(CallImport.created_at)) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) + + email_map = emails_for_user_ids(db, user_ids_from_call_imports(items)) + return CallImportListResponse( + items=[ + _serialize_call_import(db, item, user_emails=email_map) + for item in items + ], + total=total, + page=page, + page_size=page_size, + ) + + +@router.get( + "/dispatch-diagnostics", + response_model=CallImportDispatchDiagnosticsResponse, + operation_id="getCallImportDispatchDiagnostics", + dependencies=[Depends(require_admin)], +) +async def get_call_import_dispatch_diagnostics( + workspace_id: Optional[UUID] = Query( + None, + description=( + "Optional workspace filter. When omitted, returns every workspace " + "in the organization with active eval dispatch state." + ), + ), + include_idle_workspaces: bool = Query( + False, + description=( + "When true, include org workspaces with zero pending rows and " + "zero in-flight slots." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportDispatchDiagnosticsResponse: + """Live eval slot usage and fair-dispatch state for operators. + + Org admins use this to diagnose cross-workspace starvation (e.g. one + workspace's 10k run blocking another's pending eval rows) by inspecting + Redis in-flight counters, pending dispatch rows, and scheduler cursors. + """ + del api_key + payload = build_call_import_dispatch_diagnostics( + db, + organization_id, + workspace_id=workspace_id, + include_idle_workspaces=include_idle_workspaces, + ) + return CallImportDispatchDiagnosticsResponse.model_validate(payload) + + +@router.get( + "/datasets", + response_model=List[str], + operation_id="listCallImportDatasets", +) +async def list_call_import_datasets( + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> List[str]: + """Return the distinct, non-null dataset labels in use for the active + workspace. + + Scoped per-workspace so each workspace's Dataset dropdown only shows + its own segregation labels. + """ + rows = ( + db.query(CallImport.dataset) + .filter( + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + CallImport.dataset.isnot(None), + CallImport.dataset != "", + ) + .distinct() + .order_by(CallImport.dataset.asc()) + .all() + ) + return [row[0] for row in rows if row[0]] + + +@router.get( + "/diarisation-prompt-default", + response_model=CallImportDiarisationPromptDefaultResponse, + operation_id="getCallImportDiarisationPromptDefault", +) +async def get_call_import_diarisation_prompt_default( + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), +) -> CallImportDiarisationPromptDefaultResponse: + """Return the canonical LLM diariser prompt. + + The Transcribe / Run Evaluation modals call this on open so they + can pre-fill the prompt textarea. Returning the constant from the + backend (rather than hard-coding it in the frontend) keeps the + fallback used by the worker and the placeholder shown in the UI + in lock-step — operators always see the *actual* default they'd + get if they leave the field blank. + + Registered before ``GET /{call_import_id}`` so the static path is + not mistaken for a UUID import id (which would 422). + """ + del api_key, organization_id + from app.workers.tasks.helpers.llm_diarisation import ( + DEFAULT_DIARIZATION_PROMPT, + ) + + return CallImportDiarisationPromptDefaultResponse( + prompt=DEFAULT_DIARIZATION_PROMPT + ) + + +@router.patch( + "/{call_import_id}", + response_model=CallImportResponse, + operation_id="updateCallImport", +) +async def update_call_import( + call_import_id: UUID, + payload: CallImportUpdate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportResponse: + """Edit dataset / tag assignments (and schema, pre-import) on a batch. + + ``dataset = ""`` clears the label; ``tag_ids = []`` removes all tag + assignments. Fields omitted from the body are left untouched. + + ``schema_id`` is only honoured while the batch is in + ``uploaded`` / ``mapped`` state — once rows have been materialised + the schema is locked. Changing the schema resets any persisted + mapping (the user must re-MAP) and rewinds status to ``uploaded``. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + body = payload.model_dump(exclude_unset=True) + if "original_filename" in body: + call_import.original_filename = _normalize_import_display_name( + body["original_filename"] + ) + + if "dataset" in body: + call_import.dataset = _normalize_dataset(body["dataset"]) + + if "tag_ids" in body: + tag_ids = body["tag_ids"] or [] + call_import.tags = _resolve_tags(db, organization_id, tag_ids) + + if "schema_id" in body and body["schema_id"] is not None: + if call_import.status not in ( + CallImportStatus.UPLOADED, + CallImportStatus.MAPPED, + ): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Cannot reassign schema on a batch in status " + f"'{call_import.status.value}'." + ), + ) + new_schema = _resolve_schema( + db, organization_id, workspace_id, body["schema_id"] + ) + if call_import.schema_id != new_schema.id: + # Switching schemas invalidates the persisted mapping — + # parameter names won't line up with the new schema, so + # reset to UPLOADED and force a fresh MAP. + call_import.schema_id = new_schema.id + call_import.parameter_mapping = {} + call_import.skipped_columns = [] + call_import.sheet_name = None + call_import.status = CallImportStatus.UPLOADED + + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + return _serialize_call_import(db, call_import) + + +@router.get( + "/{call_import_id}", + response_model=CallImportDetailResponse, + operation_id="getCallImportDetail", +) +async def get_call_import_detail( + call_import_id: UUID, + row_limit: int = Query(500, ge=0, le=5000), + row_offset: int = Query(0, ge=0), + q: Optional[str] = Query( + None, + description=( + "Optional case-insensitive substring filter on " + "``conversation_id``. When set, ``filtered_total_rows`` in " + "the response reflects the post-filter row count so the UI " + "can paginate against the filtered slice." + ), + ), + diarised_status: Optional[str] = Query( + None, + description=( + "Optional filter on ``CallImportRow.diarised_transcript_status``. " + "Accepts one of ``pending``, ``running``, ``completed``, " + "``failed``. When set, ``filtered_total_rows`` reflects the " + "post-filter row count (combined with the ``q`` filter when " + "both are supplied) so the UI can paginate against the same " + "slice it's displaying." + ), + pattern="^(pending|running|completed|failed)$", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportDetailResponse: + """Fetch a single import batch with a slice of its rows. + + ``row_limit=0`` is intentionally allowed so callers that only need the + batch metadata (e.g. the evaluation-detail page rendering the parent's + column mapping) can skip the rows payload entirely. + """ + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + if call_import.status == CallImportStatus.PROCESSING: + from app.services.call_imports.bulk_ops import rollup_call_import_batch_status + + prior_status = call_import.status + rollup_call_import_batch_status(db, call_import) + if call_import.status != prior_status: + db.commit() + db.refresh(call_import) + + search_term = (q or "").strip() + diarised_status_filter = (diarised_status or "").strip() or None + filtered_total_rows: Optional[int] = None + has_row_filters = bool(search_term or diarised_status_filter) + + if has_row_filters: + if is_sharding_enabled(): + from app.db_sharding.scatter_gather import count_call_import_rows_filtered + + filtered_total_rows = count_call_import_rows_filtered( + db, + call_import.id, + search_term=search_term, + diarised_status_filter=diarised_status_filter, + ) + else: + rows_query = db.query(CallImportRow).filter( + CallImportRow.call_import_id == call_import.id + ) + if search_term: + rows_query = rows_query.filter( + CallImportRow.conversation_id.ilike(f"%{search_term}%") + ) + if diarised_status_filter: + rows_query = rows_query.filter( + CallImportRow.diarised_transcript_status == diarised_status_filter + ) + filtered_total_rows = rows_query.count() + + if row_limit == 0: + rows: List[CallImportRow] = [] + elif is_sharding_enabled(): + from app.db_sharding.scatter_gather import ( + fetch_call_import_rows_filtered_page, + fetch_call_import_rows_page, + ) + + if has_row_filters: + rows = fetch_call_import_rows_filtered_page( + db, + call_import.id, + search_term=search_term, + diarised_status_filter=diarised_status_filter, + offset=row_offset, + limit=row_limit, + ) + else: + rows = fetch_call_import_rows_page( + db, + call_import.id, + offset=row_offset, + limit=row_limit, + ) + else: + rows_query = db.query(CallImportRow).filter( + CallImportRow.call_import_id == call_import.id + ) + if search_term: + rows_query = rows_query.filter( + CallImportRow.conversation_id.ilike(f"%{search_term}%") + ) + if diarised_status_filter: + rows_query = rows_query.filter( + CallImportRow.diarised_transcript_status == diarised_status_filter + ) + rows = ( + rows_query.order_by(CallImportRow.row_index) + .offset(row_offset) + .limit(row_limit) + .all() + ) + + # Batch-wide diarisation status aggregate. One ``GROUP BY`` query + # across the whole batch — much cheaper than paging through every + # row to recount on the client and lets the UI render a + # transcribe/diarise progress bar without a separate roundtrip. + if is_sharding_enabled(): + from app.db_sharding.scatter_gather import aggregate_diarised_transcript_counts + + diarised_status_counts = aggregate_diarised_transcript_counts( + db, call_import.id + ) + else: + diarised_status_counts: Dict[str, int] = {} + for status_value, count in ( + db.query(CallImportRow.diarised_transcript_status, func.count()) + .filter(CallImportRow.call_import_id == call_import.id) + .group_by(CallImportRow.diarised_transcript_status) + .all() + ): + if isinstance(status_value, str): + diarised_status_counts[status_value] = int(count or 0) + + detail = CallImportDetailResponse.model_validate( + _serialize_call_import(db, call_import).model_dump() + ) + detail.rows = [CallImportRowResponse.model_validate(r) for r in rows] + detail.filtered_total_rows = filtered_total_rows + detail.diarised_pending_rows = diarised_status_counts.get("pending", 0) + detail.diarised_running_rows = diarised_status_counts.get("running", 0) + detail.diarised_completed_rows = diarised_status_counts.get("completed", 0) + detail.diarised_failed_rows = diarised_status_counts.get("failed", 0) + return detail + + +@router.get( + "/{call_import_id}/row-ids", + response_model=CallImportRowIdsResponse, + operation_id="listCallImportRowIds", +) +async def list_call_import_row_ids( + call_import_id: UUID, + q: Optional[str] = Query( + None, + description=( + "Optional case-insensitive substring filter on " + "``conversation_id``. Same semantics as the detail endpoint." + ), + ), + diarised_status: Optional[str] = Query( + None, + description=( + "Optional filter on ``CallImportRow.diarised_transcript_status``. " + "Accepts ``pending`` / ``running`` / ``completed`` / ``failed``." + ), + pattern="^(pending|running|completed|failed)$", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportRowIdsResponse: + """Return every matching ``CallImportRow.id`` for cross-page bulk select. + + Lightweight companion to ``GET /{call_import_id}`` — the detail + endpoint caps ``row_limit`` at 5000 and ships the entire row body + on each page, so harvesting ids that way is wasteful when the + user just wants to bulk-delete or bulk-transcribe everything that + matches the current filters. This endpoint applies the same ``q`` + and ``diarised_status`` filters and returns only the ids. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + search_term = (q or "").strip() + status_filter = (diarised_status or "").strip() or None + + if is_sharding_enabled(): + from app.db_sharding.scatter_gather import list_call_import_row_ids_filtered + + ids = list_call_import_row_ids_filtered( + db, + call_import.id, + search_term=search_term, + diarised_status_filter=status_filter, + ) + return CallImportRowIdsResponse(ids=ids, total=len(ids)) + + rows_query = db.query(CallImportRow.id).filter( + CallImportRow.call_import_id == call_import.id + ) + if search_term: + rows_query = rows_query.filter( + CallImportRow.conversation_id.ilike(f"%{search_term}%") + ) + if status_filter: + rows_query = rows_query.filter( + CallImportRow.diarised_transcript_status == status_filter + ) + + ids = [ + row_id + for (row_id,) in rows_query.order_by(CallImportRow.row_index).all() + ] + return CallImportRowIdsResponse(ids=ids, total=len(ids)) + + +def _revoke_pending_tasks(rows: List[CallImportRow]) -> None: + """Best-effort revoke of in-flight Celery tasks for the given rows. + + Failures are logged and swallowed — Celery's control plane is async and + best-effort by design, and we always do an idempotent S3 cleanup + afterwards so a missed revoke can't leak storage. + """ + task_ids = [ + r.celery_task_id + for r in rows + if r.celery_task_id + and r.status in (CallImportRowStatus.PENDING, CallImportRowStatus.PROCESSING) + ] + if not task_ids: + return + + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke(task_ids, terminate=False) + logger.info("Revoked {} pending call-import tasks", len(task_ids)) + except Exception as exc: # noqa: BLE001 + logger.warning("Failed to revoke pending call-import tasks: {}", exc) + + +def _delete_s3_objects( + organization_id: UUID, + call_import_id: UUID, + rows: List[CallImportRow], +) -> tuple[int, int]: + """Delete every recording associated with ``rows`` plus a prefix sweep. + + The prefix sweep also cleans up the staged source file written at + UPLOAD time (``…/call_imports/{id}/source.{csv,xlsx}``) — both the + per-row recording keys and the source artefact share the same + organization-scoped prefix, so a single sweep covers them all. + + Returns ``(deleted_count, error_count)``. Never raises — callers proceed + with the DB delete regardless; orphans, if any, can be cleaned up by + re-running the same delete (it's idempotent). + """ + from app.services.storage.s3_service import s3_service + + if not s3_service.is_enabled(): + return 0, 0 + + keys = [r.recording_s3_key for r in rows if r.recording_s3_key] + deleted = 0 + errors = 0 + + if keys: + try: + d, errs = s3_service.delete_keys(keys) + deleted += d + errors += len(errs) + if errs: + logger.warning( + "S3 bulk-delete reported {} errors for call_import {}", + len(errs), + call_import_id, + ) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Bulk S3 delete failed for call_import {}: {}", call_import_id, exc + ) + errors += len(keys) + + # Belt-and-braces sweep: catch anything that landed under the import's + # prefix but never made it into a row's recording_s3_key (narrow + # window where the S3 upload succeeded but the DB commit didn't). + sweep_prefix = ( + f"{s3_service.prefix}organizations/{organization_id}/" + f"call_imports/{call_import_id}/" + ) + try: + d, errs = s3_service.delete_keys_by_prefix(sweep_prefix) + deleted += d + errors += len(errs) + except Exception as exc: # noqa: BLE001 + logger.exception( + "S3 prefix sweep failed for {}: {}", sweep_prefix, exc + ) + + return deleted, errors + + +@router.delete( + "/{call_import_id}", + response_model=CallImportDeleteResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="deleteCallImport", +) +async def delete_call_import( + call_import_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportDeleteResponse: + """Delete a call-import batch asynchronously. + + Flips the batch to ``deleting`` and enqueues background teardown so + large imports (thousands of rows + S3 objects) do not block the API. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + return CallImportDeleteResponse( + id=call_import_id, + status="completed", + ) + + if call_import.status == CallImportStatus.DELETING: + return CallImportDeleteResponse( + id=call_import.id, + status="accepted", + ) + + call_import.status = CallImportStatus.DELETING + call_import.error_message = None + stamp_call_import_actor(call_import, principal) + db.commit() + + from app.workers.tasks.call_import_bulk_ops import delete_call_import_task + + delete_call_import_task.delay( + str(call_import_id), + str(organization_id), + ) + + return CallImportDeleteResponse( + id=call_import.id, + status="accepted", + ) + + +def _locate_call_import_row_or_404( + catalog_db: Session, + *, + call_import_id: UUID, + row_id: UUID, + organization_id: UUID, +) -> Tuple[Session, CallImportRow, Optional[Session]]: + """Find a call import row on the correct DB session for mutation. + + When sharding is enabled rows live on shard databases; ``get_db`` only + opens the catalog. Returns ``(row_db, row, extra_catalog_to_close)`` + where ``extra_catalog_to_close`` is the catalog session opened by + :func:`locate_call_import_row` (distinct from the route's catalog + session) and must be closed via :func:`close_row_sessions`. + """ + from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row + + try: + row_db, located_catalog, row, _shard_id = locate_call_import_row(row_id) + except LookupError: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import row not found", + ) from None + if ( + row.call_import_id != call_import_id + or row.organization_id != organization_id + ): + close_row_sessions( + row_db, + located_catalog if located_catalog is not row_db else None, + ) + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import row not found", + ) + extra_catalog = located_catalog if located_catalog is not row_db else None + return row_db, row, extra_catalog + + +@router.delete( + "/{call_import_id}/rows/{row_id}", + status_code=status.HTTP_204_NO_CONTENT, + operation_id="deleteCallImportRow", +) +async def delete_call_import_row( + call_import_id: UUID, + row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> Response: + """Delete a single CallImportRow and its S3 recording. + + The parent ``CallImport`` is left in place. After deletion we recompute + its ``total_rows`` / ``completed_rows`` / ``failed_rows`` / ``status`` + so the UI's progress bar stays consistent with reality. + """ + from app.services.storage.s3_service import s3_service + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.db_sharding.row_ops import close_row_sessions + + row_db, row, extra_catalog = _locate_call_import_row_or_404( + db, + call_import_id=call_import.id, + row_id=row_id, + organization_id=organization_id, + ) + try: + _revoke_pending_tasks([row]) + + if row.recording_s3_key and s3_service.is_enabled(): + try: + s3_service.delete_file_by_key(row.recording_s3_key) + except Exception as exc: # noqa: BLE001 — best-effort, DB is source of truth + logger.warning( + "Failed to delete S3 object {} for row {}: {}", + row.recording_s3_key, + row.id, + exc, + ) + + row_db.delete(row) + row_db.commit() + + _recompute_call_import_counters(db, call_import) + stamp_call_import_actor(call_import, principal) + db.commit() + finally: + close_row_sessions(row_db, extra_catalog) + + logger.info( + "Deleted call_import_row {} (call_import={}, org={})", + row_id, + call_import.id, + organization_id, + ) + + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +def _recompute_call_import_counters( + db: Session, call_import: CallImport +) -> None: + """Resync ``total/completed/failed_rows`` + status on the parent batch. + + Called after row-level mutations (single delete, bulk delete) so the + UI's progress bar stays consistent with the actual row set. The + rules mirror :func:`delete_call_import_row` so behavior doesn't + diverge between the per-row and bulk paths. + """ + + from app.services.call_imports.bulk_ops import rollup_call_import_batch_status + + rollup_call_import_batch_status(db, call_import) + + +@router.post( + "/{call_import_id}/retry-failed", + response_model=CallImportRetryFailedRowsResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="retryFailedCallImportRows", +) +async def retry_failed_call_import_rows( + call_import_id: UUID, + payload: Optional[CallImportRetryFailedRowsRequest] = Body(None), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportRetryFailedRowsResponse: + """Re-enqueue every failed import row in this batch. + + Useful when transient provider issues are resolved and the operator wants + a one-click "try failed downloads again" pass without re-uploading the CSV. + + Pass ``provider`` + ``telephony_integration_id`` (or both omitted for + direct-URL retry) to change how recordings are fetched on this pass. + When the body is omitted entirely, the batch keeps its existing pinned + credentials. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + if payload is not None: + if payload.telephony_integration_id is not None: + integration = _resolve_telephony_integration( + db, + organization_id, + payload.telephony_integration_id, + payload.provider or "", + ) + _validate_telephony_credentials_live(db, organization_id, integration) + call_import.provider = integration.provider + call_import.telephony_integration_id = integration.id + if (integration.provider or "").lower() == "exotel": + schema = _resolve_schema( + db, + organization_id, + call_import.workspace_id, + call_import.schema_id, + ) + _validate_exotel_import_ready( + list(schema.parameters), + dict(call_import.parameter_mapping or {}), + ) + else: + call_import.provider = None + call_import.telephony_integration_id = None + db.flush() + + candidate_rows = ( + db.query(CallImportRow) + .filter( + CallImportRow.call_import_id == call_import.id, + CallImportRow.status.in_( + (CallImportRowStatus.FAILED, CallImportRowStatus.PENDING) + ), + ) + .order_by(CallImportRow.row_index.asc()) + .all() + ) + retryable_rows = [ + row + for row in candidate_rows + if row.status == CallImportRowStatus.FAILED + or _is_stuck_pending_import_row(row) + ] + if not retryable_rows: + return CallImportRetryFailedRowsResponse( + requeued=0, + enqueue_failed=0, + skipped=0, + ) + + from app.workers.concurrency.fair_import_dispatch import ( + schedule_fair_import_dispatch, + ) + + # Reset rows to pending BEFORE enqueue so the UI reflects "retry in + # progress" immediately even if the worker queue is backlogged. + for row in retryable_rows: + row.status = CallImportRowStatus.PENDING + row.error_message = None + row.celery_task_id = None + row.attempts = 0 + + db.flush() + _recompute_call_import_counters(db, call_import) + stamp_call_import_actor(call_import, principal) + db.commit() + + try: + schedule_fair_import_dispatch(max_workspace_turns=999) + requeued = len(retryable_rows) + enqueue_failed = 0 + skipped = 0 + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to schedule fair import dispatch for import {}", + call_import.id, + ) + requeued = 0 + enqueue_failed = len(retryable_rows) + skipped = 0 + for row in retryable_rows: + db.refresh(row) + if row.status != CallImportRowStatus.PENDING: + skipped += 1 + enqueue_failed -= 1 + continue + row.status = CallImportRowStatus.FAILED + row.error_message = f"Failed to enqueue retry: {exc}" + db.flush() + _recompute_call_import_counters(db, call_import) + db.commit() + + return CallImportRetryFailedRowsResponse( + requeued=requeued, + enqueue_failed=enqueue_failed, + skipped=skipped, + ) + + +@router.post( + "/{call_import_id}/rows/bulk-delete", + response_model=CallImportRowBulkDeleteResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="bulkDeleteCallImportRows", +) +async def bulk_delete_call_import_rows( + call_import_id: UUID, + payload: CallImportRowBulkDelete, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportRowBulkDeleteResponse: + """Delete multiple ``CallImportRow`` rows in one request. + + Unknown / cross-tenant row ids are silently skipped — the response + reports how many actually went away so a UI that holds onto stale + ids (e.g. after another tab already deleted a row) doesn't 404 + the entire bulk action. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + if not payload.row_ids: + return CallImportRowBulkDeleteResponse(deleted=0, status="completed") + + from app.workers.tasks.call_import_bulk_ops import bulk_delete_call_import_rows_task + + row_id_strs = [str(rid) for rid in payload.row_ids] + + stamp_call_import_actor(call_import, principal) + db.commit() + + bulk_delete_call_import_rows_task.delay( + str(call_import_id), + str(organization_id), + row_id_strs, + ) + + return CallImportRowBulkDeleteResponse(deleted=0, status="accepted") + + +# --------------------------------------------------------------------------- +# Diarization / transcription endpoints +# --------------------------------------------------------------------------- + + +def _select_rows_for_transcription( + db: Session, + call_import: CallImport, + payload: CallImportTranscribeRequest, + requested_row_ids: Optional[List[UUID]] = None, +) -> tuple[List[CallImportRow], Dict[str, int]]: + """Pick which rows to enqueue for diarisation (delegates to bulk_ops).""" + from app.services.call_imports.bulk_ops import select_rows_for_transcription + + try: + return select_rows_for_transcription( + db, call_import, payload, requested_row_ids=requested_row_ids + ) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc), + ) from exc + + +@router.post( + "/{call_import_id}/transcribe", + response_model=CallImportTranscribeResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="transcribeCallImport", +) +async def transcribe_call_import( + call_import_id: UUID, + payload: CallImportTranscribeRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportTranscribeResponse: + """Fan out diarization tasks for many rows in a single call. + + Returns a summary with how many rows were queued and how many were + skipped (broken down by reason) so the UI can show a meaningful + toast even when nothing actually got enqueued (e.g. "All 12 rows + already have transcripts"). + """ + + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.workers.tasks.call_import_bulk_ops import bulk_diarize_call_import_task + + stamp_call_import_actor(call_import, principal) + db.commit() + + bulk_diarize_call_import_task.delay( + str(call_import_id), + str(organization_id), + payload.model_dump(mode="json"), + [str(rid) for rid in payload.row_ids] if payload.row_ids else None, + ) + + return CallImportTranscribeResponse( + queued=0, + skipped_rows=0, + skipped_reason_counts={}, + accepted=True, + ) + + +@router.post( + "/{call_import_id}/rows/{row_id}/transcribe", + response_model=CallImportTranscribeResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="transcribeCallImportRow", +) +async def transcribe_call_import_row( + call_import_id: UUID, + row_id: UUID, + payload: CallImportTranscribeRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportTranscribeResponse: + """Diarize / transcribe a single row. + + Thin wrapper over the batch endpoint that hard-codes a single + ``row_ids`` filter. Skip counts still surface so the UI can render + "Skipped — transcript present" diagnostics consistently. + """ + + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.services.call_imports.bulk_ops import execute_bulk_diarization + + try: + result = execute_bulk_diarization( + db, + call_import, + payload, + requested_row_ids=[row_id], + ) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc), + ) from exc + + stamp_call_import_actor(call_import, principal) + db.commit() + + return CallImportTranscribeResponse( + queued=result.queued, + skipped_rows=result.skipped_rows, + skipped_reason_counts=result.skipped_reason_counts, + ) + + +# --------------------------------------------------------------------------- +# Cancel-in-flight diarisation +# --------------------------------------------------------------------------- +# +# Long-running multimodal LLM diarisation calls (especially LLM-only mode on +# slow audio) can sit in ``pending`` / ``running`` for tens of minutes when an +# upstream provider stalls. Without an abort affordance the operator's only +# recourse is to wait for Celery's ``time_limit`` to fire — which can be +# several minutes — or to manually mutate the DB. These helpers + the two +# endpoints below give the UI a first-class "Stop diarisation" button. +# +# Why ``terminate=True``: the legacy ``_revoke_pending_tasks`` helper uses +# ``terminate=False`` because it's called from delete-flow paths where the +# task may simply not get to run (a worker pulls it off the queue and drops +# it). For a user-initiated cancel we want SIGTERM to interrupt the worker +# mid-LLM call so the audio HTTP request actually aborts. ``terminate=True`` +# routes SIGTERM to the executing process; ``signal="SIGTERM"`` is the +# default but we spell it out so the intent is obvious to reviewers. + +# Sentinel error message stamped on cancelled rows. Read by the transcribe +# worker's finaliser (see ``app/workers/tasks/transcribe_call_import_row.py``) +# to detect a row that was cancelled mid-flight and AVOID overwriting it +# with whatever partial result the worker had managed to compute before the +# SIGTERM landed. +CANCELLED_BY_USER_ERROR: str = "Diarisation cancelled by user" + + +def _cancellable_diarisation_states() -> Tuple[str, ...]: + """States that a diarisation row can be cancelled from. + + Kept as a tiny helper so adding a future ``"queued"`` / ``"retrying"`` + state only needs one edit. + """ + return ("pending", "running") + + +def _revoke_diarisation_task(row: CallImportRow) -> None: + """Best-effort revoke of a single row's diarisation Celery task. + + Always swallows control-plane exceptions — Celery's control bus is + inherently best-effort and a missed revoke is not catastrophic + because the DB row is already flipped to ``failed`` by the caller + before this runs (so the UI immediately reflects the cancel; if + the task happens to finish anyway, the worker's finaliser skips + over the row via :data:`CANCELLED_BY_USER_ERROR`). + """ + task_id = (row.celery_task_id or "").strip() + if not task_id: + return + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke( + task_id, terminate=True, signal="SIGTERM" + ) + logger.info( + "Revoked diarisation task {} for call-import row {}", + task_id, + row.id, + ) + except Exception as exc: # noqa: BLE001 — revoke is best-effort + logger.warning( + "Failed to revoke diarisation task {} for row {}: {}", + task_id, + row.id, + exc, + ) + + +def _apply_diarisation_cancel(rows: List[CallImportRow]) -> Tuple[int, int]: + """Cancel diarisation on every cancellable row in ``rows``. + + Returns ``(cancelled, skipped)`` so the caller can build a typed + response without re-querying the DB. The caller is responsible for + ``db.commit()`` after this returns — we deliberately don't commit + here so a batch endpoint can flush all rows in one transaction. + """ + cancellable_states = _cancellable_diarisation_states() + cancelled = 0 + skipped = 0 + for row in rows: + if (row.diarised_transcript_status or "").lower() not in cancellable_states: + skipped += 1 + continue + # Flip the row state BEFORE we revoke so the UI's next poll + # already shows the cancel, even if Celery's control plane is + # slow to ack. + row.diarised_transcript_status = "failed" + row.diarised_transcript_error = CANCELLED_BY_USER_ERROR + _revoke_diarisation_task(row) + # Drop the task id so a follow-up retry (or a stale poll) can't + # accidentally re-revoke or get confused. + row.celery_task_id = None + cancelled += 1 + return cancelled, skipped + + +@router.post( + "/{call_import_id}/rows/{row_id}/cancel-diarisation", + response_model=CallImportRowResponse, + status_code=status.HTTP_200_OK, + operation_id="cancelCallImportRowDiarisation", +) +async def cancel_call_import_row_diarisation( + call_import_id: UUID, + row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportRowResponse: + """Abort an in-flight (or queued) diarisation for a single row. + + Idempotent: calling on a row that's already terminal (``completed`` + / ``failed`` / ``idle``) returns the row unchanged with a 200, so + the UI can fire this from a "Stop" button without having to + pre-check the state. + + Race notes: + + * The row's ``diarised_transcript_status`` is flipped to ``failed`` + with :data:`CANCELLED_BY_USER_ERROR` BEFORE the Celery revoke, + so the polling UI sees the cancel immediately. + * If the worker happens to finish between our DB flip and the + SIGTERM landing, its finaliser will detect the cancelled + sentinel on the row and skip its own status / score writes + (see :mod:`app.workers.tasks.transcribe_call_import_row`). + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.db_sharding.row_ops import close_row_sessions + + row_db, row, extra_catalog = _locate_call_import_row_or_404( + db, + call_import_id=call_import_id, + row_id=row_id, + organization_id=organization_id, + ) + try: + _apply_diarisation_cancel([row]) + row_db.commit() + stamp_call_import_actor(call_import, principal) + db.commit() + row_db.refresh(row) + return CallImportRowResponse.model_validate(row) + finally: + close_row_sessions(row_db, extra_catalog) + + +@router.post( + "/{call_import_id}/cancel-diarisation", + response_model=CallImportCancelDiarisationResponse, + status_code=status.HTTP_200_OK, + operation_id="cancelCallImportDiarisation", +) +async def cancel_call_import_diarisation( + call_import_id: UUID, + payload: Optional[CallImportCancelDiarisationRequest] = None, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportCancelDiarisationResponse: + """Abort in-flight diarisation for many rows in a single call. + + Default body (no ``row_ids``) cancels every row in this import + whose ``diarised_transcript_status`` is ``pending`` or + ``running`` — the "stop everything" button. Pass ``row_ids`` to + scope the cancel to the rows the operator has selected. + + Returns ``(cancelled, skipped)`` so the UI can render a tight + toast ("Cancelled 3 rows · 1 skipped (already completed)"). + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + base_query = db.query(CallImportRow).filter( + CallImportRow.call_import_id == call_import_id + ) + + requested_ids = ( + payload.row_ids if payload and payload.row_ids is not None else None + ) + if requested_ids is not None: + if not requested_ids: + # Empty list is "no rows requested" — treat as a no-op + # 200 rather than 400 so the UI can pass through an empty + # selection without a special-case. + return CallImportCancelDiarisationResponse(cancelled=0, skipped=0) + rows = base_query.filter(CallImportRow.id.in_(requested_ids)).all() + found_ids = {r.id for r in rows} + # Treat requested-but-not-found ids as ``skipped`` so the UI's + # numbers reconcile (a stale selection that includes deleted + # rows shouldn't 404 the whole call). + missing = [rid for rid in requested_ids if rid not in found_ids] + skipped_missing = len(missing) + else: + # Implicit "cancel every cancellable row in this import" path. + rows = base_query.filter( + CallImportRow.diarised_transcript_status.in_( + list(_cancellable_diarisation_states()) + ) + ).all() + skipped_missing = 0 + + cancelled, skipped = _apply_diarisation_cancel(rows) + stamp_call_import_actor(call_import, principal) + db.commit() + return CallImportCancelDiarisationResponse( + cancelled=cancelled, + skipped=skipped + skipped_missing, + ) + + +def _render_diarised_segments_text( + segments: Optional[List[Dict[str, Any]]], + *, + swap: bool = False, +) -> str: + """Render ``CallImportRow.diarised_segments`` as ``: `` lines. + + Mirrors the worker's ``_render_turns_as_text`` (kept duplicated so + the route doesn't need to import a Celery task module just to + rebuild the rendered transcript). Only ``agent`` and ``user`` are + swapped — multi-party calls keep their ``speaker_N`` labels through + a swap so we don't silently collapse a third speaker into the user + side. + """ + if not segments: + return "" + out: List[str] = [] + for turn in segments: + if not isinstance(turn, dict): + continue + speaker = (turn.get("speaker") or "").strip() + text = (turn.get("text") or "").strip() + if not speaker or not text: + continue + if swap: + if speaker == "agent": + speaker = "user" + elif speaker == "user": + speaker = "agent" + out.append(f"{speaker}: {text}") + return "\n".join(out) + + +@router.post( + "/{call_import_id}/rows/{row_id}/diarised-speaker-swap", + response_model=CallImportRowResponse, + operation_id="toggleCallImportRowSpeakerSwap", +) +async def toggle_call_import_row_speaker_swap( + call_import_id: UUID, + row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportRowResponse: + """Flip the user <-> agent mapping on a diarised row. + + The worker's "first speaker is the agent" heuristic is right most of + the time but does fail on inbound recordings where the customer + greets first, on recordings where the agent stays silent for the + intro, etc. Rather than rerun the (expensive) STT + pyannote + pipeline for those cases, we let reviewers flip the mapping in + place: the structured ``diarised_segments`` are the source of truth + and we re-render the plain-text ``diarised_transcript`` from them + with the swap applied. The next CSV export will then show the + corrected labels. + + Returns the updated row so the frontend can refresh without an + extra round-trip. + """ + + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.db_sharding.row_ops import close_row_sessions + + row_db, row, extra_catalog = _locate_call_import_row_or_404( + db, + call_import_id=call_import_id, + row_id=row_id, + organization_id=organization_id, + ) + try: + segments = ( + row.diarised_segments if isinstance(row.diarised_segments, list) else None + ) + if not segments: + # Without structured turns the swap toggle would have nothing to + # re-render — surface a clear error rather than silently + # flipping a flag the UI never read. + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This row has no structured diarised segments to swap. " + "Re-run diarisation to generate per-speaker turns first." + ), + ) + + new_swap = not bool(row.diarised_speaker_swap) + row.diarised_speaker_swap = new_swap + row.diarised_transcript = ( + _render_diarised_segments_text(segments, swap=new_swap) or None + ) + row_db.commit() + stamp_call_import_actor(call_import, principal) + db.commit() + row_db.refresh(row) + return CallImportRowResponse.model_validate(row) + finally: + close_row_sessions(row_db, extra_catalog) + + +# --------------------------------------------------------------------------- +# Cross-run insights for the import detail page +# --------------------------------------------------------------------------- + + +@router.get( + "/{call_import_id}/insights", + response_model=CallImportInsightsResponse, + operation_id="getCallImportInsights", +) +async def get_call_import_insights( + call_import_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportInsightsResponse: + """Aggregate signals across every evaluation run on this import. + + Powers the Insights tab on the call-import detail page: returns + per-metric "latest run" summaries plus a trend series of mean values + across runs so the UI can render a small line chart per metric. Also + bundles transcript coverage stats since those are the cheapest + pre-eval health-check (e.g. "30 of 50 rows still missing + transcripts"). + """ + + del api_key + + from app.models.database import ( + CallImportEvaluation, + CallImportEvaluationRow, + Metric, + ) + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + rows = ( + db.query(CallImportRow) + .filter(CallImportRow.call_import_id == call_import_id) + .all() + ) + # A row "has a transcript" if EITHER the production (CSV) or the + # diarised (worker) column is populated — the insights tile reports + # the union so users see total coverage regardless of which source + # produced the value. + rows_with_transcript = sum( + 1 + for r in rows + if (r.transcript or "").strip() + or (r.diarised_transcript or "").strip() + ) + rows_without_transcript = len(rows) - rows_with_transcript + source_counts: Dict[str, int] = {} + for r in rows: + has_production = bool((r.transcript or "").strip()) + has_diarised = bool((r.diarised_transcript or "").strip()) + if has_production: + key = r.transcript_source or "csv" + source_counts[key] = source_counts.get(key, 0) + 1 + if has_diarised: + source_counts["diarised"] = source_counts.get("diarised", 0) + 1 + + evaluations = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .order_by(CallImportEvaluation.created_at.asc()) + .all() + ) + + # Defer heavy lifting to the aggregation helper so this endpoint and + # the per-run aggregate endpoint share the exact same metric + # bucketing math (no chance of "trend" disagreeing with "latest" on + # the same data set). + from app.api.v1.routes.call_import_evaluations import ( + _compute_metric_aggregates, + ) + + metric_history: Dict[str, List[CallImportInsightsRunPoint]] = {} + metric_meta: Dict[str, Metric] = {} + metric_latest: Dict[str, CallImportMetricAggregate] = {} + + for evaluation in evaluations: + eval_rows = ( + db.query(CallImportEvaluationRow) + .filter(CallImportEvaluationRow.evaluation_id == evaluation.id) + .all() + ) + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + for agg in aggregates: + if agg.metric_id not in metric_meta: + # ``agg.metric_id`` is normally a UUID string, but the + # aggregator also emits ids that surface in row scores + # without a matching ``Metric`` row (e.g. a metric the + # user deleted mid-run, or LLM-discovered slugs). Those + # are not valid UUIDs, so coerce defensively and skip + # the metric registry lookup when the cast fails — the + # ``meta is None`` branch below already handles the + # display via the values stored on ``agg`` itself. + try: + metric_uuid = UUID(agg.metric_id) + except (ValueError, AttributeError, TypeError): + metric_uuid = None + if metric_uuid is not None: + metric_obj = ( + db.query(Metric) + .filter( + Metric.id == metric_uuid, + Metric.organization_id == organization_id, + ) + .first() + ) + if metric_obj is not None: + metric_meta[agg.metric_id] = metric_obj + history = metric_history.setdefault(agg.metric_id, []) + history.append( + CallImportInsightsRunPoint( + evaluation_id=evaluation.id, + name=evaluation.name, + created_at=evaluation.created_at, + mean=agg.mean, + completed_rows=agg.count, + ) + ) + metric_latest[agg.metric_id] = agg + + metrics_payload: List[CallImportInsightsMetric] = [] + for metric_id, latest in metric_latest.items(): + meta = metric_meta.get(metric_id) + metrics_payload.append( + CallImportInsightsMetric( + metric_id=metric_id, + metric_name=(meta.name if meta else latest.metric_name), + metric_type=(meta.metric_type if meta else latest.metric_type), + latest=latest, + trend=metric_history.get(metric_id, []), + ) + ) + + return CallImportInsightsResponse( + call_import_id=call_import_id, + total_rows=len(rows), + rows_with_transcript=rows_with_transcript, + rows_without_transcript=rows_without_transcript, + transcript_source_counts=source_counts, + evaluation_count=len(evaluations), + metrics=metrics_payload, + ) + + +from app.core.auth.capabilities import CALLS_DELETE, CALLS_IMPORT, CALLS_VIEW +from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities + +apply_workspace_route_capabilities( + router, + view_capability=CALLS_VIEW, + manage_capability=CALLS_IMPORT, + delete_capability=CALLS_DELETE, +) diff --git a/app/api/v1/routes/integrations.py b/app/api/v1/routes/integrations.py index 0d880f8f..338f295b 100644 --- a/app/api/v1/routes/integrations.py +++ b/app/api/v1/routes/integrations.py @@ -32,7 +32,7 @@ def _integration_response( effective_routing = get_credential_effective_routing_label( organization_id, db, - integration.routing_mode, + integration, ) response = IntegrationResponse.model_validate(integration) return response.model_copy(update={"effective_routing": effective_routing}) diff --git a/app/api/v1/routes/metric_studio.py b/app/api/v1/routes/metric_studio.py new file mode 100644 index 00000000..aeb155f3 --- /dev/null +++ b/app/api/v1/routes/metric_studio.py @@ -0,0 +1,409 @@ +"""Metrics Studio API routes.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional +from uuid import UUID, uuid4 + +from fastapi import APIRouter, Depends, HTTPException, Query, status +from sqlalchemy.orm import Session + +from app.database import get_db +from app.dependencies import get_api_key, get_organization_id, get_workspace_id +from app.models.database import ( + Metric, + MetricStudioRun, + MetricStudioRunResult, +) +from app.models.schemas import ( + MetricStudioRunCreate, + MetricStudioRunListResponse, + MetricStudioRunResponse, + MetricStudioRunResultListResponse, + MetricStudioRunResultResponse, + MetricStudioRunRetryRequest, +) +from app.services.metric_studio.metric_selection import expand_studio_metric_selection +from app.services.metric_studio.source_resolver import resolve_source + +router = APIRouter(prefix="/metric-studio", tags=["metric-studio"]) + + +def _serialize_run(run: MetricStudioRun) -> MetricStudioRunResponse: + return MetricStudioRunResponse( + id=run.id, + organization_id=run.organization_id, + workspace_id=run.workspace_id, + name=run.name, + selected_metric_ids=[str(mid) for mid in (run.selected_metric_ids or [])], + selected_metric_groups=run.selected_metric_groups, + transcript_source=run.transcript_source or "diarised", + llm_provider=run.llm_provider, + llm_model=run.llm_model, + status=run.status, + total_items=run.total_items or 0, + completed_items=run.completed_items or 0, + failed_items=run.failed_items or 0, + error_message=run.error_message, + started_at=run.started_at, + finished_at=run.finished_at, + created_at=run.created_at, + updated_at=run.updated_at, + ) + + +def _resolve_evaluation_transcript_metadata( + db: Session, + *, + run: MetricStudioRun, + row: MetricStudioRunResult, +) -> Dict[str, Any]: + metadata = dict(row.source_metadata or {}) + if metadata.get("evaluation_transcript"): + metadata.setdefault( + "transcript_source_used", + run.transcript_source or "diarised", + ) + return metadata + if row.status != "completed": + return metadata + try: + sample = resolve_source( + db, + organization_id=run.organization_id, + workspace_id=run.workspace_id, + source_kind=row.source_kind, + source_ref=row.source_ref, + display_label=row.display_label, + ) + except HTTPException: + return metadata + transcript_source = (run.transcript_source or "diarised").lower() + if transcript_source == "production": + transcript = sample.transcript + else: + transcript = sample.diarised_transcript or sample.transcript + if transcript: + metadata["evaluation_transcript"] = transcript + metadata["transcript_source_used"] = transcript_source + return metadata + + +def _serialize_result( + row: MetricStudioRunResult, + *, + db: Optional[Session] = None, + run: Optional[MetricStudioRun] = None, +) -> MetricStudioRunResultResponse: + source_metadata = row.source_metadata + if db is not None and run is not None: + source_metadata = _resolve_evaluation_transcript_metadata(db, run=run, row=row) + return MetricStudioRunResultResponse( + id=row.id, + run_id=row.run_id, + source_kind=row.source_kind, + source_ref=row.source_ref, + display_label=row.display_label, + source_metadata=source_metadata, + status=row.status, + metric_scores=row.metric_scores or {}, + error_message=row.error_message, + started_at=row.started_at, + finished_at=row.finished_at, + created_at=row.created_at, + updated_at=row.updated_at, + ) + + +def _rollup_run_status(db: Session, run: MetricStudioRun) -> None: + results = ( + db.query(MetricStudioRunResult) + .filter(MetricStudioRunResult.run_id == run.id) + .all() + ) + completed = sum(1 for r in results if r.status == "completed") + failed = sum(1 for r in results if r.status == "failed") + pending = sum(1 for r in results if r.status in {"pending", "running"}) + run.completed_items = completed + run.failed_items = failed + if pending: + run.status = "running" + elif failed and completed: + run.status = "partial" + elif failed: + run.status = "failed" + else: + run.status = "completed" + run.finished_at = datetime.now(timezone.utc) + db.flush() + + +@router.post( + "/runs", + response_model=MetricStudioRunResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="createMetricStudioRun", +) +async def create_metric_studio_run( + payload: MetricStudioRunCreate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> MetricStudioRunResponse: + del api_key + + org_metrics = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.id.in_(payload.metric_ids), + ) + .all() + ) + by_id = {metric.id: metric for metric in org_metrics} + unknown_ids = [mid for mid in payload.metric_ids if mid not in by_id] + if unknown_ids: + raise HTTPException( + status_code=400, + detail=f"Unknown metric ids: {', '.join(str(mid) for mid in unknown_ids)}", + ) + + effective_metrics, parent_to_children = expand_studio_metric_selection( + db, organization_id, payload.metric_ids + ) + if not effective_metrics: + raise HTTPException( + status_code=400, + detail="No scorable metrics after expanding the selection.", + ) + + leaf_metric_ids = [m.id for m in effective_metrics] + selected_metric_groups: Dict[str, List[str]] = { + str(pid): [str(c.id) for c in children] + for pid, children in parent_to_children.items() + } + + if payload.llm_provider or payload.llm_model: + if not (payload.llm_provider and payload.llm_model): + raise HTTPException( + status_code=400, + detail="Both llm_provider and llm_model are required when overriding the run LLM.", + ) + + run = MetricStudioRun( + id=uuid4(), + organization_id=organization_id, + workspace_id=workspace_id, + name=payload.name, + selected_metric_ids=[str(mid) for mid in leaf_metric_ids], + selected_metric_groups=selected_metric_groups or None, + transcript_source=payload.transcript_source, + llm_provider=payload.llm_provider, + llm_model=payload.llm_model, + llm_credential_id=payload.llm_credential_id, + llm_config=payload.llm_config, + metric_llm_overrides=payload.metric_llm_overrides, + status="pending", + total_items=len(payload.sources), + started_at=datetime.now(timezone.utc), + ) + db.add(run) + db.flush() + + result_rows: List[MetricStudioRunResult] = [] + for source in payload.sources: + sample = resolve_source( + db, + organization_id=organization_id, + workspace_id=workspace_id, + source_kind=source.source_kind, + source_ref=source.source_ref, + display_label=source.display_label, + ) + result_row = MetricStudioRunResult( + id=uuid4(), + run_id=run.id, + workspace_id=workspace_id, + source_kind=sample.source_kind, + source_ref=sample.source_ref, + display_label=sample.label, + source_metadata=sample.metadata, + status="pending", + ) + db.add(result_row) + result_rows.append(result_row) + + db.commit() + db.refresh(run) + + from app.workers.tasks.evaluate_studio_run_item import ( + evaluate_studio_run_item_task, + ) + + run.status = "running" + db.commit() + + for result_row in result_rows: + async_result = evaluate_studio_run_item_task.delay(str(result_row.id)) + result_row.celery_task_id = async_result.id + result_row.status = "running" + result_row.started_at = datetime.now(timezone.utc) + db.commit() + + return _serialize_run(run) + + +@router.get("/runs", response_model=MetricStudioRunListResponse) +def list_metric_studio_runs( + skip: int = Query(0, ge=0), + limit: int = Query(50, ge=1, le=200), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> MetricStudioRunListResponse: + query = ( + db.query(MetricStudioRun) + .filter( + MetricStudioRun.organization_id == organization_id, + MetricStudioRun.workspace_id == workspace_id, + ) + .order_by(MetricStudioRun.created_at.desc()) + ) + total = query.count() + runs = query.offset(skip).limit(limit).all() + return MetricStudioRunListResponse( + items=[_serialize_run(run) for run in runs], + total=total, + ) + + +@router.get("/runs/{run_id}", response_model=MetricStudioRunResponse) +def get_metric_studio_run( + run_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> MetricStudioRunResponse: + run = ( + db.query(MetricStudioRun) + .filter( + MetricStudioRun.id == run_id, + MetricStudioRun.organization_id == organization_id, + MetricStudioRun.workspace_id == workspace_id, + ) + .first() + ) + if not run: + raise HTTPException(status_code=404, detail="Studio run not found.") + return _serialize_run(run) + + +@router.get("/runs/{run_id}/results", response_model=MetricStudioRunResultListResponse) +def list_metric_studio_run_results( + run_id: UUID, + skip: int = Query(0, ge=0), + limit: int = Query(100, ge=1, le=500), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> MetricStudioRunResultListResponse: + run = ( + db.query(MetricStudioRun) + .filter( + MetricStudioRun.id == run_id, + MetricStudioRun.organization_id == organization_id, + MetricStudioRun.workspace_id == workspace_id, + ) + .first() + ) + if not run: + raise HTTPException(status_code=404, detail="Studio run not found.") + + query = ( + db.query(MetricStudioRunResult) + .filter(MetricStudioRunResult.run_id == run_id) + .order_by(MetricStudioRunResult.created_at.asc()) + ) + total = query.count() + rows = query.offset(skip).limit(limit).all() + return MetricStudioRunResultListResponse( + items=[_serialize_result(row, db=db, run=run) for row in rows], + total=total, + ) + + +@router.post("/runs/{run_id}/retry", response_model=MetricStudioRunResponse) +def retry_metric_studio_run( + run_id: UUID, + body: MetricStudioRunRetryRequest, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> MetricStudioRunResponse: + run = ( + db.query(MetricStudioRun) + .filter( + MetricStudioRun.id == run_id, + MetricStudioRun.organization_id == organization_id, + MetricStudioRun.workspace_id == workspace_id, + ) + .first() + ) + if not run: + raise HTTPException(status_code=404, detail="Studio run not found.") + + query = db.query(MetricStudioRunResult).filter( + MetricStudioRunResult.run_id == run_id + ) + if body.result_ids: + query = query.filter(MetricStudioRunResult.id.in_(body.result_ids)) + else: + query = query.filter(MetricStudioRunResult.status == "failed") + + rows = query.all() + if not rows: + raise HTTPException(status_code=400, detail="No results eligible for retry.") + + from app.workers.tasks.evaluate_studio_run_item import ( + evaluate_studio_run_item_task, + ) + + run.status = "running" + run.finished_at = None + for row in rows: + row.status = "running" + row.error_message = None + row.metric_scores = {} + row.started_at = datetime.now(timezone.utc) + row.finished_at = None + async_result = evaluate_studio_run_item_task.delay(str(row.id)) + row.celery_task_id = async_result.id + db.commit() + _rollup_run_status(db, run) + db.commit() + db.refresh(run) + return _serialize_run(run) + + +@router.delete("/runs/{run_id}", status_code=status.HTTP_204_NO_CONTENT) +def delete_metric_studio_run( + run_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> None: + run = ( + db.query(MetricStudioRun) + .filter( + MetricStudioRun.id == run_id, + MetricStudioRun.organization_id == organization_id, + MetricStudioRun.workspace_id == workspace_id, + ) + .first() + ) + if not run: + raise HTTPException(status_code=404, detail="Studio run not found.") + db.delete(run) + db.commit() diff --git a/app/api/v1/routes/metrics.py b/app/api/v1/routes/metrics.py index 760580b5..5d54f619 100644 --- a/app/api/v1/routes/metrics.py +++ b/app/api/v1/routes/metrics.py @@ -2,6 +2,7 @@ import json import re +from datetime import datetime, timezone from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, status from sqlalchemy.orm import Session from sqlalchemy import and_, or_ @@ -18,6 +19,9 @@ MetricCreate, MetricCreateWithChildren, MetricChildDraft, + MetricDraftCreate, + MetricDraftCreateWithChildren, + MetricPromoteResponse, MetricUpdate, MetricResponse, PromoteDiscoveredChildRequest, @@ -134,6 +138,26 @@ def _validate_hierarchy_fields( ) +def _filter_enabled_metric_trees( + trees: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Keep enabled standalones and category parents with enabled children.""" + filtered: List[Dict[str, Any]] = [] + for tree in trees: + if tree.get("selection_mode"): + enabled_children = [ + child + for child in (tree.get("children") or []) + if child.get("enabled") + ] + if not enabled_children: + continue + filtered.append({**tree, "children": enabled_children}) + elif tree.get("enabled"): + filtered.append(tree) + return filtered + + def _serialize_metric_tree(metric: Metric) -> Dict[str, Any]: """Convert a Metric ORM row into a dict shaped for ``MetricResponse``. @@ -180,6 +204,9 @@ def _serialize_metric_tree(metric: Metric) -> Dict[str, Any]: "compare_transcripts": bool( getattr(metric, "compare_transcripts", False) ), + "lifecycle": getattr(metric, "lifecycle", None) or "active", + "promoted_from_draft_at": getattr(metric, "promoted_from_draft_at", None), + "studio_notes": getattr(metric, "studio_notes", None), "children": children_payload, "created_at": metric.created_at, "updated_at": metric.updated_at, @@ -320,27 +347,114 @@ def create_metric( @router.post( - "/with-children", + "/drafts", response_model=MetricResponse, status_code=201, - operation_id="createMetricWithChildren", + operation_id="createMetricDraft", ) -def create_metric_with_children( - payload: MetricCreateWithChildren, +def create_metric_draft( + metric_data: MetricDraftCreate, organization_id: UUID = Depends(get_organization_id), workspace_id: UUID = Depends(get_workspace_id), db: Session = Depends(get_db), ): - """Atomically create a parent category metric plus its children. + """Create a draft metric for Metrics Studio (hidden from production flows).""" + _validate_hierarchy_fields( + organization_id, + db, + parent_metric_id=metric_data.parent_metric_id, + selection_mode=metric_data.selection_mode, + metric_type=metric_data.metric_type, + allow_discovery=metric_data.allow_discovery, + ) - The parent gets ``metric_type=text`` (it's a category label, not a - score) and ``selection_mode`` from the payload. Every child is - forced to ``boolean`` so the LLM-evaluation path treats them as - yes/no labels. Both the parent and all children are stamped with - the same scope: either the active workspace (``scope="workspace"``, - default) or ``workspace_id=NULL`` (``scope="organization"``, the - org-shared shape). - """ + if metric_data.parent_metric_id is not None: + parent_row = ( + db.query(Metric) + .filter( + Metric.id == metric_data.parent_metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if parent_row is None: + raise HTTPException(status_code=400, detail="Parent metric not found.") + effective_workspace_id: Optional[UUID] = parent_row.workspace_id + elif metric_data.scope == "organization": + effective_workspace_id = None + else: + effective_workspace_id = workspace_id + + workspace_filter = ( + Metric.workspace_id.is_(None) + if effective_workspace_id is None + else Metric.workspace_id == effective_workspace_id + ) + parent_filter = ( + Metric.parent_metric_id.is_(None) + if metric_data.parent_metric_id is None + else Metric.parent_metric_id == metric_data.parent_metric_id + ) + existing = ( + db.query(Metric) + .filter( + Metric.name == metric_data.name, + Metric.organization_id == organization_id, + workspace_filter, + parent_filter, + ) + .first() + ) + if existing: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="A metric with this name already exists", + ) + + effective_metric_type = metric_data.metric_type + if metric_data.parent_metric_id is not None: + effective_metric_type = MetricType.BOOLEAN + + metric = Metric( + organization_id=organization_id, + workspace_id=effective_workspace_id, + name=metric_data.name, + description=metric_data.description, + example=metric_data.example, + metric_type=effective_metric_type, + metric_category=metric_data.metric_category, + trigger=metric_data.trigger, + enabled=False, + is_default=False, + metric_origin=metric_data.metric_origin or "custom", + supported_surfaces=metric_data.supported_surfaces or ["agent"], + enabled_surfaces=[], + custom_data_type=metric_data.custom_data_type, + custom_config=metric_data.custom_config, + tags=metric_data.tags, + capture_rationale=bool(metric_data.capture_rationale), + parent_metric_id=metric_data.parent_metric_id, + selection_mode=metric_data.selection_mode, + allow_discovery=bool(metric_data.allow_discovery), + compare_transcripts=bool(metric_data.compare_transcripts), + lifecycle="draft", + studio_notes=metric_data.studio_notes, + ) + db.add(metric) + db.commit() + db.refresh(metric) + return _serialize_metric_tree(metric) + + +def _create_metric_with_children( + db: Session, + *, + organization_id: UUID, + workspace_id: UUID, + payload: MetricCreateWithChildren, + lifecycle: str = "active", + studio_notes: Optional[str] = None, +) -> Metric: if payload.selection_mode not in _VALID_SELECTION_MODES: raise HTTPException( status_code=400, @@ -350,8 +464,6 @@ def create_metric_with_children( ), ) - # Org-shared categories live with ``workspace_id=NULL`` so every - # workspace in the org sees the same category + children. effective_workspace_id: Optional[UUID] = ( None if payload.scope == "organization" else workspace_id ) @@ -376,9 +488,6 @@ def create_metric_with_children( detail=f"A top-level metric named '{payload.name}' already exists.", ) - # Detect duplicate child names within the same request before any - # writes — the DB has no compound uniqueness constraint, so we - # enforce it in code. child_names_seen: set[str] = set() for child in payload.children: key = (child.name or "").strip().lower() @@ -401,10 +510,9 @@ def create_metric_with_children( if payload.enabled_surfaces is not None else (payload.supported_surfaces or ["agent"]) if payload.enabled else [] ) + is_draft = lifecycle == "draft" + parent_enabled_surfaces: List[str] = [] if is_draft else enabled_surfaces - # ``allow_discovery`` requires a parent (selection_mode set). - # Both single_choice and multi_label parents are valid hosts; the - # prompt builder + mapper handle the per-mode semantics. if payload.allow_discovery and not payload.selection_mode: raise HTTPException( status_code=400, @@ -419,36 +527,28 @@ def create_metric_with_children( workspace_id=effective_workspace_id, name=payload.name, description=payload.description, - # The parent itself stores no numeric value — its "result" is the - # set of true children. Treat it as text so the rest of the - # stack (aggregation, CSV export, etc.) renders the chosen child - # name as the parent's "value". metric_type=MetricType.TEXT, metric_category=payload.metric_category, trigger=MetricTrigger.ALWAYS, - enabled=len(enabled_surfaces) > 0, + enabled=not is_draft and len(parent_enabled_surfaces) > 0, is_default=False, metric_origin="custom", supported_surfaces=payload.supported_surfaces or ["agent"], - enabled_surfaces=enabled_surfaces, + enabled_surfaces=parent_enabled_surfaces, tags=payload.tags, - # Hierarchical mode now captures rationale at the PARENT level - # (the LLM emits one rationale per category, never per child), - # so honour the user's toggle here and force children below to - # capture_rationale=False. capture_rationale=bool(payload.capture_rationale), selection_mode=payload.selection_mode, allow_discovery=bool(payload.allow_discovery), + lifecycle=lifecycle, + studio_notes=studio_notes, ) db.add(parent) db.flush() for child_draft in payload.children: + child_enabled = bool(child_draft.enabled) and len(parent_enabled_surfaces) > 0 child = Metric( organization_id=organization_id, - # Children inherit the parent's scope (workspace UUID or - # NULL for org-shared) so the whole category subtree stays - # in one place. workspace_id=effective_workspace_id, name=child_draft.name, description=child_draft.description, @@ -456,28 +556,120 @@ def create_metric_with_children( metric_type=MetricType.BOOLEAN, metric_category=payload.metric_category, trigger=MetricTrigger.ALWAYS, - enabled=bool(child_draft.enabled) and len(enabled_surfaces) > 0, + enabled=not is_draft and child_enabled, is_default=False, metric_origin="custom", supported_surfaces=payload.supported_surfaces or ["agent"], - enabled_surfaces=( - enabled_surfaces if child_draft.enabled else [] - ), + enabled_surfaces=parent_enabled_surfaces if child_draft.enabled else [], custom_data_type="boolean", custom_config={}, tags=child_draft.tags, - # Children in hierarchical mode never carry their own - # rationale — the parent owns the single rationale string - # for the whole group. Force false regardless of payload so - # legacy clients can't accidentally enable per-child - # rationales that the worker would then ignore. capture_rationale=False, parent_metric_id=parent.id, + lifecycle=lifecycle, ) db.add(child) db.commit() db.refresh(parent) + return parent + + +@router.post( + "/drafts/with-children", + response_model=MetricResponse, + status_code=201, + operation_id="createMetricDraftWithChildren", +) +def create_metric_draft_with_children( + payload: MetricDraftCreateWithChildren, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """Atomically create a draft parent category metric plus its children.""" + parent = _create_metric_with_children( + db, + organization_id=organization_id, + workspace_id=workspace_id, + payload=payload, + lifecycle="draft", + studio_notes=payload.studio_notes, + ) + return _serialize_metric_tree(parent) + + +@router.post( + "/{metric_id}/promote", + response_model=MetricPromoteResponse, + operation_id="promoteMetricDraft", +) +def promote_metric_draft( + metric_id: UUID, + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +): + """Promote a draft metric to active production use.""" + metric = ( + db.query(Metric) + .filter( + Metric.id == metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if not metric: + raise HTTPException(status_code=404, detail="Metric not found") + if (metric.lifecycle or "active") != "draft": + raise HTTPException( + status_code=400, + detail="Only draft metrics can be promoted.", + ) + + promoted_at = datetime.now(timezone.utc) + metric.lifecycle = "active" + metric.enabled = True + metric.promoted_from_draft_at = promoted_at + if not (metric.enabled_surfaces or []): + metric.enabled_surfaces = ["agent"] + if not (metric.supported_surfaces or []): + metric.supported_surfaces = ["agent"] + db.commit() + db.refresh(metric) + return MetricPromoteResponse( + metric=_serialize_metric_tree(metric), + promoted_at=promoted_at, + ) + + +@router.post( + "/with-children", + response_model=MetricResponse, + status_code=201, + operation_id="createMetricWithChildren", +) +def create_metric_with_children( + payload: MetricCreateWithChildren, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """Atomically create a parent category metric plus its children. + + The parent gets ``metric_type=text`` (it's a category label, not a + score) and ``selection_mode`` from the payload. Every child is + forced to ``boolean`` so the LLM-evaluation path treats them as + yes/no labels. Both the parent and all children are stamped with + the same scope: either the active workspace (``scope="workspace"``, + default) or ``workspace_id=NULL`` (``scope="organization"``, the + org-shared shape). + """ + parent = _create_metric_with_children( + db, + organization_id=organization_id, + workspace_id=workspace_id, + payload=payload, + ) return _serialize_metric_tree(parent) @@ -841,6 +1033,21 @@ def promote_discovered_metric( @router.get("", response_model=List[MetricResponse]) def list_metrics( surface: Optional[str] = None, + include_drafts: bool = Query( + False, + description="When true, include draft metrics (Studio-only) in the listing.", + ), + drafts_only: bool = Query( + False, + description="When true, return only draft metrics.", + ), + enabled_only: bool = Query( + False, + description=( + "When true, return only metrics enabled in the active workspace. " + "Category parents are included when at least one child is enabled." + ), + ), include_children: bool = Query( True, description=( @@ -878,6 +1085,12 @@ def list_metrics( Metric.metric_origin == "default", ), ) + if drafts_only: + query = query.filter(Metric.lifecycle == "draft") + elif not include_drafts: + query = query.filter( + or_(Metric.lifecycle.is_(None), Metric.lifecycle == "active") + ) metrics = ( query.order_by(Metric.is_default.desc(), Metric.created_at.desc()).all() ) @@ -889,7 +1102,10 @@ def list_metrics( ] if not include_children: - return [_serialize_metric_tree(m) for m in metrics] + payload = [_serialize_metric_tree(m) for m in metrics] + if enabled_only: + return _filter_enabled_metric_trees(payload) + return payload # Top-level rows = anything without a parent, OR a child whose parent # is not visible at this surface (so users still see "orphaned" @@ -900,7 +1116,10 @@ def list_metrics( for m in metrics if m.parent_metric_id is None or m.parent_metric_id not in visible_ids ] - return [_serialize_metric_tree(m) for m in top_level] + payload = [_serialize_metric_tree(m) for m in top_level] + if enabled_only: + return _filter_enabled_metric_trees(payload) + return payload @router.get("/{metric_id}", response_model=MetricResponse) diff --git a/app/api/v1/routes/observability.py b/app/api/v1/routes/observability.py index 7af456c0..4c26a549 100644 --- a/app/api/v1/routes/observability.py +++ b/app/api/v1/routes/observability.py @@ -103,6 +103,15 @@ def _serialize_call_recording( payload: Dict[str, Any] = { "id": str(call_recording.id), "call_short_id": call_recording.call_short_id, + "display_name": ( + agent.name + if agent and agent.name + else ( + f"{call_recording.provider_platform} call" + if call_recording.provider_platform + else call_recording.call_short_id + ) + ), "status": call_recording.status.value if call_recording.status else None, "call_event": call_event, "is_live": call_event in live_events, diff --git a/app/api/v1/routes/platform_admin.py b/app/api/v1/routes/platform_admin.py new file mode 100644 index 00000000..f29630ff --- /dev/null +++ b/app/api/v1/routes/platform_admin.py @@ -0,0 +1,417 @@ +"""Platform admin routes for cross-org management.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import List, Optional +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query, status +from pydantic import BaseModel, EmailStr, Field +from sqlalchemy import func +from sqlalchemy.orm import Session + +from app.core.auth.platform_admin import ( + PlatformAdminPrincipal, + create_platform_access_token, + get_platform_admin, + platform_admin_feature_enabled, +) +from app.core.auth.refresh_tokens import revoke_all_user_refresh_tokens +from app.core.password import hash_password, validate_password_strength, verify_password +from app.database import get_db +from app.models.database import ( + Organization, + OrganizationMember, + PlatformAdmin, + SignupReferenceCode, + User, +) +from app.services.signup_reference_codes import hash_reference_code + +router = APIRouter(prefix="/platform", tags=["Platform Admin"]) + + +class PlatformLoginRequest(BaseModel): + email: EmailStr + password: str + + +class PlatformAdminSummary(BaseModel): + id: str + email: str + + +class PlatformTokenResponse(BaseModel): + access_token: str + token_type: str = "Bearer" + expires_in: int + admin: PlatformAdminSummary + + +class OrganizationListItem(BaseModel): + id: str + name: str + is_active: bool + member_count: int + created_at: Optional[str] = None + disabled_at: Optional[str] = None + + +class OrganizationListResponse(BaseModel): + items: List[OrganizationListItem] + total: int + offset: int + limit: int + + +class OrganizationStatsResponse(BaseModel): + total: int + active: int + disabled: int + + +class OrganizationUpdateRequest(BaseModel): + is_active: bool + + +class OrgUserItem(BaseModel): + id: str + email: str + role: str + is_active: bool + + +class PlatformPasswordResetRequest(BaseModel): + new_password: str = Field(min_length=8, max_length=32) + + +class PlatformPasswordResetResponse(BaseModel): + user_id: str + email: str + message: str = "Password reset successfully" + + +class SignupCodeCreateRequest(BaseModel): + code: str = Field(min_length=4, max_length=64) + label: Optional[str] = Field(default=None, max_length=255) + max_uses: Optional[int] = Field(default=None, ge=1) + expires_at: Optional[datetime] = None + + +class SignupCodeResponse(BaseModel): + id: str + label: Optional[str] = None + max_uses: Optional[int] = None + use_count: int + expires_at: Optional[str] = None + is_active: bool + created_at: Optional[str] = None + code: Optional[str] = None + + +class SignupCodeUpdateRequest(BaseModel): + is_active: Optional[bool] = None + max_uses: Optional[int] = Field(default=None, ge=1) + label: Optional[str] = Field(default=None, max_length=255) + + +def _validate_password_or_400(password: str) -> None: + try: + validate_password_strength(password) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + + +def _serialize_org(org: Organization, member_count: int) -> OrganizationListItem: + return OrganizationListItem( + id=str(org.id), + name=org.name, + is_active=bool(org.is_active), + member_count=member_count, + created_at=org.created_at.isoformat() if org.created_at else None, + disabled_at=org.disabled_at.isoformat() if org.disabled_at else None, + ) + + +def _serialize_signup_code(row: SignupReferenceCode, *, include_code: bool = False, code: Optional[str] = None) -> SignupCodeResponse: + return SignupCodeResponse( + id=str(row.id), + label=row.label, + max_uses=row.max_uses, + use_count=row.use_count or 0, + expires_at=row.expires_at.isoformat() if row.expires_at else None, + is_active=bool(row.is_active), + created_at=row.created_at.isoformat() if row.created_at else None, + code=code if include_code else None, + ) + + +@router.post("/auth/login", response_model=PlatformTokenResponse) +def platform_login(payload: PlatformLoginRequest, db: Session = Depends(get_db)) -> PlatformTokenResponse: + if not platform_admin_feature_enabled(db): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Not found") + + admin = ( + db.query(PlatformAdmin) + .filter(PlatformAdmin.email == payload.email, PlatformAdmin.is_active == True) # noqa: E712 + .first() + ) + if admin is None or not verify_password(payload.password, admin.password_hash): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid email or password.", + ) + + admin.last_login_at = datetime.now(timezone.utc) + db.commit() + + access_token, expires_in = create_platform_access_token( + platform_admin_id=admin.id, + email=admin.email, + ) + return PlatformTokenResponse( + access_token=access_token, + expires_in=expires_in, + admin=PlatformAdminSummary(id=str(admin.id), email=admin.email), + ) + + +@router.get("/auth/me", response_model=PlatformAdminSummary) +def platform_me( + principal: PlatformAdminPrincipal = Depends(get_platform_admin), +) -> PlatformAdminSummary: + return PlatformAdminSummary(id=str(principal.platform_admin_id), email=principal.email) + + +@router.get("/organizations", response_model=OrganizationListResponse) +def list_organizations( + offset: int = Query(0, ge=0), + limit: int = Query(50, ge=1, le=200), + search: Optional[str] = Query(default=None, max_length=255), + is_active: Optional[bool] = Query(default=None), + _principal: PlatformAdminPrincipal = Depends(get_platform_admin), + db: Session = Depends(get_db), +) -> OrganizationListResponse: + query = db.query(Organization) + if search: + query = query.filter(Organization.name.ilike(f"%{search}%")) + if is_active is not None: + query = query.filter(Organization.is_active == is_active) + + total = query.count() + orgs = ( + query.order_by(Organization.created_at.desc()) + .offset(offset) + .limit(limit) + .all() + ) + + member_counts = dict( + db.query(OrganizationMember.organization_id, func.count(OrganizationMember.id)) + .filter(OrganizationMember.organization_id.in_([org.id for org in orgs])) + .group_by(OrganizationMember.organization_id) + .all() + ) if orgs else {} + + return OrganizationListResponse( + items=[_serialize_org(org, member_counts.get(org.id, 0)) for org in orgs], + total=total, + offset=offset, + limit=limit, + ) + + +@router.get("/organizations/stats", response_model=OrganizationStatsResponse) +def organization_stats( + _principal: PlatformAdminPrincipal = Depends(get_platform_admin), + db: Session = Depends(get_db), +) -> OrganizationStatsResponse: + total = db.query(func.count(Organization.id)).scalar() or 0 + active = ( + db.query(func.count(Organization.id)) + .filter(Organization.is_active == True) # noqa: E712 + .scalar() + or 0 + ) + disabled = total - active + return OrganizationStatsResponse(total=total, active=active, disabled=disabled) + + +@router.patch("/organizations/{org_id}", response_model=OrganizationListItem) +def update_organization( + org_id: UUID, + payload: OrganizationUpdateRequest, + _principal: PlatformAdminPrincipal = Depends(get_platform_admin), + db: Session = Depends(get_db), +) -> OrganizationListItem: + org = db.query(Organization).filter(Organization.id == org_id).first() + if org is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Organization not found") + + org.is_active = payload.is_active + org.disabled_at = None if payload.is_active else datetime.now(timezone.utc) + db.commit() + db.refresh(org) + + member_count = ( + db.query(func.count(OrganizationMember.id)) + .filter(OrganizationMember.organization_id == org.id) + .scalar() + or 0 + ) + return _serialize_org(org, member_count) + + +@router.get("/organizations/{org_id}/users", response_model=List[OrgUserItem]) +def list_organization_users( + org_id: UUID, + role: Optional[str] = Query(default=None), + _principal: PlatformAdminPrincipal = Depends(get_platform_admin), + db: Session = Depends(get_db), +) -> List[OrgUserItem]: + org = db.query(Organization).filter(Organization.id == org_id).first() + if org is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Organization not found") + + query = ( + db.query(User, OrganizationMember) + .join(OrganizationMember, OrganizationMember.user_id == User.id) + .filter(OrganizationMember.organization_id == org_id) + ) + if role: + query = query.filter(OrganizationMember.role == role) + + rows = query.order_by(User.email.asc()).all() + items: List[OrgUserItem] = [] + for user, member in rows: + role_value = member.role.value if hasattr(member.role, "value") else member.role + items.append( + OrgUserItem( + id=str(user.id), + email=user.email, + role=role_value, + is_active=bool(user.is_active), + ) + ) + return items + + +@router.post( + "/organizations/{org_id}/users/{user_id}/reset-password", + response_model=PlatformPasswordResetResponse, +) +def platform_reset_user_password( + org_id: UUID, + user_id: UUID, + payload: PlatformPasswordResetRequest, + _principal: PlatformAdminPrincipal = Depends(get_platform_admin), + db: Session = Depends(get_db), +) -> PlatformPasswordResetResponse: + member = ( + db.query(OrganizationMember) + .filter( + OrganizationMember.organization_id == org_id, + OrganizationMember.user_id == user_id, + ) + .first() + ) + if member is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="User is not a member of this organization", + ) + + user = db.query(User).filter(User.id == user_id).first() + if user is None or not user.is_active: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="User not found or inactive", + ) + + _validate_password_or_400(payload.new_password) + user.password_hash = hash_password(payload.new_password) + revoke_all_user_refresh_tokens(db, user_id=user.id) + db.commit() + + return PlatformPasswordResetResponse(user_id=str(user.id), email=user.email) + + +@router.get("/signup-codes", response_model=List[SignupCodeResponse]) +def list_signup_codes( + _principal: PlatformAdminPrincipal = Depends(get_platform_admin), + db: Session = Depends(get_db), +) -> List[SignupCodeResponse]: + rows = ( + db.query(SignupReferenceCode) + .order_by(SignupReferenceCode.created_at.desc()) + .all() + ) + return [_serialize_signup_code(row) for row in rows] + + +@router.post("/signup-codes", response_model=SignupCodeResponse, status_code=status.HTTP_201_CREATED) +def create_signup_code( + payload: SignupCodeCreateRequest, + principal: PlatformAdminPrincipal = Depends(get_platform_admin), + db: Session = Depends(get_db), +) -> SignupCodeResponse: + code_hash = hash_reference_code(payload.code) + existing = db.query(SignupReferenceCode).filter(SignupReferenceCode.code_hash == code_hash).first() + if existing is not None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="A reference code with this value already exists.", + ) + + row = SignupReferenceCode( + code_hash=code_hash, + label=payload.label, + max_uses=payload.max_uses, + expires_at=payload.expires_at, + is_active=True, + created_by=principal.platform_admin_id, + ) + db.add(row) + db.commit() + db.refresh(row) + return _serialize_signup_code(row, include_code=True, code=payload.code.strip()) + + +@router.patch("/signup-codes/{code_id}", response_model=SignupCodeResponse) +def update_signup_code( + code_id: UUID, + payload: SignupCodeUpdateRequest, + _principal: PlatformAdminPrincipal = Depends(get_platform_admin), + db: Session = Depends(get_db), +) -> SignupCodeResponse: + row = db.query(SignupReferenceCode).filter(SignupReferenceCode.id == code_id).first() + if row is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Reference code not found") + + if payload.is_active is not None: + row.is_active = payload.is_active + if payload.max_uses is not None: + row.max_uses = payload.max_uses + if payload.label is not None: + row.label = payload.label + + db.commit() + db.refresh(row) + return _serialize_signup_code(row) + + +@router.delete("/signup-codes/{code_id}", response_model=SignupCodeResponse) +def deactivate_signup_code( + code_id: UUID, + _principal: PlatformAdminPrincipal = Depends(get_platform_admin), + db: Session = Depends(get_db), +) -> SignupCodeResponse: + row = db.query(SignupReferenceCode).filter(SignupReferenceCode.id == code_id).first() + if row is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Reference code not found") + + row.is_active = False + db.commit() + db.refresh(row) + return _serialize_signup_code(row) diff --git a/app/api/v1/routes/playground.py b/app/api/v1/routes/playground.py index 19e97a65..8671d7ba 100644 --- a/app/api/v1/routes/playground.py +++ b/app/api/v1/routes/playground.py @@ -766,14 +766,31 @@ async def list_call_recordings( "status": r.status, "metric_scores": r.metric_scores, "result_id": r.result_id, + "name": r.name, } for r in results } - + + agent_ids = [cr.agent_id for cr in call_recordings if cr.agent_id] + agents_by_id = {} + if agent_ids: + agents = db.query(Agent).filter(Agent.id.in_(agent_ids)).all() + agents_by_id = {a.id: a for a in agents} + + def _display_name(cr: CallRecording) -> str: + linked = result_info.get(str(cr.evaluator_result_id), {}) if cr.evaluator_result_id else {} + if linked.get("name"): + return linked["name"] + agent = agents_by_id.get(cr.agent_id) if cr.agent_id else None + if agent and agent.name: + return agent.name + return cr.call_short_id + return [ { "id": str(cr.id), "call_short_id": cr.call_short_id, + "display_name": _display_name(cr), "status": cr.status if cr.status else None, "provider_platform": cr.provider_platform, "provider_call_id": cr.provider_call_id, diff --git a/app/api/v1/routes/workspaces.py b/app/api/v1/routes/workspaces.py index 5f7a4a8d..f48c0d48 100644 --- a/app/api/v1/routes/workspaces.py +++ b/app/api/v1/routes/workspaces.py @@ -56,6 +56,7 @@ def _workspace_response( name=workspace.name, slug=workspace.slug, is_default=workspace.is_default, + is_active=workspace.is_active, created_at=workspace.created_at, updated_at=workspace.updated_at, role_id=role.id if role else None, @@ -84,6 +85,7 @@ def list_workspaces( if not member_ws_ids: return [] query = query.filter(Workspace.id.in_(member_ws_ids)) + query = query.filter(Workspace.is_active.is_(True)) workspaces = query.order_by(Workspace.is_default.desc(), Workspace.name.asc()).all() return [_workspace_response(db, workspace=ws, principal=principal) for ws in workspaces] @@ -168,7 +170,13 @@ def update_workspace( organization_id: UUID = Depends(get_organization_id), db: Session = Depends(get_db), ): - """Rename a workspace (slug stays put to keep deep-links stable).""" + """Rename a workspace or change active status (org admin only for the latter).""" + if payload.name is None and payload.is_active is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="At least one of name or is_active must be provided.", + ) + workspace = ( db.query(Workspace) .filter( @@ -180,22 +188,58 @@ def update_workspace( if workspace is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Workspace not found.") + org_role = get_org_role(principal, db) + is_org_admin = org_role == RoleEnum.ADMIN + caps, _, role = resolve_workspace_capabilities( db, principal=principal, workspace_id=workspace_id, organization_id=organization_id, ) - if WORKSPACE_SETTINGS not in caps: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=capability_denied_message( - WORKSPACE_SETTINGS, - role_name=role.name if role else None, - ), - ) - workspace.name = payload.name.strip() + if payload.is_active is not None: + if not is_org_admin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Only organization admins can change workspace active status.", + ) + if payload.is_active is False: + if workspace.is_default: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="The default workspace cannot be deactivated.", + ) + if not workspace.is_active: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Workspace is already inactive.", + ) + workspace.is_active = False + else: + if workspace.is_active: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Workspace is already active.", + ) + workspace.is_active = True + + if payload.name is not None: + if not workspace.is_active: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Cannot rename an inactive workspace. Reactivate it first.", + ) + if WORKSPACE_SETTINGS not in caps: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=capability_denied_message( + WORKSPACE_SETTINGS, + role_name=role.name if role else None, + ), + ) + workspace.name = payload.name.strip() + db.commit() db.refresh(workspace) return _workspace_response(db, workspace=workspace, principal=principal) diff --git a/app/config.py b/app/config.py index 0eec6b2a..fc194f81 100644 --- a/app/config.py +++ b/app/config.py @@ -1,913 +1,917 @@ -"""Configuration management using Pydantic settings.""" - -import json -import os -import re -import yaml -from pathlib import Path -from typing import Annotated, Any, Dict, List, Optional, Union -from pydantic import field_validator -from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict - - -class Settings(BaseSettings): - """Application settings.""" - - # Application - APP_NAME: str = "Voice AI Evaluation Platform" - APP_VERSION: str = "0.1.0" - API_V1_PREFIX: str = "/api/v1" - DEBUG: bool = True - SECRET_KEY: str = "your-secret-key-here-change-in-production" - - # Server - HOST: str = "0.0.0.0" - PORT: int = 8000 - # api = CRUD + quick webhooks only; media = live voice WebSockets only; all = single-process dev - SERVICE_MODE: str = "api" - MEDIA_WS_BASE_URL: str = "" - MEDIA_PORT: int = 8001 - - # Database - DATABASE_URL: Optional[str] = None - POSTGRES_USER: str = "efficientai" - POSTGRES_PASSWORD: str = "password" - POSTGRES_HOST: str = "localhost" - POSTGRES_PORT: int = 5432 - POSTGRES_DB: str = "efficientai" - - # Database sharding (call-import row shards; default off) - DB_SHARDING_ENABLED: bool = False - DB_CATALOG_URL: Optional[str] = None - DB_SHARD_ROW_CHUNK_SIZE: int = 500 - DB_POOL_SIZE: int = 10 - DB_MAX_OVERFLOW: int = 20 - DB_SHARD_ENTRIES: List[dict] = [] - - # Redis - REDIS_URL: Optional[str] = None - REDIS_HOST: str = "localhost" - REDIS_PORT: int = 6379 - REDIS_DB: int = 0 - - # File Storage - UPLOAD_DIR: str = "./uploads" - MAX_FILE_SIZE_MB: int = 500 - ALLOWED_AUDIO_FORMATS: Annotated[List[str], NoDecode] = ["wav", "mp3", "flac", "m4a"] - BLOB_STORAGE_PROVIDER: str = "s3" # "s3", "gcs", or "azure" - - # S3 Configuration - S3_ENABLED: bool = False - S3_BUCKET_NAME: Optional[str] = None - S3_REGION: str = "us-east-1" - S3_ACCESS_KEY_ID: Optional[str] = None - S3_SECRET_ACCESS_KEY: Optional[str] = None - S3_ENDPOINT_URL: Optional[str] = None # For S3-compatible services - S3_PREFIX: str = "audio/" # Prefix for audio files in bucket - - # GCS Configuration - GCS_ENABLED: bool = False - GCS_BUCKET_NAME: Optional[str] = None - GCS_PROJECT_ID: Optional[str] = None - GCS_CREDENTIALS_PATH: Optional[str] = None - GCS_SIGNING_SERVICE_ACCOUNT_EMAIL: Optional[str] = None - GCS_PREFIX: str = "audio/" - - # Azure Blob Storage Configuration - AZURE_BLOB_ENABLED: bool = False - AZURE_ACCOUNT_NAME: Optional[str] = None - AZURE_ACCOUNT_KEY: Optional[str] = None - AZURE_CONNECTION_STRING: Optional[str] = None - AZURE_CONTAINER_NAME: Optional[str] = None - AZURE_PREFIX: str = "audio/" - - # Celery - CELERY_BROKER_URL: Optional[str] = None - CELERY_RESULT_BACKEND: Optional[str] = None - - # Call-import worker concurrency limits (Redis fair-share for evaluations) - EVAL_WORKSPACE_INFLIGHT_LIMIT: int = 100 - EVAL_ORG_INFLIGHT_LIMIT: int = 128 - EVAL_GLOBAL_INFLIGHT_LIMIT: int = 128 - EVAL_JOB_INFLIGHT_LIMIT: int = 75 - EVAL_FAIR_DISPATCH_BATCH_SIZE: int = 75 - DIARIZATION_FAIR_DISPATCH_BATCH_SIZE: int = 75 - IMPORT_FAIR_DISPATCH_BATCH_SIZE: int = 75 - IMPORT_WORKSPACE_INFLIGHT_LIMIT: int = 8 - IMPORT_ORG_INFLIGHT_LIMIT: int = 16 - IMPORT_GLOBAL_INFLIGHT_LIMIT: int = 16 - TELEPHONY_IMPORT_CREDIT_LIMIT: int = 1000 - TELEPHONY_IMPORT_CREDIT_WINDOW_SECONDS: int = 60 - TELEPHONY_IMPORT_BACKOFF_BASE_SECONDS: int = 15 - TELEPHONY_IMPORT_BACKOFF_MAX_SECONDS: int = 60 - - # CORS - CORS_ORIGINS: List[str] = ["http://localhost:3000", "http://localhost:8000"] - - # API Settings - API_KEY_HEADER: str = "X-API-Key" - RATE_LIMIT_PER_MINUTE: int = 60 - - # Authentication - AUTH_PROVIDERS: Annotated[List[str], NoDecode] = ["api_key"] - AUTH_LOCAL_ALLOW_SIGNUP: bool = True - AUTH_LOCAL_TOKEN_TTL_MINUTES: int = 15 - AUTH_REFRESH_TOKEN_TTL_DAYS: int = 7 - AUTH_OIDC_ISSUER: Optional[str] = None - AUTH_OIDC_CLIENT_ID: Optional[str] = None - AUTH_OIDC_AUDIENCE: Optional[str] = None - AUTH_OIDC_JWKS_URI: Optional[str] = None - AUTH_OIDC_ORG_CLAIM_PATH: List[str] = [] - AUTH_OIDC_DEFAULT_ORG_NAME: Optional[str] = None - - # Frontend - FRONTEND_DIR: str = "./frontend/dist" - - # Content Security Policy (Report-Only by default; set CSP_REPORT_ONLY=false to enforce) - CSP_ENABLED: bool = True - CSP_REPORT_ONLY: bool = True - CSP_POLICY: str = ( - "default-src 'self'; " - "script-src 'self'; " - "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; " - "font-src 'self' https://fonts.gstatic.com; " - "img-src 'self' data: blob: https:; " - "connect-src 'self' wss: ws:; " - "media-src 'self' blob: https:; " - "frame-src 'self' blob:; " - "object-src 'none'; " - "base-uri 'self'; " - "form-action 'self'; " - "frame-ancestors 'self'" - ) - - # Operational endpoints (/health, /metrics) - OPERATIONAL_PUBLIC: bool = False - OPERATIONAL_TRUSTED_IPS: List[str] = [] - - # SMTP / Email Notifications (for Alerts) - SMTP_HOST: Optional[str] = None # e.g., "smtp.gmail.com" - SMTP_PORT: int = 587 - SMTP_USERNAME: Optional[str] = None - SMTP_PASSWORD: Optional[str] = None - SMTP_FROM_EMAIL: Optional[str] = None # e.g., "alerts@efficientai.dev" - SMTP_FROM_NAME: str = "EfficientAI Alerts" - SMTP_USE_TLS: bool = True - - # Speaker Diarization (Optional) - HUGGINGFACE_TOKEN: Optional[str] = None # For pyannote.audio speaker diarization - DIARIZATION_NUM_SPEAKERS: Optional[int] = 2 # Force pyannote to detect this many speakers (None = auto-detect) - - # Observability / Loki - OBSERVABILITY_ENABLED: bool = False - LOKI_ENABLED: bool = False - LOKI_URL: str = "http://loki:3100" - LOKI_STORAGE: str = "filesystem" # "filesystem" or "s3" - LOKI_MULTI_TENANT: bool = False - LOKI_PLATFORM_TENANT: str = "platform" - LOKI_S3_BUCKET_NAME: Optional[str] = None - LOKI_S3_REGION: str = "us-east-1" - LOKI_S3_ACCESS_KEY_ID: Optional[str] = None - LOKI_S3_SECRET_ACCESS_KEY: Optional[str] = None - LOKI_S3_PREFIX: str = "logs/" - - # Enterprise License (JWT signed with RS256) - EFFICIENTAI_LICENSE: Optional[str] = None - - # Plivo Telephony (optional) - PLIVO_AUTH_ID: str = "" - PLIVO_AUTH_TOKEN: str = "" - PLIVO_VERIFY_APP_UUID: str = "" - PLIVO_WEBHOOK_BASE_URL: str = "" - - # Vobiz Telephony (platform-level, optional) - VOBIZ_AUTH_ID: str = "" - VOBIZ_AUTH_TOKEN: str = "" - VOBIZ_API_BASE: str = "https://api.vobiz.ai" - VOBIZ_MEDIA_BASE: str = "https://media.vobiz.ai" - VOBIZ_WEBHOOK_BASE_URL: str = "" - VOBIZ_WEBHOOK_VERIFY: bool = True - # When False (default), answer XML omits Vobiz ; use pipeline WAV merge only. - VOBIZ_CARRIER_SESSION_RECORDING: bool = False - VOBIZ_FROM_NUMBER: str = "" - VOBIZ_OUTBOUND_POOL: List[str] = [] - VOBIZ_OUTBOUND_POOL_MAX_CONCURRENT_PER_ORG: int = 5 - VOBIZ_DEFAULT_COUNTRY_CODE: str = "91" - - # Provider-agnostic platform outbound pool (preferred over vobiz.outbound_pool). - TELEPHONY_OUTBOUND_POOL: List[Any] = [] - TELEPHONY_OUTBOUND_POOL_MAX_CONCURRENT_PER_ORG: Optional[int] = None - PLIVO_OUTBOUND_POOL: List[str] = [] - EXOTEL_OUTBOUND_POOL: List[str] = [] - - # Recording URL fetch safety (SSRF guards for CSV/direct-URL imports) - RECORDING_URL_ALLOWED_HOST_SUFFIXES: List[str] = [ - "exotel.com", - "plivo.com", - "vobiz.ai", - "amazonaws.com", - "cloudfront.net", - ] - - # Live telephony pipeline recording merge (dual-track → natural mono) - TELEPHONY_BOT_PLAYBACK_DELAY_MS: int = 400 - TELEPHONY_MERGE_CORRELATION_DOUBLE_COUNT: float = 0.35 - - # Judge Alignment (AlignEval-style hybrid integration). - # Operator-only knobs. Per-org thresholds and judge model selection - # live in the database / UI, not here. - JUDGE_ALIGNMENT_ENABLED: bool = True - JUDGE_ALIGNMENT_CSV_MAX_ROWS: int = 5000 - - # Flexprice usage-based billing (optional; disabled when unset) - FLEXPRICE_ENABLED: bool = False - FLEXPRICE_API_KEY: Optional[str] = None - FLEXPRICE_API_HOST: str = "https://us.api.flexprice.io/v1" - # LLM gateway (optional platform-wide proxy for batch LLM calls). - LLM_GATEWAY_ENABLED: bool = False - LLM_GATEWAY_TYPE: str = "bifrost" # bifrost | litellm_proxy - LLM_GATEWAY_BASE_URL: Optional[str] = None - LLM_GATEWAY_VIRTUAL_KEY: Optional[str] = None - LLM_GATEWAY_MASTER_KEY: Optional[str] = None - LLM_GATEWAY_PASSTHROUGH_PROVIDER_KEYS: bool = True - LLM_GATEWAY_INTERFACE: str = "litellm_shim" # litellm_shim | native_openai - - model_config = SettingsConfigDict( - env_file=".env", - env_file_encoding="utf-8", - case_sensitive=True, - # Make .env file optional - if it doesn't exist or has errors, use defaults - env_ignore_empty=True, - extra="ignore", # Ignore extra fields from env file - # Don't validate on assignment to allow validators to handle parsing - validate_assignment=False, - ) - - @field_validator("ALLOWED_AUDIO_FORMATS", mode="before") - @classmethod - def parse_allowed_formats(cls, v: Union[str, List[str], None]) -> List[str]: - """Parse ALLOWED_AUDIO_FORMATS from various formats.""" - # Handle None or empty values - if v is None: - return ["wav", "mp3", "flac", "m4a"] - - if isinstance(v, list): - return v if v else ["wav", "mp3", "flac", "m4a"] - - if isinstance(v, str): - # Handle empty or whitespace-only strings - v = v.strip() - if not v: - return ["wav", "mp3", "flac", "m4a"] - - # Try JSON first - if v.startswith("["): - try: - parsed = json.loads(v) - return parsed if isinstance(parsed, list) and parsed else ["wav", "mp3", "flac", "m4a"] - except (json.JSONDecodeError, ValueError): - pass - - # Fall back to comma-separated - formats = [fmt.strip() for fmt in v.split(",") if fmt.strip()] - return formats if formats else ["wav", "mp3", "flac", "m4a"] - - return ["wav", "mp3", "flac", "m4a"] # Default - - @field_validator("CORS_ORIGINS", mode="before") - @classmethod - def parse_cors_origins(cls, v: Union[str, List[str], None]) -> List[str]: - """Parse CORS_ORIGINS from various formats.""" - # Handle None or empty values - if v is None: - return ["http://localhost:3000", "http://localhost:8000"] - - if isinstance(v, list): - return v if v else ["http://localhost:3000", "http://localhost:8000"] - - if isinstance(v, str): - # Handle empty or whitespace-only strings - v = v.strip() - if not v: - return ["http://localhost:3000", "http://localhost:8000"] - - # Try JSON first - if v.startswith("["): - try: - parsed = json.loads(v) - if isinstance(parsed, list) and parsed: - return parsed - except (json.JSONDecodeError, ValueError): - pass - - # Fall back to comma-separated - origins = [origin.strip() for origin in v.split(",") if origin.strip()] - return origins if origins else ["http://localhost:3000", "http://localhost:8000"] - - return ["http://localhost:3000", "http://localhost:8000"] # Default - - @field_validator("AUTH_PROVIDERS", mode="before") - @classmethod - def parse_auth_providers(cls, v: Union[str, List[str], None]) -> List[str]: - """Parse AUTH_PROVIDERS from JSON arrays or CSV strings.""" - if v is None: - return ["api_key"] - - if isinstance(v, list): - providers = [str(provider).strip().lower() for provider in v if str(provider).strip()] - return providers or ["api_key"] - - if isinstance(v, str): - raw = v.strip() - if not raw: - return ["api_key"] - if raw.startswith("["): - try: - parsed = json.loads(raw) - if isinstance(parsed, list): - providers = [ - str(provider).strip().lower() - for provider in parsed - if str(provider).strip() - ] - return providers or ["api_key"] - except (json.JSONDecodeError, ValueError): - pass - providers = [provider.strip().lower() for provider in raw.split(",") if provider.strip()] - return providers or ["api_key"] - - return ["api_key"] - - @field_validator("AUTH_OIDC_ORG_CLAIM_PATH", mode="before") - @classmethod - def parse_auth_oidc_org_claim_path(cls, v: Union[str, List[str], None]) -> List[str]: - """Parse AUTH_OIDC_ORG_CLAIM_PATH from JSON arrays or dot notation.""" - if v is None: - return [] - - if isinstance(v, list): - return [str(item).strip() for item in v if str(item).strip()] - - if isinstance(v, str): - raw = v.strip() - if not raw: - return [] - if raw.startswith("["): - try: - parsed = json.loads(raw) - if isinstance(parsed, list): - return [str(item).strip() for item in parsed if str(item).strip()] - except (json.JSONDecodeError, ValueError): - pass - return [part.strip() for part in raw.split(".") if part.strip()] - - return [] - - def __init__(self, **kwargs): - super().__init__(**kwargs) - # Build DATABASE_URL if not provided - if not self.DATABASE_URL: - self.DATABASE_URL = ( - f"postgresql://{self.POSTGRES_USER}:{self.POSTGRES_PASSWORD}" - f"@{self.POSTGRES_HOST}:{self.POSTGRES_PORT}/{self.POSTGRES_DB}" - ) - - # Build REDIS_URL if not provided - if not self.REDIS_URL: - self.REDIS_URL = f"redis://{self.REDIS_HOST}:{self.REDIS_PORT}/{self.REDIS_DB}" - - # Build Celery URLs if not provided - if not self.CELERY_BROKER_URL: - self.CELERY_BROKER_URL = self.REDIS_URL - if not self.CELERY_RESULT_BACKEND: - self.CELERY_RESULT_BACKEND = self.REDIS_URL - - -def validate_auth_configuration() -> None: - """Fail fast when external OIDC is licensed and enabled but misconfigured.""" - from app.core.license import has_auth_feature - - providers = {p.strip().lower() for p in (settings.AUTH_PROVIDERS or [])} - if "external_oidc" not in providers: - return - # Listing external_oidc in providers alone does not activate SSO — the - # enterprise license must include oidc_sso (same gate as ExternalOIDCProvider). - if not has_auth_feature("oidc_sso"): - return - missing = [] - if not settings.AUTH_OIDC_ISSUER: - missing.append("AUTH_OIDC_ISSUER") - if not settings.AUTH_OIDC_AUDIENCE: - missing.append("AUTH_OIDC_AUDIENCE") - if missing: - raise RuntimeError( - f"external_oidc is enabled but required settings are missing: {', '.join(missing)}" - ) - - -def apply_service_mode(mode: str) -> None: - """Sync SERVICE_MODE on the module-level settings singleton and os.environ.""" - import os - - normalized = (mode or "api").strip().lower() - os.environ["SERVICE_MODE"] = normalized - settings.SERVICE_MODE = normalized - - -_ENV_REF_PATTERN = re.compile(r"^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$") - - -def _expand_env_ref(value: Any) -> Any: - """Replace ``${VAR}`` with ``os.environ[VAR]`` when loading YAML secrets.""" - if not isinstance(value, str): - return value - match = _ENV_REF_PATTERN.match(value.strip()) - if not match: - return value - return os.environ.get(match.group(1), "") - - -def load_config_from_file(config_path: str) -> None: - """Load configuration from a YAML file and update global settings.""" - import yaml - - config_file = Path(config_path) - if not config_file.exists(): - raise FileNotFoundError(f"Config file not found: {config_path}") - - with open(config_file, "r") as f: - config_data = yaml.safe_load(f) or {} - - # Update settings with YAML values - if "app" in config_data: - app_config = config_data["app"] - if "name" in app_config: - settings.APP_NAME = app_config["name"] - if "version" in app_config: - settings.APP_VERSION = app_config["version"] - if "debug" in app_config: - settings.DEBUG = app_config["debug"] - if "secret_key" in app_config: - settings.SECRET_KEY = app_config["secret_key"] - - if "server" in config_data: - server_config = config_data["server"] - if "host" in server_config: - settings.HOST = server_config["host"] - if "port" in server_config: - settings.PORT = server_config["port"] - - if "database" in config_data: - db_config = config_data["database"] - if "url" in db_config: - settings.DATABASE_URL = db_config["url"] - else: - if "user" in db_config: - settings.POSTGRES_USER = db_config["user"] - if "password" in db_config: - settings.POSTGRES_PASSWORD = db_config["password"] - if "host" in db_config: - settings.POSTGRES_HOST = db_config["host"] - if "port" in db_config: - settings.POSTGRES_PORT = db_config["port"] - if "db" in db_config: - settings.POSTGRES_DB = db_config["db"] - # Rebuild DATABASE_URL - settings.DATABASE_URL = ( - f"postgresql://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}" - f"@{settings.POSTGRES_HOST}:{settings.POSTGRES_PORT}/{settings.POSTGRES_DB}" - ) - if "pool_size" in db_config: - settings.DB_POOL_SIZE = int(db_config["pool_size"]) - if "max_overflow" in db_config: - settings.DB_MAX_OVERFLOW = int(db_config["max_overflow"]) - if "catalog_url" in db_config: - settings.DB_CATALOG_URL = db_config["catalog_url"] - sharding_cfg = db_config.get("sharding") or {} - if isinstance(sharding_cfg, dict): - if "enabled" in sharding_cfg: - settings.DB_SHARDING_ENABLED = bool(sharding_cfg["enabled"]) - if "row_chunk_size" in sharding_cfg: - settings.DB_SHARD_ROW_CHUNK_SIZE = int(sharding_cfg["row_chunk_size"]) - if "shards" in db_config and isinstance(db_config["shards"], list): - settings.DB_SHARD_ENTRIES = list(db_config["shards"]) - - if "redis" in config_data: - redis_config = config_data["redis"] - if "url" in redis_config: - settings.REDIS_URL = redis_config["url"] - else: - if "host" in redis_config: - settings.REDIS_HOST = redis_config["host"] - if "port" in redis_config: - settings.REDIS_PORT = redis_config["port"] - if "db" in redis_config: - settings.REDIS_DB = redis_config["db"] - # Rebuild REDIS_URL - settings.REDIS_URL = f"redis://{settings.REDIS_HOST}:{settings.REDIS_PORT}/{settings.REDIS_DB}" - - if "celery" in config_data: - celery_config = config_data["celery"] - if "broker_url" in celery_config: - settings.CELERY_BROKER_URL = celery_config["broker_url"] - if "result_backend" in celery_config: - settings.CELERY_RESULT_BACKEND = celery_config["result_backend"] - - if "workers" in config_data: - workers_config = config_data["workers"] - if "eval_workspace_inflight_limit" in workers_config: - settings.EVAL_WORKSPACE_INFLIGHT_LIMIT = int( - workers_config["eval_workspace_inflight_limit"] - ) - if "eval_org_inflight_limit" in workers_config: - settings.EVAL_ORG_INFLIGHT_LIMIT = int( - workers_config["eval_org_inflight_limit"] - ) - if "eval_global_inflight_limit" in workers_config: - settings.EVAL_GLOBAL_INFLIGHT_LIMIT = int( - workers_config["eval_global_inflight_limit"] - ) - if "eval_fair_dispatch_batch_size" in workers_config: - settings.EVAL_FAIR_DISPATCH_BATCH_SIZE = int( - workers_config["eval_fair_dispatch_batch_size"] - ) - if "eval_job_inflight_limit" in workers_config: - settings.EVAL_JOB_INFLIGHT_LIMIT = int( - workers_config["eval_job_inflight_limit"] - ) - if "diarization_fair_dispatch_batch_size" in workers_config: - settings.DIARIZATION_FAIR_DISPATCH_BATCH_SIZE = int( - workers_config["diarization_fair_dispatch_batch_size"] - ) - if "import_fair_dispatch_batch_size" in workers_config: - settings.IMPORT_FAIR_DISPATCH_BATCH_SIZE = int( - workers_config["import_fair_dispatch_batch_size"] - ) - if "import_workspace_inflight_limit" in workers_config: - settings.IMPORT_WORKSPACE_INFLIGHT_LIMIT = int( - workers_config["import_workspace_inflight_limit"] - ) - if "import_org_inflight_limit" in workers_config: - settings.IMPORT_ORG_INFLIGHT_LIMIT = int( - workers_config["import_org_inflight_limit"] - ) - if "import_global_inflight_limit" in workers_config: - settings.IMPORT_GLOBAL_INFLIGHT_LIMIT = int( - workers_config["import_global_inflight_limit"] - ) - if "telephony_import_credit_limit" in workers_config: - settings.TELEPHONY_IMPORT_CREDIT_LIMIT = int( - workers_config["telephony_import_credit_limit"] - ) - if "telephony_import_credit_window_seconds" in workers_config: - settings.TELEPHONY_IMPORT_CREDIT_WINDOW_SECONDS = int( - workers_config["telephony_import_credit_window_seconds"] - ) - if "telephony_import_backoff_base_seconds" in workers_config: - settings.TELEPHONY_IMPORT_BACKOFF_BASE_SECONDS = int( - workers_config["telephony_import_backoff_base_seconds"] - ) - if "telephony_import_backoff_max_seconds" in workers_config: - settings.TELEPHONY_IMPORT_BACKOFF_MAX_SECONDS = int( - workers_config["telephony_import_backoff_max_seconds"] - ) - - if "storage" in config_data: - storage_config = config_data["storage"] - if "upload_dir" in storage_config: - settings.UPLOAD_DIR = storage_config["upload_dir"] - if "max_file_size_mb" in storage_config: - settings.MAX_FILE_SIZE_MB = storage_config["max_file_size_mb"] - if "allowed_audio_formats" in storage_config: - settings.ALLOWED_AUDIO_FORMATS = storage_config["allowed_audio_formats"] - if "blob_provider" in storage_config: - settings.BLOB_STORAGE_PROVIDER = str(storage_config["blob_provider"]).strip().lower() - - if "s3" in config_data: - s3_config = config_data["s3"] - if "enabled" in s3_config: - settings.S3_ENABLED = s3_config["enabled"] - if "bucket_name" in s3_config: - settings.S3_BUCKET_NAME = s3_config["bucket_name"] - if "region" in s3_config: - settings.S3_REGION = s3_config["region"] - if "access_key_id" in s3_config: - resolved = _expand_env_ref(s3_config["access_key_id"]) - if resolved: - settings.S3_ACCESS_KEY_ID = resolved - if "secret_access_key" in s3_config: - resolved = _expand_env_ref(s3_config["secret_access_key"]) - if resolved: - settings.S3_SECRET_ACCESS_KEY = resolved - if "endpoint_url" in s3_config: - settings.S3_ENDPOINT_URL = s3_config["endpoint_url"] - if "prefix" in s3_config: - settings.S3_PREFIX = s3_config["prefix"] - - if "gcs" in config_data: - gcs_config = config_data["gcs"] - if "enabled" in gcs_config: - settings.GCS_ENABLED = gcs_config["enabled"] - if "bucket_name" in gcs_config: - settings.GCS_BUCKET_NAME = gcs_config["bucket_name"] - if "project_id" in gcs_config: - settings.GCS_PROJECT_ID = gcs_config["project_id"] - if "credentials_path" in gcs_config: - settings.GCS_CREDENTIALS_PATH = gcs_config["credentials_path"] - if "signing_service_account_email" in gcs_config: - settings.GCS_SIGNING_SERVICE_ACCOUNT_EMAIL = gcs_config[ - "signing_service_account_email" - ] - if "prefix" in gcs_config: - settings.GCS_PREFIX = gcs_config["prefix"] - - if "azure" in config_data: - azure_config = config_data["azure"] - if "enabled" in azure_config: - settings.AZURE_BLOB_ENABLED = azure_config["enabled"] - if "account_name" in azure_config: - settings.AZURE_ACCOUNT_NAME = azure_config["account_name"] - if "account_key" in azure_config: - settings.AZURE_ACCOUNT_KEY = azure_config["account_key"] - if "connection_string" in azure_config: - settings.AZURE_CONNECTION_STRING = azure_config["connection_string"] - if "container_name" in azure_config: - settings.AZURE_CONTAINER_NAME = azure_config["container_name"] - if "prefix" in azure_config: - settings.AZURE_PREFIX = azure_config["prefix"] - - # Backward compatibility: infer provider when blob_provider not set explicitly - if "storage" not in config_data or "blob_provider" not in (config_data.get("storage") or {}): - enabled_providers = sum( - [ - settings.S3_ENABLED, - settings.GCS_ENABLED, - settings.AZURE_BLOB_ENABLED, - ] - ) - if enabled_providers == 1: - if settings.S3_ENABLED: - settings.BLOB_STORAGE_PROVIDER = "s3" - elif settings.GCS_ENABLED: - settings.BLOB_STORAGE_PROVIDER = "gcs" - elif settings.AZURE_BLOB_ENABLED: - settings.BLOB_STORAGE_PROVIDER = "azure" - - if "smtp" in config_data: - smtp_config = config_data["smtp"] - if "host" in smtp_config: - settings.SMTP_HOST = smtp_config["host"] - if "port" in smtp_config: - settings.SMTP_PORT = smtp_config["port"] - if "username" in smtp_config: - settings.SMTP_USERNAME = smtp_config["username"] - if "password" in smtp_config: - settings.SMTP_PASSWORD = smtp_config["password"] - if "from_email" in smtp_config: - settings.SMTP_FROM_EMAIL = smtp_config["from_email"] - if "from_name" in smtp_config: - settings.SMTP_FROM_NAME = smtp_config["from_name"] - if "use_tls" in smtp_config: - settings.SMTP_USE_TLS = smtp_config["use_tls"] - - if "diarization" in config_data: - diarization_config = config_data["diarization"] - if "huggingface_token" in diarization_config and diarization_config["huggingface_token"]: - settings.HUGGINGFACE_TOKEN = diarization_config["huggingface_token"] - if "num_speakers" in diarization_config: - val = diarization_config["num_speakers"] - settings.DIARIZATION_NUM_SPEAKERS = int(val) if val is not None else None - - if "cors" in config_data: - cors_config = config_data["cors"] - if "origins" in cors_config: - settings.CORS_ORIGINS = cors_config["origins"] - - if "api" in config_data: - api_config = config_data["api"] - if "prefix" in api_config: - settings.API_V1_PREFIX = api_config["prefix"] - if "key_header" in api_config: - settings.API_KEY_HEADER = api_config["key_header"] - if "rate_limit_per_minute" in api_config: - settings.RATE_LIMIT_PER_MINUTE = api_config["rate_limit_per_minute"] - - if "auth" in config_data: - auth_config = config_data["auth"] - if "providers" in auth_config: - settings.AUTH_PROVIDERS = auth_config["providers"] - - local_config = auth_config.get("local_password", {}) - if isinstance(local_config, dict): - if "allow_signup" in local_config: - settings.AUTH_LOCAL_ALLOW_SIGNUP = bool(local_config["allow_signup"]) - if "token_ttl_minutes" in local_config: - settings.AUTH_LOCAL_TOKEN_TTL_MINUTES = int(local_config["token_ttl_minutes"]) - if "refresh_token_ttl_days" in local_config: - settings.AUTH_REFRESH_TOKEN_TTL_DAYS = int(local_config["refresh_token_ttl_days"]) - - oidc_config = auth_config.get("oidc", {}) - if isinstance(oidc_config, dict): - if "issuer" in oidc_config: - settings.AUTH_OIDC_ISSUER = oidc_config["issuer"] - if "client_id" in oidc_config: - settings.AUTH_OIDC_CLIENT_ID = oidc_config["client_id"] - if "audience" in oidc_config: - settings.AUTH_OIDC_AUDIENCE = oidc_config["audience"] - if "jwks_uri" in oidc_config: - settings.AUTH_OIDC_JWKS_URI = oidc_config["jwks_uri"] - if "org_claim_path" in oidc_config: - settings.AUTH_OIDC_ORG_CLAIM_PATH = oidc_config["org_claim_path"] - if "default_org_name" in oidc_config: - settings.AUTH_OIDC_DEFAULT_ORG_NAME = oidc_config["default_org_name"] - - if "license" in config_data: - license_config = config_data["license"] - if "key" in license_config: - settings.EFFICIENTAI_LICENSE = license_config["key"] - - if "observability" in config_data: - obs_config = config_data["observability"] - if "enabled" in obs_config: - settings.OBSERVABILITY_ENABLED = bool(obs_config["enabled"]) - if "loki" in obs_config: - loki_config = obs_config["loki"] - if "enabled" in loki_config: - settings.LOKI_ENABLED = bool(loki_config["enabled"]) - if "url" in loki_config: - settings.LOKI_URL = loki_config["url"] - if "storage" in loki_config: - settings.LOKI_STORAGE = loki_config["storage"] - if "multi_tenant" in loki_config: - settings.LOKI_MULTI_TENANT = loki_config["multi_tenant"] - if "platform_tenant" in loki_config: - settings.LOKI_PLATFORM_TENANT = loki_config["platform_tenant"] - if "s3" in loki_config: - loki_s3 = loki_config["s3"] - if "bucket_name" in loki_s3: - settings.LOKI_S3_BUCKET_NAME = loki_s3["bucket_name"] - if "region" in loki_s3: - settings.LOKI_S3_REGION = loki_s3["region"] - if "access_key_id" in loki_s3: - resolved = _expand_env_ref(loki_s3["access_key_id"]) - if resolved: - settings.LOKI_S3_ACCESS_KEY_ID = resolved - if "secret_access_key" in loki_s3: - resolved = _expand_env_ref(loki_s3["secret_access_key"]) - if resolved: - settings.LOKI_S3_SECRET_ACCESS_KEY = resolved - if "prefix" in loki_s3: - settings.LOKI_S3_PREFIX = loki_s3["prefix"] - if "plivo" in config_data: - plivo_cfg = config_data["plivo"] - if plivo_cfg.get("auth_id"): - settings.PLIVO_AUTH_ID = plivo_cfg["auth_id"] - if plivo_cfg.get("auth_token"): - settings.PLIVO_AUTH_TOKEN = plivo_cfg["auth_token"] - if plivo_cfg.get("verify_app_uuid"): - settings.PLIVO_VERIFY_APP_UUID = plivo_cfg["verify_app_uuid"] - if plivo_cfg.get("webhook_base_url"): - settings.PLIVO_WEBHOOK_BASE_URL = plivo_cfg["webhook_base_url"] - if plivo_cfg.get("outbound_pool"): - settings.PLIVO_OUTBOUND_POOL = list(plivo_cfg["outbound_pool"]) - - if "exotel" in config_data: - exotel_cfg = config_data["exotel"] - if exotel_cfg.get("outbound_pool"): - settings.EXOTEL_OUTBOUND_POOL = list(exotel_cfg["outbound_pool"]) - - if "vobiz" in config_data: - vobiz_cfg = config_data["vobiz"] - if vobiz_cfg.get("auth_id"): - settings.VOBIZ_AUTH_ID = vobiz_cfg["auth_id"] - if vobiz_cfg.get("auth_token"): - settings.VOBIZ_AUTH_TOKEN = vobiz_cfg["auth_token"] - if vobiz_cfg.get("api_base"): - settings.VOBIZ_API_BASE = vobiz_cfg["api_base"] - if vobiz_cfg.get("media_base"): - settings.VOBIZ_MEDIA_BASE = vobiz_cfg["media_base"] - if vobiz_cfg.get("webhook_base_url"): - settings.VOBIZ_WEBHOOK_BASE_URL = vobiz_cfg["webhook_base_url"] - if "webhook_verify" in vobiz_cfg: - settings.VOBIZ_WEBHOOK_VERIFY = bool(vobiz_cfg["webhook_verify"]) - if "carrier_session_recording" in vobiz_cfg: - settings.VOBIZ_CARRIER_SESSION_RECORDING = bool(vobiz_cfg["carrier_session_recording"]) - if vobiz_cfg.get("media_ws_base_url"): - settings.MEDIA_WS_BASE_URL = vobiz_cfg["media_ws_base_url"] - if vobiz_cfg.get("from_number"): - settings.VOBIZ_FROM_NUMBER = vobiz_cfg["from_number"] - if vobiz_cfg.get("outbound_pool"): - settings.VOBIZ_OUTBOUND_POOL = list(vobiz_cfg["outbound_pool"]) - if vobiz_cfg.get("outbound_pool_max_concurrent_per_org") is not None: - settings.VOBIZ_OUTBOUND_POOL_MAX_CONCURRENT_PER_ORG = int( - vobiz_cfg["outbound_pool_max_concurrent_per_org"] - ) - if vobiz_cfg.get("default_country_code"): - settings.VOBIZ_DEFAULT_COUNTRY_CODE = str(vobiz_cfg["default_country_code"]).lstrip("+") - - if "telephony" in config_data: - telephony_cfg = config_data["telephony"] - if telephony_cfg.get("outbound_pool"): - settings.TELEPHONY_OUTBOUND_POOL = list(telephony_cfg["outbound_pool"]) - if telephony_cfg.get("outbound_pool_max_concurrent_per_org") is not None: - settings.TELEPHONY_OUTBOUND_POOL_MAX_CONCURRENT_PER_ORG = int( - telephony_cfg["outbound_pool_max_concurrent_per_org"] - ) - - if "judge_alignment" in config_data: - ja_cfg = config_data["judge_alignment"] - if "enabled" in ja_cfg: - settings.JUDGE_ALIGNMENT_ENABLED = bool(ja_cfg["enabled"]) - if "csv_max_rows" in ja_cfg: - settings.JUDGE_ALIGNMENT_CSV_MAX_ROWS = int(ja_cfg["csv_max_rows"]) - - def _apply_llm_gateway_settings(gateway_cfg: dict, *, gateway_type: str) -> None: - if "enabled" in gateway_cfg: - settings.LLM_GATEWAY_ENABLED = bool(gateway_cfg["enabled"]) - settings.LLM_GATEWAY_TYPE = gateway_type - if gateway_cfg.get("base_url"): - settings.LLM_GATEWAY_BASE_URL = gateway_cfg["base_url"] - if gateway_cfg.get("virtual_key"): - settings.LLM_GATEWAY_VIRTUAL_KEY = gateway_cfg["virtual_key"] - if gateway_cfg.get("master_key"): - settings.LLM_GATEWAY_MASTER_KEY = gateway_cfg["master_key"] - if "passthrough_provider_keys" in gateway_cfg: - settings.LLM_GATEWAY_PASSTHROUGH_PROVIDER_KEYS = bool( - gateway_cfg["passthrough_provider_keys"] - ) - if gateway_cfg.get("gateway_interface"): - interface = str(gateway_cfg["gateway_interface"]).strip().lower() - if interface in ("litellm_shim", "native_openai"): - settings.LLM_GATEWAY_INTERFACE = interface - - if "llm_gateway" in config_data: - llm_cfg = config_data["llm_gateway"] - gateway_type = (llm_cfg.get("type") or "bifrost").strip().lower() - if gateway_type not in ("bifrost", "litellm_proxy"): - gateway_type = "bifrost" - _apply_llm_gateway_settings(llm_cfg, gateway_type=gateway_type) - - if "operational" in config_data: - operational_config = config_data["operational"] - if "public" in operational_config: - settings.OPERATIONAL_PUBLIC = bool(operational_config["public"]) - if "trusted_ips" in operational_config: - settings.OPERATIONAL_TRUSTED_IPS = operational_config["trusted_ips"] - - if "flexprice" in config_data: - flexprice_config = config_data["flexprice"] - if "enabled" in flexprice_config: - settings.FLEXPRICE_ENABLED = bool(flexprice_config["enabled"]) - if flexprice_config.get("api_key"): - settings.FLEXPRICE_API_KEY = flexprice_config["api_key"] - if flexprice_config.get("api_host"): - settings.FLEXPRICE_API_HOST = flexprice_config["api_host"] - - # Update Celery URLs if they weren't explicitly set - if not settings.CELERY_BROKER_URL: - settings.CELERY_BROKER_URL = settings.REDIS_URL - if not settings.CELERY_RESULT_BACKEND: - settings.CELERY_RESULT_BACKEND = settings.REDIS_URL - - -# Initialize settings with error handling for problematic env vars -# If .env file has invalid format, we'll use defaults (YAML config will override anyway) -try: - settings = Settings() -except Exception as e: - # If there's an error loading from .env (e.g., invalid JSON in list fields), - # create settings with defaults. The YAML config loaded later will override these. - import warnings - import os - - warnings.warn( - f"Error loading .env file, using defaults. YAML config will override. Error: {str(e)[:100]}", - UserWarning, - stacklevel=2 - ) - - # Try to create settings without .env file by temporarily removing it - env_file = ".env" - if os.path.exists(env_file): - # Temporarily rename .env to avoid loading it - backup_file = f"{env_file}.backup" - try: - os.rename(env_file, backup_file) - settings = Settings() - os.rename(backup_file, env_file) - except Exception: - # If rename fails or Settings still fails, restore and use defaults - if os.path.exists(backup_file): - try: - os.rename(backup_file, env_file) - except Exception: - pass - # Create with explicit defaults - manually construct with default values - settings = Settings( - ALLOWED_AUDIO_FORMATS=["wav", "mp3", "flac", "m4a"], - CORS_ORIGINS=["http://localhost:3000", "http://localhost:8000"], - _env_file=None, # Don't load .env - ) - else: - # No .env file, create normally - settings = Settings() +"""Configuration management using Pydantic settings.""" + +import json +import os +import re +import yaml +from pathlib import Path +from typing import Annotated, Any, Dict, List, Optional, Union +from pydantic import field_validator +from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict + + +class Settings(BaseSettings): + """Application settings.""" + + # Application + APP_NAME: str = "Voice AI Evaluation Platform" + APP_VERSION: str = "0.1.0" + API_V1_PREFIX: str = "/api/v1" + DEBUG: bool = True + SECRET_KEY: str = "your-secret-key-here-change-in-production" + + # Server + HOST: str = "0.0.0.0" + PORT: int = 8000 + # api = CRUD + quick webhooks only; media = live voice WebSockets only; all = single-process dev + SERVICE_MODE: str = "api" + MEDIA_WS_BASE_URL: str = "" + MEDIA_PORT: int = 8001 + + # Database + DATABASE_URL: Optional[str] = None + POSTGRES_USER: str = "efficientai" + POSTGRES_PASSWORD: str = "password" + POSTGRES_HOST: str = "localhost" + POSTGRES_PORT: int = 5432 + POSTGRES_DB: str = "efficientai" + + # Database sharding (call-import row shards; default off) + DB_SHARDING_ENABLED: bool = False + DB_CATALOG_URL: Optional[str] = None + DB_SHARD_ROW_CHUNK_SIZE: int = 500 + DB_POOL_SIZE: int = 10 + DB_MAX_OVERFLOW: int = 20 + DB_SHARD_ENTRIES: List[dict] = [] + + # Redis + REDIS_URL: Optional[str] = None + REDIS_HOST: str = "localhost" + REDIS_PORT: int = 6379 + REDIS_DB: int = 0 + + # File Storage + UPLOAD_DIR: str = "./uploads" + MAX_FILE_SIZE_MB: int = 500 + ALLOWED_AUDIO_FORMATS: Annotated[List[str], NoDecode] = ["wav", "mp3", "flac", "m4a"] + BLOB_STORAGE_PROVIDER: str = "s3" # "s3", "gcs", or "azure" + + # S3 Configuration + S3_ENABLED: bool = False + S3_BUCKET_NAME: Optional[str] = None + S3_REGION: str = "us-east-1" + S3_ACCESS_KEY_ID: Optional[str] = None + S3_SECRET_ACCESS_KEY: Optional[str] = None + S3_ENDPOINT_URL: Optional[str] = None # For S3-compatible services + S3_PREFIX: str = "audio/" # Prefix for audio files in bucket + + # GCS Configuration + GCS_ENABLED: bool = False + GCS_BUCKET_NAME: Optional[str] = None + GCS_PROJECT_ID: Optional[str] = None + GCS_CREDENTIALS_PATH: Optional[str] = None + GCS_SIGNING_SERVICE_ACCOUNT_EMAIL: Optional[str] = None + GCS_PREFIX: str = "audio/" + + # Azure Blob Storage Configuration + AZURE_BLOB_ENABLED: bool = False + AZURE_ACCOUNT_NAME: Optional[str] = None + AZURE_ACCOUNT_KEY: Optional[str] = None + AZURE_CONNECTION_STRING: Optional[str] = None + AZURE_CONTAINER_NAME: Optional[str] = None + AZURE_PREFIX: str = "audio/" + + # Celery + CELERY_BROKER_URL: Optional[str] = None + CELERY_RESULT_BACKEND: Optional[str] = None + + # Call-import worker concurrency limits (Redis fair-share for evaluations) + EVAL_WORKSPACE_INFLIGHT_LIMIT: int = 100 + EVAL_ORG_INFLIGHT_LIMIT: int = 128 + EVAL_GLOBAL_INFLIGHT_LIMIT: int = 128 + EVAL_JOB_INFLIGHT_LIMIT: int = 75 + EVAL_FAIR_DISPATCH_BATCH_SIZE: int = 75 + DIARIZATION_FAIR_DISPATCH_BATCH_SIZE: int = 75 + IMPORT_FAIR_DISPATCH_BATCH_SIZE: int = 75 + IMPORT_WORKSPACE_INFLIGHT_LIMIT: int = 8 + IMPORT_ORG_INFLIGHT_LIMIT: int = 16 + IMPORT_GLOBAL_INFLIGHT_LIMIT: int = 16 + TELEPHONY_IMPORT_CREDIT_LIMIT: int = 1000 + TELEPHONY_IMPORT_CREDIT_WINDOW_SECONDS: int = 60 + TELEPHONY_IMPORT_BACKOFF_BASE_SECONDS: int = 15 + TELEPHONY_IMPORT_BACKOFF_MAX_SECONDS: int = 60 + + # CORS + CORS_ORIGINS: List[str] = ["http://localhost:3000", "http://localhost:8000"] + + # API Settings + API_KEY_HEADER: str = "X-API-Key" + RATE_LIMIT_PER_MINUTE: int = 60 + + # Authentication + AUTH_PROVIDERS: Annotated[List[str], NoDecode] = ["api_key"] + AUTH_LOCAL_ALLOW_SIGNUP: bool = True + AUTH_GATED_SIGNUP_ENABLED: bool = False + AUTH_LOCAL_TOKEN_TTL_MINUTES: int = 15 + AUTH_REFRESH_TOKEN_TTL_DAYS: int = 7 + AUTH_OIDC_ISSUER: Optional[str] = None + AUTH_OIDC_CLIENT_ID: Optional[str] = None + AUTH_OIDC_AUDIENCE: Optional[str] = None + AUTH_OIDC_JWKS_URI: Optional[str] = None + AUTH_OIDC_ORG_CLAIM_PATH: List[str] = [] + AUTH_OIDC_DEFAULT_ORG_NAME: Optional[str] = None + + # Frontend + FRONTEND_DIR: str = "./frontend/dist" + + # Content Security Policy (Report-Only by default; set CSP_REPORT_ONLY=false to enforce) + CSP_ENABLED: bool = True + CSP_REPORT_ONLY: bool = True + CSP_POLICY: str = ( + "default-src 'self'; " + "script-src 'self'; " + "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; " + "font-src 'self' https://fonts.gstatic.com; " + "img-src 'self' data: blob: https:; " + "connect-src 'self' wss: ws:; " + "media-src 'self' blob: https:; " + "frame-src 'self' blob:; " + "object-src 'none'; " + "base-uri 'self'; " + "form-action 'self'; " + "frame-ancestors 'self'" + ) + + # Operational endpoints (/health, /metrics) + OPERATIONAL_PUBLIC: bool = False + OPERATIONAL_TRUSTED_IPS: List[str] = [] + + # SMTP / Email Notifications (for Alerts) + SMTP_HOST: Optional[str] = None # e.g., "smtp.gmail.com" + SMTP_PORT: int = 587 + SMTP_USERNAME: Optional[str] = None + SMTP_PASSWORD: Optional[str] = None + SMTP_FROM_EMAIL: Optional[str] = None # e.g., "alerts@efficientai.dev" + SMTP_FROM_NAME: str = "EfficientAI Alerts" + SMTP_USE_TLS: bool = True + + # Speaker Diarization (Optional) + HUGGINGFACE_TOKEN: Optional[str] = None # For pyannote.audio speaker diarization + DIARIZATION_NUM_SPEAKERS: Optional[int] = 2 # Force pyannote to detect this many speakers (None = auto-detect) + + # Observability / Loki + OBSERVABILITY_ENABLED: bool = False + LOKI_ENABLED: bool = False + LOKI_URL: str = "http://loki:3100" + LOKI_STORAGE: str = "filesystem" # "filesystem" or "s3" + LOKI_MULTI_TENANT: bool = False + LOKI_PLATFORM_TENANT: str = "platform" + LOKI_S3_BUCKET_NAME: Optional[str] = None + LOKI_S3_REGION: str = "us-east-1" + LOKI_S3_ACCESS_KEY_ID: Optional[str] = None + LOKI_S3_SECRET_ACCESS_KEY: Optional[str] = None + LOKI_S3_PREFIX: str = "logs/" + + # Enterprise License (JWT signed with RS256) + EFFICIENTAI_LICENSE: Optional[str] = None + + # Plivo Telephony (optional) + PLIVO_AUTH_ID: str = "" + PLIVO_AUTH_TOKEN: str = "" + PLIVO_VERIFY_APP_UUID: str = "" + PLIVO_WEBHOOK_BASE_URL: str = "" + + # Vobiz Telephony (platform-level, optional) + VOBIZ_AUTH_ID: str = "" + VOBIZ_AUTH_TOKEN: str = "" + VOBIZ_API_BASE: str = "https://api.vobiz.ai" + VOBIZ_MEDIA_BASE: str = "https://media.vobiz.ai" + VOBIZ_WEBHOOK_BASE_URL: str = "" + VOBIZ_WEBHOOK_VERIFY: bool = True + # When False (default), answer XML omits Vobiz ; use pipeline WAV merge only. + VOBIZ_CARRIER_SESSION_RECORDING: bool = False + VOBIZ_FROM_NUMBER: str = "" + VOBIZ_OUTBOUND_POOL: List[str] = [] + VOBIZ_OUTBOUND_POOL_MAX_CONCURRENT_PER_ORG: int = 5 + VOBIZ_DEFAULT_COUNTRY_CODE: str = "91" + + # Provider-agnostic platform outbound pool (preferred over vobiz.outbound_pool). + TELEPHONY_OUTBOUND_POOL: List[Any] = [] + TELEPHONY_OUTBOUND_POOL_MAX_CONCURRENT_PER_ORG: Optional[int] = None + PLIVO_OUTBOUND_POOL: List[str] = [] + EXOTEL_OUTBOUND_POOL: List[str] = [] + + # Recording URL fetch safety (SSRF guards for CSV/direct-URL imports) + RECORDING_URL_ALLOWED_HOST_SUFFIXES: List[str] = [ + "exotel.com", + "plivo.com", + "vobiz.ai", + "amazonaws.com", + "cloudfront.net", + ] + + # Live telephony pipeline recording merge (dual-track → natural mono) + TELEPHONY_BOT_PLAYBACK_DELAY_MS: int = 400 + TELEPHONY_MERGE_CORRELATION_DOUBLE_COUNT: float = 0.35 + + # Judge Alignment (AlignEval-style hybrid integration). + # Operator-only knobs. Per-org thresholds and judge model selection + # live in the database / UI, not here. + JUDGE_ALIGNMENT_ENABLED: bool = True + JUDGE_ALIGNMENT_CSV_MAX_ROWS: int = 5000 + + # Flexprice usage-based billing (optional; disabled when unset) + FLEXPRICE_ENABLED: bool = False + FLEXPRICE_API_KEY: Optional[str] = None + FLEXPRICE_API_HOST: str = "https://us.api.flexprice.io/v1" + # LLM gateway (optional platform-wide proxy for batch LLM calls). + LLM_GATEWAY_ENABLED: bool = False + LLM_GATEWAY_TYPE: str = "bifrost" # bifrost | litellm_proxy + LLM_GATEWAY_BASE_URL: Optional[str] = None + LLM_GATEWAY_VIRTUAL_KEY: Optional[str] = None + LLM_GATEWAY_MASTER_KEY: Optional[str] = None + LLM_GATEWAY_PASSTHROUGH_PROVIDER_KEYS: bool = True + LLM_GATEWAY_INTERFACE: str = "litellm_shim" # litellm_shim | native_openai + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + case_sensitive=True, + # Make .env file optional - if it doesn't exist or has errors, use defaults + env_ignore_empty=True, + extra="ignore", # Ignore extra fields from env file + # Don't validate on assignment to allow validators to handle parsing + validate_assignment=False, + ) + + @field_validator("ALLOWED_AUDIO_FORMATS", mode="before") + @classmethod + def parse_allowed_formats(cls, v: Union[str, List[str], None]) -> List[str]: + """Parse ALLOWED_AUDIO_FORMATS from various formats.""" + # Handle None or empty values + if v is None: + return ["wav", "mp3", "flac", "m4a"] + + if isinstance(v, list): + return v if v else ["wav", "mp3", "flac", "m4a"] + + if isinstance(v, str): + # Handle empty or whitespace-only strings + v = v.strip() + if not v: + return ["wav", "mp3", "flac", "m4a"] + + # Try JSON first + if v.startswith("["): + try: + parsed = json.loads(v) + return parsed if isinstance(parsed, list) and parsed else ["wav", "mp3", "flac", "m4a"] + except (json.JSONDecodeError, ValueError): + pass + + # Fall back to comma-separated + formats = [fmt.strip() for fmt in v.split(",") if fmt.strip()] + return formats if formats else ["wav", "mp3", "flac", "m4a"] + + return ["wav", "mp3", "flac", "m4a"] # Default + + @field_validator("CORS_ORIGINS", mode="before") + @classmethod + def parse_cors_origins(cls, v: Union[str, List[str], None]) -> List[str]: + """Parse CORS_ORIGINS from various formats.""" + # Handle None or empty values + if v is None: + return ["http://localhost:3000", "http://localhost:8000"] + + if isinstance(v, list): + return v if v else ["http://localhost:3000", "http://localhost:8000"] + + if isinstance(v, str): + # Handle empty or whitespace-only strings + v = v.strip() + if not v: + return ["http://localhost:3000", "http://localhost:8000"] + + # Try JSON first + if v.startswith("["): + try: + parsed = json.loads(v) + if isinstance(parsed, list) and parsed: + return parsed + except (json.JSONDecodeError, ValueError): + pass + + # Fall back to comma-separated + origins = [origin.strip() for origin in v.split(",") if origin.strip()] + return origins if origins else ["http://localhost:3000", "http://localhost:8000"] + + return ["http://localhost:3000", "http://localhost:8000"] # Default + + @field_validator("AUTH_PROVIDERS", mode="before") + @classmethod + def parse_auth_providers(cls, v: Union[str, List[str], None]) -> List[str]: + """Parse AUTH_PROVIDERS from JSON arrays or CSV strings.""" + if v is None: + return ["api_key"] + + if isinstance(v, list): + providers = [str(provider).strip().lower() for provider in v if str(provider).strip()] + return providers or ["api_key"] + + if isinstance(v, str): + raw = v.strip() + if not raw: + return ["api_key"] + if raw.startswith("["): + try: + parsed = json.loads(raw) + if isinstance(parsed, list): + providers = [ + str(provider).strip().lower() + for provider in parsed + if str(provider).strip() + ] + return providers or ["api_key"] + except (json.JSONDecodeError, ValueError): + pass + providers = [provider.strip().lower() for provider in raw.split(",") if provider.strip()] + return providers or ["api_key"] + + return ["api_key"] + + @field_validator("AUTH_OIDC_ORG_CLAIM_PATH", mode="before") + @classmethod + def parse_auth_oidc_org_claim_path(cls, v: Union[str, List[str], None]) -> List[str]: + """Parse AUTH_OIDC_ORG_CLAIM_PATH from JSON arrays or dot notation.""" + if v is None: + return [] + + if isinstance(v, list): + return [str(item).strip() for item in v if str(item).strip()] + + if isinstance(v, str): + raw = v.strip() + if not raw: + return [] + if raw.startswith("["): + try: + parsed = json.loads(raw) + if isinstance(parsed, list): + return [str(item).strip() for item in parsed if str(item).strip()] + except (json.JSONDecodeError, ValueError): + pass + return [part.strip() for part in raw.split(".") if part.strip()] + + return [] + + def __init__(self, **kwargs): + super().__init__(**kwargs) + # Build DATABASE_URL if not provided + if not self.DATABASE_URL: + self.DATABASE_URL = ( + f"postgresql://{self.POSTGRES_USER}:{self.POSTGRES_PASSWORD}" + f"@{self.POSTGRES_HOST}:{self.POSTGRES_PORT}/{self.POSTGRES_DB}" + ) + + # Build REDIS_URL if not provided + if not self.REDIS_URL: + self.REDIS_URL = f"redis://{self.REDIS_HOST}:{self.REDIS_PORT}/{self.REDIS_DB}" + + # Build Celery URLs if not provided + if not self.CELERY_BROKER_URL: + self.CELERY_BROKER_URL = self.REDIS_URL + if not self.CELERY_RESULT_BACKEND: + self.CELERY_RESULT_BACKEND = self.REDIS_URL + + +def validate_auth_configuration() -> None: + """Fail fast when external OIDC is licensed and enabled but misconfigured.""" + from app.core.license import has_auth_feature + + providers = {p.strip().lower() for p in (settings.AUTH_PROVIDERS or [])} + if "external_oidc" not in providers: + return + # Listing external_oidc in providers alone does not activate SSO — the + # enterprise license must include oidc_sso (same gate as ExternalOIDCProvider). + if not has_auth_feature("oidc_sso"): + return + missing = [] + if not settings.AUTH_OIDC_ISSUER: + missing.append("AUTH_OIDC_ISSUER") + if not settings.AUTH_OIDC_AUDIENCE: + missing.append("AUTH_OIDC_AUDIENCE") + if missing: + raise RuntimeError( + f"external_oidc is enabled but required settings are missing: {', '.join(missing)}" + ) + + +def apply_service_mode(mode: str) -> None: + """Sync SERVICE_MODE on the module-level settings singleton and os.environ.""" + import os + + normalized = (mode or "api").strip().lower() + os.environ["SERVICE_MODE"] = normalized + settings.SERVICE_MODE = normalized + + +_ENV_REF_PATTERN = re.compile(r"^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$") + + +def _expand_env_ref(value: Any) -> Any: + """Replace ``${VAR}`` with ``os.environ[VAR]`` when loading YAML secrets.""" + if not isinstance(value, str): + return value + match = _ENV_REF_PATTERN.match(value.strip()) + if not match: + return value + return os.environ.get(match.group(1), "") + + +def load_config_from_file(config_path: str) -> None: + """Load configuration from a YAML file and update global settings.""" + import yaml + + config_file = Path(config_path) + if not config_file.exists(): + raise FileNotFoundError(f"Config file not found: {config_path}") + + with open(config_file, "r") as f: + config_data = yaml.safe_load(f) or {} + + # Update settings with YAML values + if "app" in config_data: + app_config = config_data["app"] + if "name" in app_config: + settings.APP_NAME = app_config["name"] + if "version" in app_config: + settings.APP_VERSION = app_config["version"] + if "debug" in app_config: + settings.DEBUG = app_config["debug"] + if "secret_key" in app_config: + settings.SECRET_KEY = app_config["secret_key"] + + if "server" in config_data: + server_config = config_data["server"] + if "host" in server_config: + settings.HOST = server_config["host"] + if "port" in server_config: + settings.PORT = server_config["port"] + + if "database" in config_data: + db_config = config_data["database"] + if "url" in db_config: + settings.DATABASE_URL = db_config["url"] + else: + if "user" in db_config: + settings.POSTGRES_USER = db_config["user"] + if "password" in db_config: + settings.POSTGRES_PASSWORD = db_config["password"] + if "host" in db_config: + settings.POSTGRES_HOST = db_config["host"] + if "port" in db_config: + settings.POSTGRES_PORT = db_config["port"] + if "db" in db_config: + settings.POSTGRES_DB = db_config["db"] + # Rebuild DATABASE_URL + settings.DATABASE_URL = ( + f"postgresql://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}" + f"@{settings.POSTGRES_HOST}:{settings.POSTGRES_PORT}/{settings.POSTGRES_DB}" + ) + if "pool_size" in db_config: + settings.DB_POOL_SIZE = int(db_config["pool_size"]) + if "max_overflow" in db_config: + settings.DB_MAX_OVERFLOW = int(db_config["max_overflow"]) + if "catalog_url" in db_config: + settings.DB_CATALOG_URL = db_config["catalog_url"] + sharding_cfg = db_config.get("sharding") or {} + if isinstance(sharding_cfg, dict): + if "enabled" in sharding_cfg: + settings.DB_SHARDING_ENABLED = bool(sharding_cfg["enabled"]) + if "row_chunk_size" in sharding_cfg: + settings.DB_SHARD_ROW_CHUNK_SIZE = int(sharding_cfg["row_chunk_size"]) + if "shards" in db_config and isinstance(db_config["shards"], list): + settings.DB_SHARD_ENTRIES = list(db_config["shards"]) + + if "redis" in config_data: + redis_config = config_data["redis"] + if "url" in redis_config: + settings.REDIS_URL = redis_config["url"] + else: + if "host" in redis_config: + settings.REDIS_HOST = redis_config["host"] + if "port" in redis_config: + settings.REDIS_PORT = redis_config["port"] + if "db" in redis_config: + settings.REDIS_DB = redis_config["db"] + # Rebuild REDIS_URL + settings.REDIS_URL = f"redis://{settings.REDIS_HOST}:{settings.REDIS_PORT}/{settings.REDIS_DB}" + + if "celery" in config_data: + celery_config = config_data["celery"] + if "broker_url" in celery_config: + settings.CELERY_BROKER_URL = celery_config["broker_url"] + if "result_backend" in celery_config: + settings.CELERY_RESULT_BACKEND = celery_config["result_backend"] + + if "workers" in config_data: + workers_config = config_data["workers"] + if "eval_workspace_inflight_limit" in workers_config: + settings.EVAL_WORKSPACE_INFLIGHT_LIMIT = int( + workers_config["eval_workspace_inflight_limit"] + ) + if "eval_org_inflight_limit" in workers_config: + settings.EVAL_ORG_INFLIGHT_LIMIT = int( + workers_config["eval_org_inflight_limit"] + ) + if "eval_global_inflight_limit" in workers_config: + settings.EVAL_GLOBAL_INFLIGHT_LIMIT = int( + workers_config["eval_global_inflight_limit"] + ) + if "eval_fair_dispatch_batch_size" in workers_config: + settings.EVAL_FAIR_DISPATCH_BATCH_SIZE = int( + workers_config["eval_fair_dispatch_batch_size"] + ) + if "eval_job_inflight_limit" in workers_config: + settings.EVAL_JOB_INFLIGHT_LIMIT = int( + workers_config["eval_job_inflight_limit"] + ) + if "diarization_fair_dispatch_batch_size" in workers_config: + settings.DIARIZATION_FAIR_DISPATCH_BATCH_SIZE = int( + workers_config["diarization_fair_dispatch_batch_size"] + ) + if "import_fair_dispatch_batch_size" in workers_config: + settings.IMPORT_FAIR_DISPATCH_BATCH_SIZE = int( + workers_config["import_fair_dispatch_batch_size"] + ) + if "import_workspace_inflight_limit" in workers_config: + settings.IMPORT_WORKSPACE_INFLIGHT_LIMIT = int( + workers_config["import_workspace_inflight_limit"] + ) + if "import_org_inflight_limit" in workers_config: + settings.IMPORT_ORG_INFLIGHT_LIMIT = int( + workers_config["import_org_inflight_limit"] + ) + if "import_global_inflight_limit" in workers_config: + settings.IMPORT_GLOBAL_INFLIGHT_LIMIT = int( + workers_config["import_global_inflight_limit"] + ) + if "telephony_import_credit_limit" in workers_config: + settings.TELEPHONY_IMPORT_CREDIT_LIMIT = int( + workers_config["telephony_import_credit_limit"] + ) + if "telephony_import_credit_window_seconds" in workers_config: + settings.TELEPHONY_IMPORT_CREDIT_WINDOW_SECONDS = int( + workers_config["telephony_import_credit_window_seconds"] + ) + if "telephony_import_backoff_base_seconds" in workers_config: + settings.TELEPHONY_IMPORT_BACKOFF_BASE_SECONDS = int( + workers_config["telephony_import_backoff_base_seconds"] + ) + if "telephony_import_backoff_max_seconds" in workers_config: + settings.TELEPHONY_IMPORT_BACKOFF_MAX_SECONDS = int( + workers_config["telephony_import_backoff_max_seconds"] + ) + + if "storage" in config_data: + storage_config = config_data["storage"] + if "upload_dir" in storage_config: + settings.UPLOAD_DIR = storage_config["upload_dir"] + if "max_file_size_mb" in storage_config: + settings.MAX_FILE_SIZE_MB = storage_config["max_file_size_mb"] + if "allowed_audio_formats" in storage_config: + settings.ALLOWED_AUDIO_FORMATS = storage_config["allowed_audio_formats"] + if "blob_provider" in storage_config: + settings.BLOB_STORAGE_PROVIDER = str(storage_config["blob_provider"]).strip().lower() + + if "s3" in config_data: + s3_config = config_data["s3"] + if "enabled" in s3_config: + settings.S3_ENABLED = s3_config["enabled"] + if "bucket_name" in s3_config: + settings.S3_BUCKET_NAME = s3_config["bucket_name"] + if "region" in s3_config: + settings.S3_REGION = s3_config["region"] + if "access_key_id" in s3_config: + resolved = _expand_env_ref(s3_config["access_key_id"]) + if resolved: + settings.S3_ACCESS_KEY_ID = resolved + if "secret_access_key" in s3_config: + resolved = _expand_env_ref(s3_config["secret_access_key"]) + if resolved: + settings.S3_SECRET_ACCESS_KEY = resolved + if "endpoint_url" in s3_config: + settings.S3_ENDPOINT_URL = s3_config["endpoint_url"] + if "prefix" in s3_config: + settings.S3_PREFIX = s3_config["prefix"] + + if "gcs" in config_data: + gcs_config = config_data["gcs"] + if "enabled" in gcs_config: + settings.GCS_ENABLED = gcs_config["enabled"] + if "bucket_name" in gcs_config: + settings.GCS_BUCKET_NAME = gcs_config["bucket_name"] + if "project_id" in gcs_config: + settings.GCS_PROJECT_ID = gcs_config["project_id"] + if "credentials_path" in gcs_config: + settings.GCS_CREDENTIALS_PATH = gcs_config["credentials_path"] + if "signing_service_account_email" in gcs_config: + settings.GCS_SIGNING_SERVICE_ACCOUNT_EMAIL = gcs_config[ + "signing_service_account_email" + ] + if "prefix" in gcs_config: + settings.GCS_PREFIX = gcs_config["prefix"] + + if "azure" in config_data: + azure_config = config_data["azure"] + if "enabled" in azure_config: + settings.AZURE_BLOB_ENABLED = azure_config["enabled"] + if "account_name" in azure_config: + settings.AZURE_ACCOUNT_NAME = azure_config["account_name"] + if "account_key" in azure_config: + settings.AZURE_ACCOUNT_KEY = azure_config["account_key"] + if "connection_string" in azure_config: + settings.AZURE_CONNECTION_STRING = azure_config["connection_string"] + if "container_name" in azure_config: + settings.AZURE_CONTAINER_NAME = azure_config["container_name"] + if "prefix" in azure_config: + settings.AZURE_PREFIX = azure_config["prefix"] + + # Backward compatibility: infer provider when blob_provider not set explicitly + if "storage" not in config_data or "blob_provider" not in (config_data.get("storage") or {}): + enabled_providers = sum( + [ + settings.S3_ENABLED, + settings.GCS_ENABLED, + settings.AZURE_BLOB_ENABLED, + ] + ) + if enabled_providers == 1: + if settings.S3_ENABLED: + settings.BLOB_STORAGE_PROVIDER = "s3" + elif settings.GCS_ENABLED: + settings.BLOB_STORAGE_PROVIDER = "gcs" + elif settings.AZURE_BLOB_ENABLED: + settings.BLOB_STORAGE_PROVIDER = "azure" + + if "smtp" in config_data: + smtp_config = config_data["smtp"] + if "host" in smtp_config: + settings.SMTP_HOST = smtp_config["host"] + if "port" in smtp_config: + settings.SMTP_PORT = smtp_config["port"] + if "username" in smtp_config: + settings.SMTP_USERNAME = smtp_config["username"] + if "password" in smtp_config: + settings.SMTP_PASSWORD = smtp_config["password"] + if "from_email" in smtp_config: + settings.SMTP_FROM_EMAIL = smtp_config["from_email"] + if "from_name" in smtp_config: + settings.SMTP_FROM_NAME = smtp_config["from_name"] + if "use_tls" in smtp_config: + settings.SMTP_USE_TLS = smtp_config["use_tls"] + + if "diarization" in config_data: + diarization_config = config_data["diarization"] + if "huggingface_token" in diarization_config and diarization_config["huggingface_token"]: + settings.HUGGINGFACE_TOKEN = diarization_config["huggingface_token"] + if "num_speakers" in diarization_config: + val = diarization_config["num_speakers"] + settings.DIARIZATION_NUM_SPEAKERS = int(val) if val is not None else None + + if "cors" in config_data: + cors_config = config_data["cors"] + if "origins" in cors_config: + settings.CORS_ORIGINS = cors_config["origins"] + + if "api" in config_data: + api_config = config_data["api"] + if "prefix" in api_config: + settings.API_V1_PREFIX = api_config["prefix"] + if "key_header" in api_config: + settings.API_KEY_HEADER = api_config["key_header"] + if "rate_limit_per_minute" in api_config: + settings.RATE_LIMIT_PER_MINUTE = api_config["rate_limit_per_minute"] + + if "auth" in config_data: + auth_config = config_data["auth"] + if "providers" in auth_config: + settings.AUTH_PROVIDERS = auth_config["providers"] + + local_config = auth_config.get("local_password", {}) + if isinstance(local_config, dict): + if "allow_signup" in local_config: + settings.AUTH_LOCAL_ALLOW_SIGNUP = bool(local_config["allow_signup"]) + if "token_ttl_minutes" in local_config: + settings.AUTH_LOCAL_TOKEN_TTL_MINUTES = int(local_config["token_ttl_minutes"]) + if "refresh_token_ttl_days" in local_config: + settings.AUTH_REFRESH_TOKEN_TTL_DAYS = int(local_config["refresh_token_ttl_days"]) + gated_config = local_config.get("gated_signup", {}) + if isinstance(gated_config, dict) and "enabled" in gated_config: + settings.AUTH_GATED_SIGNUP_ENABLED = bool(gated_config["enabled"]) + + oidc_config = auth_config.get("oidc", {}) + if isinstance(oidc_config, dict): + if "issuer" in oidc_config: + settings.AUTH_OIDC_ISSUER = oidc_config["issuer"] + if "client_id" in oidc_config: + settings.AUTH_OIDC_CLIENT_ID = oidc_config["client_id"] + if "audience" in oidc_config: + settings.AUTH_OIDC_AUDIENCE = oidc_config["audience"] + if "jwks_uri" in oidc_config: + settings.AUTH_OIDC_JWKS_URI = oidc_config["jwks_uri"] + if "org_claim_path" in oidc_config: + settings.AUTH_OIDC_ORG_CLAIM_PATH = oidc_config["org_claim_path"] + if "default_org_name" in oidc_config: + settings.AUTH_OIDC_DEFAULT_ORG_NAME = oidc_config["default_org_name"] + + if "license" in config_data: + license_config = config_data["license"] + if "key" in license_config: + settings.EFFICIENTAI_LICENSE = license_config["key"] + + if "observability" in config_data: + obs_config = config_data["observability"] + if "enabled" in obs_config: + settings.OBSERVABILITY_ENABLED = bool(obs_config["enabled"]) + if "loki" in obs_config: + loki_config = obs_config["loki"] + if "enabled" in loki_config: + settings.LOKI_ENABLED = bool(loki_config["enabled"]) + if "url" in loki_config: + settings.LOKI_URL = loki_config["url"] + if "storage" in loki_config: + settings.LOKI_STORAGE = loki_config["storage"] + if "multi_tenant" in loki_config: + settings.LOKI_MULTI_TENANT = loki_config["multi_tenant"] + if "platform_tenant" in loki_config: + settings.LOKI_PLATFORM_TENANT = loki_config["platform_tenant"] + if "s3" in loki_config: + loki_s3 = loki_config["s3"] + if "bucket_name" in loki_s3: + settings.LOKI_S3_BUCKET_NAME = loki_s3["bucket_name"] + if "region" in loki_s3: + settings.LOKI_S3_REGION = loki_s3["region"] + if "access_key_id" in loki_s3: + resolved = _expand_env_ref(loki_s3["access_key_id"]) + if resolved: + settings.LOKI_S3_ACCESS_KEY_ID = resolved + if "secret_access_key" in loki_s3: + resolved = _expand_env_ref(loki_s3["secret_access_key"]) + if resolved: + settings.LOKI_S3_SECRET_ACCESS_KEY = resolved + if "prefix" in loki_s3: + settings.LOKI_S3_PREFIX = loki_s3["prefix"] + if "plivo" in config_data: + plivo_cfg = config_data["plivo"] + if plivo_cfg.get("auth_id"): + settings.PLIVO_AUTH_ID = plivo_cfg["auth_id"] + if plivo_cfg.get("auth_token"): + settings.PLIVO_AUTH_TOKEN = plivo_cfg["auth_token"] + if plivo_cfg.get("verify_app_uuid"): + settings.PLIVO_VERIFY_APP_UUID = plivo_cfg["verify_app_uuid"] + if plivo_cfg.get("webhook_base_url"): + settings.PLIVO_WEBHOOK_BASE_URL = plivo_cfg["webhook_base_url"] + if plivo_cfg.get("outbound_pool"): + settings.PLIVO_OUTBOUND_POOL = list(plivo_cfg["outbound_pool"]) + + if "exotel" in config_data: + exotel_cfg = config_data["exotel"] + if exotel_cfg.get("outbound_pool"): + settings.EXOTEL_OUTBOUND_POOL = list(exotel_cfg["outbound_pool"]) + + if "vobiz" in config_data: + vobiz_cfg = config_data["vobiz"] + if vobiz_cfg.get("auth_id"): + settings.VOBIZ_AUTH_ID = vobiz_cfg["auth_id"] + if vobiz_cfg.get("auth_token"): + settings.VOBIZ_AUTH_TOKEN = vobiz_cfg["auth_token"] + if vobiz_cfg.get("api_base"): + settings.VOBIZ_API_BASE = vobiz_cfg["api_base"] + if vobiz_cfg.get("media_base"): + settings.VOBIZ_MEDIA_BASE = vobiz_cfg["media_base"] + if vobiz_cfg.get("webhook_base_url"): + settings.VOBIZ_WEBHOOK_BASE_URL = vobiz_cfg["webhook_base_url"] + if "webhook_verify" in vobiz_cfg: + settings.VOBIZ_WEBHOOK_VERIFY = bool(vobiz_cfg["webhook_verify"]) + if "carrier_session_recording" in vobiz_cfg: + settings.VOBIZ_CARRIER_SESSION_RECORDING = bool(vobiz_cfg["carrier_session_recording"]) + if vobiz_cfg.get("media_ws_base_url"): + settings.MEDIA_WS_BASE_URL = vobiz_cfg["media_ws_base_url"] + if vobiz_cfg.get("from_number"): + settings.VOBIZ_FROM_NUMBER = vobiz_cfg["from_number"] + if vobiz_cfg.get("outbound_pool"): + settings.VOBIZ_OUTBOUND_POOL = list(vobiz_cfg["outbound_pool"]) + if vobiz_cfg.get("outbound_pool_max_concurrent_per_org") is not None: + settings.VOBIZ_OUTBOUND_POOL_MAX_CONCURRENT_PER_ORG = int( + vobiz_cfg["outbound_pool_max_concurrent_per_org"] + ) + if vobiz_cfg.get("default_country_code"): + settings.VOBIZ_DEFAULT_COUNTRY_CODE = str(vobiz_cfg["default_country_code"]).lstrip("+") + + if "telephony" in config_data: + telephony_cfg = config_data["telephony"] + if telephony_cfg.get("outbound_pool"): + settings.TELEPHONY_OUTBOUND_POOL = list(telephony_cfg["outbound_pool"]) + if telephony_cfg.get("outbound_pool_max_concurrent_per_org") is not None: + settings.TELEPHONY_OUTBOUND_POOL_MAX_CONCURRENT_PER_ORG = int( + telephony_cfg["outbound_pool_max_concurrent_per_org"] + ) + + if "judge_alignment" in config_data: + ja_cfg = config_data["judge_alignment"] + if "enabled" in ja_cfg: + settings.JUDGE_ALIGNMENT_ENABLED = bool(ja_cfg["enabled"]) + if "csv_max_rows" in ja_cfg: + settings.JUDGE_ALIGNMENT_CSV_MAX_ROWS = int(ja_cfg["csv_max_rows"]) + + def _apply_llm_gateway_settings(gateway_cfg: dict, *, gateway_type: str) -> None: + if "enabled" in gateway_cfg: + settings.LLM_GATEWAY_ENABLED = bool(gateway_cfg["enabled"]) + settings.LLM_GATEWAY_TYPE = gateway_type + if gateway_cfg.get("base_url"): + settings.LLM_GATEWAY_BASE_URL = gateway_cfg["base_url"] + if gateway_cfg.get("virtual_key"): + settings.LLM_GATEWAY_VIRTUAL_KEY = gateway_cfg["virtual_key"] + if gateway_cfg.get("master_key"): + settings.LLM_GATEWAY_MASTER_KEY = gateway_cfg["master_key"] + if "passthrough_provider_keys" in gateway_cfg: + settings.LLM_GATEWAY_PASSTHROUGH_PROVIDER_KEYS = bool( + gateway_cfg["passthrough_provider_keys"] + ) + if gateway_cfg.get("gateway_interface"): + interface = str(gateway_cfg["gateway_interface"]).strip().lower() + if interface in ("litellm_shim", "native_openai"): + settings.LLM_GATEWAY_INTERFACE = interface + + if "llm_gateway" in config_data: + llm_cfg = config_data["llm_gateway"] + gateway_type = (llm_cfg.get("type") or "bifrost").strip().lower() + if gateway_type not in ("bifrost", "litellm_proxy"): + gateway_type = "bifrost" + _apply_llm_gateway_settings(llm_cfg, gateway_type=gateway_type) + + if "operational" in config_data: + operational_config = config_data["operational"] + if "public" in operational_config: + settings.OPERATIONAL_PUBLIC = bool(operational_config["public"]) + if "trusted_ips" in operational_config: + settings.OPERATIONAL_TRUSTED_IPS = operational_config["trusted_ips"] + + if "flexprice" in config_data: + flexprice_config = config_data["flexprice"] + if "enabled" in flexprice_config: + settings.FLEXPRICE_ENABLED = bool(flexprice_config["enabled"]) + if flexprice_config.get("api_key"): + settings.FLEXPRICE_API_KEY = flexprice_config["api_key"] + if flexprice_config.get("api_host"): + settings.FLEXPRICE_API_HOST = flexprice_config["api_host"] + + # Update Celery URLs if they weren't explicitly set + if not settings.CELERY_BROKER_URL: + settings.CELERY_BROKER_URL = settings.REDIS_URL + if not settings.CELERY_RESULT_BACKEND: + settings.CELERY_RESULT_BACKEND = settings.REDIS_URL + + +# Initialize settings with error handling for problematic env vars +# If .env file has invalid format, we'll use defaults (YAML config will override anyway) +try: + settings = Settings() +except Exception as e: + # If there's an error loading from .env (e.g., invalid JSON in list fields), + # create settings with defaults. The YAML config loaded later will override these. + import warnings + import os + + warnings.warn( + f"Error loading .env file, using defaults. YAML config will override. Error: {str(e)[:100]}", + UserWarning, + stacklevel=2 + ) + + # Try to create settings without .env file by temporarily removing it + env_file = ".env" + if os.path.exists(env_file): + # Temporarily rename .env to avoid loading it + backup_file = f"{env_file}.backup" + try: + os.rename(env_file, backup_file) + settings = Settings() + os.rename(backup_file, env_file) + except Exception: + # If rename fails or Settings still fails, restore and use defaults + if os.path.exists(backup_file): + try: + os.rename(backup_file, env_file) + except Exception: + pass + # Create with explicit defaults - manually construct with default values + settings = Settings( + ALLOWED_AUDIO_FORMATS=["wav", "mp3", "flac", "m4a"], + CORS_ORIGINS=["http://localhost:3000", "http://localhost:8000"], + _env_file=None, # Don't load .env + ) + else: + # No .env file, create normally + settings = Settings() diff --git a/app/core/auth/api_key.py b/app/core/auth/api_key.py index 6ff49ca7..48fdc346 100644 --- a/app/core/auth/api_key.py +++ b/app/core/auth/api_key.py @@ -12,6 +12,7 @@ from sqlalchemy.orm import Session +from app.core.auth.org_access import ensure_organization_active from app.core.auth.principal import AuthMethod, Principal from app.core.auth.providers import AuthError, AuthProvider, RawCredential from app.models.database import APIKey @@ -37,6 +38,8 @@ def authenticate(self, cred: RawCredential, db: Session) -> Principal: if not db_key: raise AuthError("Invalid API key") + ensure_organization_active(db, db_key.organization_id) + db_key.last_used = datetime.now(timezone.utc) db.commit() diff --git a/app/core/auth/local.py b/app/core/auth/local.py index bfadc1bb..4408bc3c 100644 --- a/app/core/auth/local.py +++ b/app/core/auth/local.py @@ -17,6 +17,7 @@ from jose import JWTError from sqlalchemy.orm import Session +from app.core.auth.org_access import ensure_organization_active from app.core.auth.principal import AuthMethod, Principal from app.core.auth.providers import AuthError, AuthProvider, RawCredential from app.core.auth.tokens import ISSUER, decode_access_token @@ -77,6 +78,8 @@ def authenticate(self, cred: RawCredential, db: Session) -> Principal: if not member: raise AuthError("User is not a member of this organization") + ensure_organization_active(db, org_id) + return Principal( organization_id=org_id, auth_method=AuthMethod.LOCAL_PASSWORD, diff --git a/app/core/auth/oidc_common.py b/app/core/auth/oidc_common.py index 6c2542c7..6a70d2bb 100644 --- a/app/core/auth/oidc_common.py +++ b/app/core/auth/oidc_common.py @@ -20,6 +20,7 @@ from loguru import logger from sqlalchemy.orm import Session +from app.core.auth.org_access import ensure_organization_active from app.core.auth.principal import Principal from app.core.auth.providers import AuthError from app.models.database import Organization, OrganizationMember, User @@ -261,6 +262,8 @@ def principal_from_oidc_claims( last_name=last_name, ) + ensure_organization_active(db, organization.id) + return Principal( organization_id=organization.id, auth_method=auth_method, diff --git a/app/core/auth/org_access.py b/app/core/auth/org_access.py new file mode 100644 index 00000000..157732fc --- /dev/null +++ b/app/core/auth/org_access.py @@ -0,0 +1,20 @@ +"""Organization access guards shared across auth providers.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.core.auth.providers import AuthError +from app.models.database import Organization + + +def ensure_organization_active(db: Session, organization_id: UUID) -> Organization: + """Raise AuthError when the organization is missing or disabled.""" + org = db.query(Organization).filter(Organization.id == organization_id).first() + if org is None: + raise AuthError("Organization not found") + if not org.is_active: + raise AuthError("Organization disabled", status_code=403) + return org diff --git a/app/core/auth/platform_admin.py b/app/core/auth/platform_admin.py new file mode 100644 index 00000000..920c0164 --- /dev/null +++ b/app/core/auth/platform_admin.py @@ -0,0 +1,129 @@ +"""Platform admin JWT issuance and FastAPI dependencies.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional, Tuple +from uuid import UUID, uuid4 + +from fastapi import Depends, Header, HTTPException, status +from jose import JWTError, jwt +from sqlalchemy.orm import Session + +from app.config import settings +from app.database import get_db +from app.models.database import PlatformAdmin + +PLATFORM_ISSUER = "efficientai-platform" +PLATFORM_SCOPE = "platform_admin" +ALGORITHM = "HS256" + + +@dataclass(frozen=True) +class PlatformAdminPrincipal: + platform_admin_id: UUID + email: str + + +def create_platform_access_token( + *, + platform_admin_id: UUID, + email: str, + expires_in_minutes: Optional[int] = None, +) -> Tuple[str, int]: + ttl_minutes = expires_in_minutes or getattr(settings, "AUTH_LOCAL_TOKEN_TTL_MINUTES", 15) + ttl_seconds = ttl_minutes * 60 + now = datetime.now(timezone.utc) + payload: Dict[str, Any] = { + "iss": PLATFORM_ISSUER, + "sub": str(platform_admin_id), + "email": email, + "scope": PLATFORM_SCOPE, + "jti": str(uuid4()), + "iat": int(now.timestamp()), + "exp": int((now + timedelta(minutes=ttl_minutes)).timestamp()), + } + token = jwt.encode(payload, settings.SECRET_KEY, algorithm=ALGORITHM) + return token, ttl_seconds + + +def decode_platform_access_token(token: str) -> Dict[str, Any]: + return jwt.decode( + token, + settings.SECRET_KEY, + algorithms=[ALGORITHM], + issuer=PLATFORM_ISSUER, + options={"verify_aud": False}, + ) + + +def _extract_bearer(authorization: Optional[str]) -> Optional[str]: + if not authorization: + return None + scheme, _, token = authorization.partition(" ") + if scheme.lower() != "bearer" or not token.strip(): + return None + return token.strip() + + +def platform_admin_feature_enabled(db: Session) -> bool: + return ( + db.query(PlatformAdmin.id) + .filter(PlatformAdmin.is_active == True) # noqa: E712 + .first() + is not None + ) + + +def require_platform_admin_feature(db: Session = Depends(get_db)) -> None: + if not platform_admin_feature_enabled(db): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Not found") + + +def get_platform_admin( + authorization: Optional[str] = Header(None, alias="Authorization"), + db: Session = Depends(get_db), + _feature: None = Depends(require_platform_admin_feature), +) -> PlatformAdminPrincipal: + bearer = _extract_bearer(authorization) + if not bearer: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Authentication required (send Authorization: Bearer ...)", + ) + + try: + claims = decode_platform_access_token(bearer) + except JWTError as exc: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=f"Invalid platform admin token: {exc}", + ) from exc + + if claims.get("scope") != PLATFORM_SCOPE: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid platform admin token scope.", + ) + + try: + admin_id = UUID(claims["sub"]) + except (KeyError, ValueError) as exc: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Malformed platform admin token.", + ) from exc + + admin = ( + db.query(PlatformAdmin) + .filter(PlatformAdmin.id == admin_id, PlatformAdmin.is_active == True) # noqa: E712 + .first() + ) + if admin is None: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Platform admin no longer active.", + ) + + return PlatformAdminPrincipal(platform_admin_id=admin.id, email=admin.email) diff --git a/app/db_sharding/scatter_gather.py b/app/db_sharding/scatter_gather.py index 9bfe600b..15722006 100644 --- a/app/db_sharding/scatter_gather.py +++ b/app/db_sharding/scatter_gather.py @@ -779,6 +779,84 @@ def count_shard(db: Session, _shard_id: str) -> int: return total +def max_call_import_row_index( + catalog_db: Session, + call_import_id: UUID, +) -> int: + """Highest ``row_index`` for an import, or ``-1`` when no rows exist.""" + if not is_sharding_enabled(): + result = ( + catalog_db.query(func.max(CallImportRow.row_index)) + .filter(CallImportRow.call_import_id == call_import_id) + .scalar() + ) + return int(result) if result is not None else -1 + + max_index = -1 + + def max_on_shard(db: Session, _shard_id: str) -> int: + result = ( + db.query(func.max(CallImportRow.row_index)) + .filter(CallImportRow.call_import_id == call_import_id) + .scalar() + ) + return int(result) if result is not None else -1 + + for part in scatter_gather_on_shards( + shard_ids_for_import(catalog_db, call_import_id), + max_on_shard, + ): + max_index = max(max_index, part) + + if max_index < 0: + result = ( + catalog_db.query(func.max(CallImportRow.row_index)) + .filter(CallImportRow.call_import_id == call_import_id) + .scalar() + ) + return int(result) if result is not None else -1 + return max_index + + +def load_call_import_conversation_ids( + catalog_db: Session, + call_import_id: UUID, +) -> List[Optional[str]]: + """All conversation ids for an import (shard-aware).""" + if not is_sharding_enabled(): + return [ + row[0] + for row in catalog_db.query(CallImportRow.conversation_id) + .filter(CallImportRow.call_import_id == call_import_id) + .all() + ] + + conversation_ids: List[Optional[str]] = [] + + def load_shard(db: Session, _shard_id: str) -> List[Optional[str]]: + return [ + row[0] + for row in db.query(CallImportRow.conversation_id) + .filter(CallImportRow.call_import_id == call_import_id) + .all() + ] + + for part in scatter_gather_on_shards( + shard_ids_for_import(catalog_db, call_import_id), + load_shard, + ): + conversation_ids.extend(part) + + if not conversation_ids: + conversation_ids = [ + row[0] + for row in catalog_db.query(CallImportRow.conversation_id) + .filter(CallImportRow.call_import_id == call_import_id) + .all() + ] + return conversation_ids + + def _non_empty_production_transcript_filter(): """Rows whose CSV ``transcript`` column has non-whitespace content.""" return ( diff --git a/app/dependencies.py b/app/dependencies.py index 6277777a..0dea6a91 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -109,6 +109,14 @@ def _resolve_workspace_row( status_code=403, detail="You don't have access to this workspace.", ) + if not workspace.is_active and not is_org_admin: + raise HTTPException( + status_code=403, + detail=( + "This workspace is inactive. Contact an organization admin " + "to reactivate it." + ), + ) return workspace if is_org_admin or principal.user_id is None: diff --git a/app/migrations/057_metric_draft_lifecycle.py b/app/migrations/057_metric_draft_lifecycle.py new file mode 100644 index 00000000..ed5ad75d --- /dev/null +++ b/app/migrations/057_metric_draft_lifecycle.py @@ -0,0 +1,64 @@ +"""Migration: Add metric draft lifecycle columns.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add lifecycle, promoted_from_draft_at, studio_notes to metrics." + + +def _column_exists(db: Session, table_name: str, column_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table_name, "column_name": column_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _column_exists(db, "metrics", "lifecycle"): + db.execute( + text( + """ + ALTER TABLE metrics + ADD COLUMN lifecycle VARCHAR(20) NOT NULL DEFAULT 'active' + """ + ) + ) + print("Added metrics.lifecycle") + + if not _column_exists(db, "metrics", "promoted_from_draft_at"): + db.execute( + text( + """ + ALTER TABLE metrics + ADD COLUMN promoted_from_draft_at TIMESTAMPTZ NULL + """ + ) + ) + print("Added metrics.promoted_from_draft_at") + + if not _column_exists(db, "metrics", "studio_notes"): + db.execute( + text( + """ + ALTER TABLE metrics + ADD COLUMN studio_notes TEXT NULL + """ + ) + ) + print("Added metrics.studio_notes") + + +def downgrade(db: Session): + if _column_exists(db, "metrics", "studio_notes"): + db.execute(text("ALTER TABLE metrics DROP COLUMN studio_notes")) + if _column_exists(db, "metrics", "promoted_from_draft_at"): + db.execute(text("ALTER TABLE metrics DROP COLUMN promoted_from_draft_at")) + if _column_exists(db, "metrics", "lifecycle"): + db.execute(text("ALTER TABLE metrics DROP COLUMN lifecycle")) diff --git a/app/migrations/057_platform_admin.py b/app/migrations/057_platform_admin.py new file mode 100644 index 00000000..05028016 --- /dev/null +++ b/app/migrations/057_platform_admin.py @@ -0,0 +1,144 @@ +""" +Migration: platform admin tables, org disable flag, signup reference codes. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add platform admin auth, org is_active flag, and signup reference codes." + + +def _column_exists(db: Session, table_name: str, column_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table_name, "column_name": column_name}, + ).first() + return row is not None + + +def _table_exists(db: Session, table_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.tables + WHERE table_name = :table_name + """ + ), + {"table_name": table_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _column_exists(db, "organizations", "is_active"): + db.execute( + text( + """ + ALTER TABLE organizations + ADD COLUMN is_active BOOLEAN NOT NULL DEFAULT TRUE + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_organizations_is_active + ON organizations (is_active) + """ + ) + ) + print("Added organizations.is_active") + + if not _column_exists(db, "organizations", "disabled_at"): + db.execute( + text( + """ + ALTER TABLE organizations + ADD COLUMN disabled_at TIMESTAMP WITH TIME ZONE NULL + """ + ) + ) + print("Added organizations.disabled_at") + + if not _table_exists(db, "platform_admins"): + db.execute( + text( + """ + CREATE TABLE platform_admins ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + email VARCHAR(255) NOT NULL UNIQUE, + password_hash VARCHAR(255) NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + last_login_at TIMESTAMP WITH TIME ZONE NULL + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_platform_admins_email + ON platform_admins (email) + """ + ) + ) + print("Added platform_admins table") + + if not _table_exists(db, "signup_reference_codes"): + db.execute( + text( + """ + CREATE TABLE signup_reference_codes ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + code_hash VARCHAR(64) NOT NULL UNIQUE, + label VARCHAR(255) NULL, + max_uses INTEGER NULL, + use_count INTEGER NOT NULL DEFAULT 0, + expires_at TIMESTAMP WITH TIME ZONE NULL, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + created_by UUID NULL REFERENCES platform_admins(id) ON DELETE SET NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_signup_reference_codes_is_active + ON signup_reference_codes (is_active) + """ + ) + ) + print("Added signup_reference_codes table") + + db.commit() + + +def downgrade(db: Session): + if _table_exists(db, "signup_reference_codes"): + db.execute(text("DROP TABLE signup_reference_codes")) + print("Dropped signup_reference_codes table") + + if _table_exists(db, "platform_admins"): + db.execute(text("DROP TABLE platform_admins")) + print("Dropped platform_admins table") + + if _column_exists(db, "organizations", "disabled_at"): + db.execute(text("ALTER TABLE organizations DROP COLUMN disabled_at")) + print("Dropped organizations.disabled_at") + + if _column_exists(db, "organizations", "is_active"): + db.execute(text("DROP INDEX IF EXISTS ix_organizations_is_active")) + db.execute(text("ALTER TABLE organizations DROP COLUMN is_active")) + print("Dropped organizations.is_active") + + db.commit() diff --git a/app/migrations/057_workspace_is_active.py b/app/migrations/057_workspace_is_active.py new file mode 100644 index 00000000..5055e6e5 --- /dev/null +++ b/app/migrations/057_workspace_is_active.py @@ -0,0 +1,44 @@ +""" +Migration: Add is_active flag on workspaces for org-admin deactivation. + +Inactive workspaces are fully locked for non-org-admin callers; org admins +retain access for inspection and reactivation. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add workspaces.is_active (default true) for org-admin deactivation." + + +def _column_exists(db: Session, table_name: str, column_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table_name, "column_name": column_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _column_exists(db, "workspaces", "is_active"): + db.execute( + text( + """ + ALTER TABLE workspaces + ADD COLUMN is_active BOOLEAN NOT NULL DEFAULT TRUE + """ + ) + ) + print("Added workspaces.is_active (boolean, default true)") + + +def downgrade(db: Session): + if _column_exists(db, "workspaces", "is_active"): + db.execute(text("ALTER TABLE workspaces DROP COLUMN is_active")) + print("Dropped workspaces.is_active") diff --git a/app/migrations/058_metric_studio_runs.py b/app/migrations/058_metric_studio_runs.py new file mode 100644 index 00000000..02792d33 --- /dev/null +++ b/app/migrations/058_metric_studio_runs.py @@ -0,0 +1,113 @@ +"""Migration: Add Metrics Studio run tables.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add metric_studio_runs and metric_studio_run_results tables." + + +def _table_exists(db: Session, table_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.tables + WHERE table_name = :table_name + """ + ), + {"table_name": table_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _table_exists(db, "metric_studio_runs"): + db.execute( + text( + """ + CREATE TABLE metric_studio_runs ( + id UUID PRIMARY KEY, + organization_id UUID NOT NULL REFERENCES organizations(id), + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE RESTRICT, + created_by_user_id UUID NULL REFERENCES users(id), + name VARCHAR(255) NULL, + selected_metric_ids JSON NOT NULL DEFAULT '[]', + selected_metric_groups JSON NULL, + transcript_source VARCHAR(20) NOT NULL DEFAULT 'diarised', + llm_provider VARCHAR(50) NULL, + llm_model VARCHAR(100) NULL, + llm_credential_id UUID NULL REFERENCES aiproviders(id) ON DELETE SET NULL, + llm_config JSON NULL, + metric_llm_overrides JSON NULL, + status VARCHAR(20) NOT NULL DEFAULT 'pending', + total_items INTEGER NOT NULL DEFAULT 0, + completed_items INTEGER NOT NULL DEFAULT 0, + failed_items INTEGER NOT NULL DEFAULT 0, + error_message TEXT NULL, + started_at TIMESTAMPTZ NULL, + finished_at TIMESTAMPTZ NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + """ + ) + ) + db.execute( + text( + "CREATE INDEX ix_metric_studio_runs_org ON metric_studio_runs (organization_id)" + ) + ) + db.execute( + text( + "CREATE INDEX ix_metric_studio_runs_workspace ON metric_studio_runs (workspace_id)" + ) + ) + db.execute( + text( + "CREATE INDEX ix_metric_studio_runs_status ON metric_studio_runs (status)" + ) + ) + print("Created metric_studio_runs") + + if not _table_exists(db, "metric_studio_run_results"): + db.execute( + text( + """ + CREATE TABLE metric_studio_run_results ( + id UUID PRIMARY KEY, + run_id UUID NOT NULL REFERENCES metric_studio_runs(id) ON DELETE CASCADE, + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE RESTRICT, + source_kind VARCHAR(40) NOT NULL, + source_ref VARCHAR(255) NOT NULL, + display_label VARCHAR(512) NULL, + source_metadata JSON NULL, + status VARCHAR(20) NOT NULL DEFAULT 'pending', + metric_scores JSON NOT NULL DEFAULT '{}', + error_message TEXT NULL, + celery_task_id VARCHAR(255) NULL, + started_at TIMESTAMPTZ NULL, + finished_at TIMESTAMPTZ NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + """ + ) + ) + db.execute( + text( + "CREATE INDEX ix_metric_studio_run_results_run ON metric_studio_run_results (run_id)" + ) + ) + db.execute( + text( + "CREATE INDEX ix_metric_studio_run_results_status ON metric_studio_run_results (status)" + ) + ) + print("Created metric_studio_run_results") + + +def downgrade(db: Session): + if _table_exists(db, "metric_studio_run_results"): + db.execute(text("DROP TABLE metric_studio_run_results")) + if _table_exists(db, "metric_studio_runs"): + db.execute(text("DROP TABLE metric_studio_runs")) diff --git a/app/migrations/059_call_import_audit_users.py b/app/migrations/059_call_import_audit_users.py index 41ec06db..ed45e73c 100644 --- a/app/migrations/059_call_import_audit_users.py +++ b/app/migrations/059_call_import_audit_users.py @@ -1,69 +1,69 @@ -""" -Migration: last_updated_by_user_id on call imports and evaluations. - -Supports surfacing who created / last modified a batch or evaluation run -via FK to users (email resolved at read time). -""" - -from sqlalchemy import text -from sqlalchemy.orm import Session - -description = ( - "Add last_updated_by_user_id to call_imports and call_import_evaluations" -) - - -def _column_exists(db: Session, table: str, column: str) -> bool: - row = db.execute( - text( - """ - SELECT 1 - FROM information_schema.columns - WHERE table_name = :table_name AND column_name = :column_name - """ - ), - {"table_name": table, "column_name": column}, - ).first() - return row is not None - - -def _table_exists(db: Session, table: str) -> bool: - row = db.execute( - text( - "SELECT 1 FROM information_schema.tables WHERE table_name = :t" - ), - {"t": table}, - ).first() - return row is not None - - -def upgrade(db: Session): - for table in ("call_imports", "call_import_evaluations"): - if not _table_exists(db, table): - print(f"{table} does not exist, skipping...") - continue - if _column_exists(db, table, "last_updated_by_user_id"): - print(f"{table}.last_updated_by_user_id already exists, skipping...") - continue - db.execute( - text( - f""" - ALTER TABLE {table} - ADD COLUMN last_updated_by_user_id UUID NULL - REFERENCES users(id) ON DELETE SET NULL - """ - ) - ) - print(f"Added {table}.last_updated_by_user_id") - - -def downgrade(db: Session): - for table in ("call_import_evaluations", "call_imports"): - if not _table_exists(db, table): - continue - if not _column_exists(db, table, "last_updated_by_user_id"): - continue - db.execute( - text(f"ALTER TABLE {table} DROP COLUMN last_updated_by_user_id") - ) - print(f"Dropped {table}.last_updated_by_user_id") +""" +Migration: last_updated_by_user_id on call imports and evaluations. + +Supports surfacing who created / last modified a batch or evaluation run +via FK to users (email resolved at read time). +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Add last_updated_by_user_id to call_imports and call_import_evaluations" +) + + +def _column_exists(db: Session, table: str, column: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table, "column_name": column}, + ).first() + return row is not None + + +def _table_exists(db: Session, table: str) -> bool: + row = db.execute( + text( + "SELECT 1 FROM information_schema.tables WHERE table_name = :t" + ), + {"t": table}, + ).first() + return row is not None + + +def upgrade(db: Session): + for table in ("call_imports", "call_import_evaluations"): + if not _table_exists(db, table): + print(f"{table} does not exist, skipping...") + continue + if _column_exists(db, table, "last_updated_by_user_id"): + print(f"{table}.last_updated_by_user_id already exists, skipping...") + continue + db.execute( + text( + f""" + ALTER TABLE {table} + ADD COLUMN last_updated_by_user_id UUID NULL + REFERENCES users(id) ON DELETE SET NULL + """ + ) + ) + print(f"Added {table}.last_updated_by_user_id") + + +def downgrade(db: Session): + for table in ("call_import_evaluations", "call_imports"): + if not _table_exists(db, table): + continue + if not _column_exists(db, table, "last_updated_by_user_id"): + continue + db.execute( + text(f"ALTER TABLE {table} DROP COLUMN last_updated_by_user_id") + ) + print(f"Dropped {table}.last_updated_by_user_id") diff --git a/app/models/database.py b/app/models/database.py index 68bcfebc..2db26e68 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -1,2779 +1,2937 @@ -"""SQLAlchemy database models.""" - -from sqlalchemy import ( - BigInteger, - Boolean, - Column, - Date, - DateTime, - DDL, - Enum, - event, - Float, - ForeignKey, - Integer, - JSON, - String, - Text, - UniqueConstraint, - select, - text, -) -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import relationship -from sqlalchemy.sql import func -import uuid -import enum -from app.models.enums import ( - EvaluationType, EvaluationStatus, EvaluatorResultStatus, RoleEnum, InvitationStatus, - LanguageEnum, CallTypeEnum, CallMediumEnum, GenderEnum, AccentEnum, BackgroundNoiseEnum, - IntegrationPlatform, ModelProvider, VoiceBundleType, TestAgentConversationStatus, - MetricType, MetricCategory, MetricTrigger, CallRecordingStatus, AlertMetricType, AlertAggregation, - AlertOperator, AlertNotifyFrequency, AlertStatus, AlertHistoryStatus, CronJobStatus, - PromptOptimizationStatus, CallImportStatus, CallImportRowStatus, -) - -def get_enum_values(enum_class): - """Helper to get values from enum class for SQLAlchemy.""" - return [e.value for e in enum_class] - -from app.database import Base - - -# Enums moved to enums.py - - -class Organization(Base): - """Organization model for multi-tenancy.""" - - __tablename__ = "organizations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - name = Column(String(255), nullable=False) - voice_playground_threshold_overrides = Column(JSON, nullable=True) - # AlignEval-style judge alignment thresholds. - # Shape: {"min_labels_to_evaluate": int, "min_labels_to_optimize": int} - # Falls back to system defaults (20 / 50) when null. - judge_alignment_settings = Column(JSON, nullable=True) - # Per-org LLM gateway overrides (enabled, gateway_type, base_url, keys). - llm_gateway_settings = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - # Relationships - api_keys = relationship("APIKey", back_populates="organization") - members = relationship("OrganizationMember", back_populates="organization", cascade="all, delete-orphan") - invitations = relationship("Invitation", back_populates="organization", cascade="all, delete-orphan") - workspaces = relationship( - "Workspace", - back_populates="organization", - cascade="all, delete-orphan", - ) - workspace_roles = relationship( - "WorkspaceRole", - back_populates="organization", - cascade="all, delete-orphan", - ) - - -class Workspace(Base): - """Workspace - in-org isolation boundary for call imports and metrics. - - Every organization has at least one workspace (``is_default = True``, - seeded by migration 033). Users pick an "active workspace" in the UI; - list endpoints filter by it so users only see calls/metrics from the - project they're currently working in. Access is governed by - ``workspace_members`` and org-scoped ``workspace_roles`` (capability - bundles); org admins implicitly access all workspaces. - """ - - __tablename__ = "workspaces" - __table_args__ = ( - UniqueConstraint("organization_id", "slug", name="uq_workspaces_org_slug"), - ) - - # ``server_default`` is required so that raw-SQL INSERTs (e.g. the - # per-org Default seed in migration 033) can omit ``id`` and let the - # database fill it in. Without it, ``create_all`` produces a column - # with NOT NULL but no DEFAULT, and the migration crashes with - # ``null value in column "id"``. - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid4, - server_default=text("gen_random_uuid()"), - ) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - slug = Column(String(255), nullable=False) - # At most one default per org. Enforced on Postgres by the partial - # unique index attached via the after_create event below; on - # SQLite (test runs) we rely on the route-level _check_slug_unique - # check + the Default-workspace conftest fixture instead, because - # SQLite doesn't support partial indexes the same way. - is_default = Column(Boolean, nullable=False, default=False, server_default="false") - # Reusable PDF/report branding metadata scoped to this workspace. Images - # live in S3. Shape: {"heading": str|null, "images": [{id, s3_key, - # content_type, filename, size_bytes, updated_at}, ...]}. - report_branding = Column(JSON, nullable=True) - created_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True - ) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - organization = relationship("Organization", back_populates="workspaces") - members = relationship( - "WorkspaceMember", - back_populates="workspace", - cascade="all, delete-orphan", - ) - - -class WorkspaceRole(Base): - """Org-scoped workspace role (system or custom) as a capability bundle.""" - - __tablename__ = "workspace_roles" - __table_args__ = ( - UniqueConstraint("organization_id", "name", name="uq_workspace_roles_org_name"), - ) - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid4, - server_default=text("gen_random_uuid()"), - ) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - capabilities = Column(JSON, nullable=False, default=list) - is_system = Column(Boolean, nullable=False, default=False, server_default="false") - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - organization = relationship("Organization", back_populates="workspace_roles") - members = relationship("WorkspaceMember", back_populates="role") - - -class WorkspaceMember(Base): - """User membership in a workspace with an assigned workspace role.""" - - __tablename__ = "workspace_members" - __table_args__ = ( - UniqueConstraint("workspace_id", "user_id", name="uq_workspace_members_ws_user"), - ) - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid4, - server_default=text("gen_random_uuid()"), - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - user_id = Column( - UUID(as_uuid=True), - ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - role_id = Column( - UUID(as_uuid=True), - ForeignKey("workspace_roles.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - added_by_user_id = Column( - UUID(as_uuid=True), - ForeignKey("users.id", ondelete="SET NULL"), - nullable=True, - ) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - workspace = relationship("Workspace", back_populates="members") - user = relationship("User", foreign_keys=[user_id]) - role = relationship("WorkspaceRole", back_populates="members") - added_by = relationship("User", foreign_keys=[added_by_user_id]) - - -# Partial unique index: "at most one default workspace per org". This -# is attached as an after_create event (rather than declared in -# ``__table_args__``) because SQLAlchemy's ``Index(..., -# postgresql_where=...)`` silently degrades to a *full* unique index on -# SQLite - which then forbids any second workspace per org and breaks -# the test suite. ``execute_if(dialect="postgresql")`` makes this DDL -# a no-op on SQLite while still emitting it on Postgres (prod, CI). -event.listen( - Workspace.__table__, - "after_create", - DDL( - "CREATE UNIQUE INDEX IF NOT EXISTS uq_workspaces_org_default " - "ON workspaces (organization_id) WHERE is_default" - ).execute_if(dialect="postgresql"), -) - - -class User(Base): - """User model for authentication and profile management.""" - - __tablename__ = "users" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - email = Column(String(255), unique=True, nullable=False, index=True) - name = Column(String(255), nullable=True) - first_name = Column(String(255), nullable=True) - last_name = Column(String(255), nullable=True) - password_hash = Column(String(255), nullable=True) # Nullable for users created via invitation - external_id = Column(String(255), unique=True, nullable=True, index=True) - auth_provider = Column(String(50), nullable=True) - mfa_enabled = Column(Boolean, default=False, nullable=False) - last_login_at = Column(DateTime(timezone=True), nullable=True) - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - # Relationships - organization_memberships = relationship("OrganizationMember", back_populates="user", cascade="all, delete-orphan") - api_keys = relationship("APIKey", back_populates="user") - invitations = relationship("Invitation", back_populates="invited_user", foreign_keys="Invitation.invited_user_id") - refresh_tokens = relationship("RefreshToken", back_populates="user", cascade="all, delete-orphan") - - -class RefreshToken(Base): - """Opaque refresh token for extending local-password sessions.""" - - __tablename__ = "refresh_tokens" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True) - token_hash = Column(String(64), unique=True, nullable=False, index=True) - expires_at = Column(DateTime(timezone=True), nullable=False) - revoked_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - user = relationship("User", back_populates="refresh_tokens") - - -class OrganizationMember(Base): - """Organization membership with role.""" - - __tablename__ = "organization_members" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False, index=True) - role = Column(String, nullable=False, default=RoleEnum.READER.value) - - # User preferences for this organization - default_agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) - - joined_at = Column(DateTime(timezone=True), server_default=func.now()) - - # Unique constraint: one membership per user per organization - __table_args__ = ( - UniqueConstraint('organization_id', 'user_id', name='uq_org_user'), - ) - - # Relationships - organization = relationship("Organization", back_populates="members") - user = relationship("User", back_populates="organization_memberships") - default_agent = relationship("Agent", foreign_keys=[default_agent_id]) - - -class Invitation(Base): - """Invitation model for inviting users to organizations.""" - - __tablename__ = "invitations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - invited_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Null if user doesn't exist yet - invited_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False) - email = Column(String(255), nullable=False) # Email of invited user - role = Column(String, nullable=False, default=RoleEnum.READER.value) - status = Column(String, nullable=False, default=InvitationStatus.PENDING.value) - - - - token = Column(String(255), unique=True, nullable=False, index=True) # Invitation token - expires_at = Column(DateTime(timezone=True), nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - accepted_at = Column(DateTime(timezone=True), nullable=True) - - # Relationships - organization = relationship("Organization", back_populates="invitations") - invited_user = relationship("User", foreign_keys=[invited_user_id], back_populates="invitations") - invited_by = relationship("User", foreign_keys=[invited_by_id]) - - -class APIKey(Base): - """API Key model for authentication.""" - - __tablename__ = "api_keys" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - key = Column(String(255), unique=True, nullable=False, index=True) - name = Column(String(255), nullable=True) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Optional: link to user - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - last_used = Column(DateTime(timezone=True), nullable=True) - - # Relationships - organization = relationship("Organization", back_populates="api_keys") - user = relationship("User", back_populates="api_keys") - - -class AudioFile(Base): - """Audio file model.""" - - __tablename__ = "audio_files" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - filename = Column(String(255), nullable=False) - file_path = Column(String(512), nullable=False) - file_size = Column(Integer, nullable=False) # Size in bytes - duration = Column(Float, nullable=True) # Duration in seconds - sample_rate = Column(Integer, nullable=True) - channels = Column(Integer, nullable=True) - format = Column(String(10), nullable=False) # wav, mp3, flac, etc. - uploaded_at = Column(DateTime(timezone=True), server_default=func.now()) - - # Relationships - evaluations = relationship("Evaluation", back_populates="audio_file") - - -class Evaluation(Base): - """Evaluation job model.""" - - __tablename__ = "evaluations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every legacy audio evaluation belongs to a - # workspace within its org. Stamped from the X-Workspace-Id header - # (falling back to the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - audio_id = Column(UUID(as_uuid=True), ForeignKey("audio_files.id"), nullable=False) - reference_text = Column(String, nullable=True) # For WER calculation - evaluation_type = Column(String, nullable=False) - model_name = Column(String(100), nullable=True) - status = Column(String, default=EvaluationStatus.PENDING.value, nullable=False) - - - - metrics_requested = Column(JSON, nullable=True) # List of requested metrics - created_at = Column(DateTime(timezone=True), server_default=func.now()) - started_at = Column(DateTime(timezone=True), nullable=True) - completed_at = Column(DateTime(timezone=True), nullable=True) - error_message = Column(String, nullable=True) - - # Relationships - audio_file = relationship("AudioFile", back_populates="evaluations") - result = relationship("EvaluationResult", back_populates="evaluation", uselist=False) - - -class EvaluationResult(Base): - """Evaluation result model.""" - - __tablename__ = "evaluation_results" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column(UUID(as_uuid=True), ForeignKey("evaluations.id"), nullable=False, unique=True) - # Workspace isolation: mirrors the parent Evaluation's workspace. - # Denormalized for fast filter-by-workspace listings without a join. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - transcript = Column(String, nullable=True) - metrics = Column(JSON, nullable=False) # {"wer": 0.05, "latency_ms": 1250, ...} - raw_output = Column(JSON, nullable=True) # Full model output - processing_time = Column(Float, nullable=True) # Processing time in seconds - model_used = Column(String(100), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - # Relationships - evaluation = relationship("Evaluation", back_populates="result") - - -# ============================================ -# VAIOPS MODELS - Voice AI Ops -# ============================================ - -# Enums moved to enums.py - - -class Agent(Base): - """Test Agent - The voice AI agent being evaluated""" - __tablename__ = "agents" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - agent_id = Column(String(6), unique=True, nullable=True, index=True) # 6-digit ID - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every agent belongs to a workspace within its - # org. Stamped from the X-Workspace-Id header (falling back to the - # org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - name = Column(String, nullable=False) - phone_number = Column(String, nullable=True) # Optional, required only for phone_call - language = Column(String, nullable=False, default=LanguageEnum.ENGLISH.value) - description = Column(String) - provider_prompt = Column(Text, nullable=True) - provider_prompt_synced_at = Column(DateTime(timezone=True), nullable=True) - call_type = Column(String, nullable=False, default=CallTypeEnum.OUTBOUND.value) - call_medium = Column(String, nullable=False, default=CallMediumEnum.PHONE_CALL.value) - telephony_phone_number_id = Column( - UUID(as_uuid=True), - ForeignKey("telephony_phone_numbers.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - - - - - # Voice configuration - either voice_bundle_id OR ai_provider_id OR voice_ai_integration_id (mutually exclusive) - voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True, index=True) - ai_provider_id = Column(UUID(as_uuid=True), ForeignKey("aiproviders.id"), nullable=True, index=True) - - # Voice AI agent integration (Retell, Vapi, etc.) - voice_ai_integration_id = Column(UUID(as_uuid=True), ForeignKey("integrations.id"), nullable=True, index=True) - voice_ai_agent_id = Column(String, nullable=True) # Agent ID from the external provider (Retell/Vapi) - prompt_variables = Column(JSON, nullable=True) - silence_hangup_secs = Column(Integer, nullable=False, server_default="15") - - created_at = Column(DateTime, server_default=func.now()) - updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) - created_by = Column(String) - - -class Persona(Base): - """Persona - TTS provider-tied voice identity for testing""" - __tablename__ = "personas" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every persona belongs to a workspace within - # its org. Stamped from the X-Workspace-Id header (falling back to - # the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - name = Column(String, nullable=False) - gender = Column(String, nullable=False, default=GenderEnum.NEUTRAL.value) - tts_provider = Column(String(100), nullable=True) - tts_voice_id = Column(String(255), nullable=True) - tts_voice_name = Column(String(255), nullable=True) - is_custom = Column(Boolean, default=False) - description = Column(Text, nullable=True) - tts_config = Column(JSON, nullable=True) - llm_temperature = Column(Float, nullable=True) - llm_max_tokens = Column(Integer, nullable=True) - response_delay_ms = Column(Integer, nullable=True) - max_turns = Column(Integer, nullable=True) - allow_interruptions = Column(Boolean, nullable=True) - - created_at = Column(DateTime, server_default=func.now()) - updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) - created_by = Column(String) - - -class Scenario(Base): - """Scenario - The conversation scenario/test case""" - __tablename__ = "scenarios" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every scenario belongs to a workspace within - # its org. Stamped from the X-Workspace-Id header (falling back to - # the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) - name = Column(String, nullable=False) - description = Column(String) - required_info = Column(JSON) - - created_at = Column(DateTime, server_default=func.now()) - updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) - created_by = Column(String) - - -# Enums moved to enums.py - - -class Integration(Base): - """Integration model for connecting with external voice AI platforms.""" - __tablename__ = "integrations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - platform = Column(String, nullable=False) - - - - name = Column(String, nullable=True) # Optional friendly name - api_key = Column(String, nullable=False) # Encrypted Private API key for the platform - public_key = Column(String, nullable=True) # Optional Public API key (e.g. for Vapi) - is_active = Column(Boolean, default=True, nullable=False) - # Multiple credentials per (org, platform) are allowed. is_default marks - # the row used when a caller does not explicitly select a credential. - # A partial unique index in migration 028 enforces at most one default - # per (org, platform) at the DB level. - is_default = Column(Boolean, default=False, nullable=False) - # inherit | gateway | direct — per-credential LLM routing override - routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated - - -class ManualTranscription(Base): - """Manual transcription model for storing transcriptions from S3 audio files.""" - - __tablename__ = "manual_transcriptions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - name = Column(String(255), nullable=True) # User-friendly name for the transcription - audio_file_key = Column(String(512), nullable=False) # S3 key or file path - transcript = Column(String, nullable=False) # Full transcript text - speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] - stt_model = Column(String(100), nullable=True) # STT model used (e.g., "whisper-1", "google-speech-v2") - stt_provider = Column(String, nullable=True) # Provider used - - - - language = Column(String(10), nullable=True) # Detected or specified language - processing_time = Column(Float, nullable=True) # Processing time in seconds - raw_output = Column(JSON, nullable=True) # Full model output for reference - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class ConversationEvaluation(Base): - """Conversation evaluation model for evaluating manual transcriptions against agent objectives.""" - - __tablename__ = "conversation_evaluations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - transcription_id = Column(UUID(as_uuid=True), ForeignKey("manual_transcriptions.id"), nullable=False, index=True) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) - - # Evaluation results - objective_achieved = Column(Boolean, nullable=False) # Binary: was the conversation objective achieved? - objective_achieved_reason = Column(String, nullable=True) # Explanation for the binary result - additional_metrics = Column(JSON, nullable=True) # Additional evaluation metrics (e.g., professionalism, clarity, etc.) - overall_score = Column(Float, nullable=True) # Overall score (0.0 to 1.0) - - # LLM metadata - llm_provider = Column(Enum(ModelProvider, native_enum=False), nullable=True) - - llm_model = Column(String(100), nullable=True) - llm_response = Column(JSON, nullable=True) # Full LLM response for reference - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class AIProvider(Base): - """AI Provider - Stores API keys for different AI platforms.""" - __tablename__ = "aiproviders" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider = Column(String, nullable=False) - - - - api_key = Column(String, nullable=False) # Encrypted API key - name = Column(String, nullable=True) # Optional friendly name - # Azure OpenAI resource endpoint (e.g. https://my-resource.openai.azure.com). - # Only used when provider is azure; other providers ignore this column. - endpoint_url = Column(String, nullable=True) - is_active = Column(Boolean, default=True, nullable=False) - # Multiple AIProvider rows per (org, provider) are allowed. is_default - # marks the row resolved when no explicit credential id is selected. - # A partial unique index in migration 028 enforces at most one default. - is_default = Column(Boolean, default=False, nullable=False) - # inherit | gateway | direct — per-credential LLM routing override - routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") - # Bifrost custom model ID used when routing via gateway - gateway_model = Column(String(255), nullable=True) - # inherit | litellm_shim | native_openai — Bifrost API surface override - gateway_interface = Column(String(20), nullable=False, default="inherit", server_default="inherit") - # Optional per-credential Bifrost/gateway base URL override - gateway_base_url = Column(String(512), nullable=True) - # Optional auth header for Bifrost (e.g. x-bf-vk, Authorization, x-api-key) - gateway_auth_header = Column(String(64), nullable=True) - # Env var name whose value is sent as the gateway auth secret - gateway_auth_secret_env = Column(String(128), nullable=True) - # Encrypted inline gateway auth secret (alternative to env var) - gateway_auth_secret = Column(String, nullable=True) - # Arbitrary HTTP headers sent with gateway-routed LiteLLM calls - gateway_extra_headers = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated - - -# Enums moved to enums.py - - -class VoiceBundle(Base): - """VoiceBundle - Composable unit combining STT, LLM, and TTS for voice AI testing, or S2S models.""" - __tablename__ = "voicebundles" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - name = Column(String, nullable=False) - description = Column(String, nullable=True) - - # Bundle type: either STT+LLM+TTS or S2S - # Using String instead of Enum to avoid SQLAlchemy enum conversion issues - # The enum conversion is handled in the Pydantic schemas - bundle_type = Column(String(50), nullable=False, default=VoiceBundleType.STT_LLM_TTS.value) - - # STT Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S - stt_provider = Column(String, nullable=True) - # Optional explicit credential row (aiproviders.id or integrations.id). - # When NULL the credential resolver picks the default row for the - # provider. No FK is set because the target table varies by provider. - stt_credential_id = Column(UUID(as_uuid=True), nullable=True) - - stt_model = Column(String, nullable=True) # e.g., "whisper-1", "google-speech-v2" - - # LLM Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S - llm_provider = Column(String, nullable=True) - llm_credential_id = Column(UUID(as_uuid=True), nullable=True) - - llm_model = Column(String, nullable=True) # e.g., "gpt-4", "claude-3-opus" - llm_temperature = Column(Float, nullable=True, default=0.7) - llm_max_tokens = Column(Integer, nullable=True) - llm_config = Column(JSON, nullable=True) # Additional LLM configuration (extensible) - - # TTS Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S - tts_provider = Column(String, nullable=True) - tts_credential_id = Column(UUID(as_uuid=True), nullable=True) - - tts_model = Column(String, nullable=True) # e.g., "tts-1", "neural-voice" - tts_voice = Column(String, nullable=True) # Voice selection if applicable - tts_config = Column(JSON, nullable=True) # Additional TTS configuration (extensible) - - # S2S Configuration - required for S2S type, optional for STT_LLM_TTS - s2s_provider = Column(String, nullable=True) - s2s_credential_id = Column(UUID(as_uuid=True), nullable=True) - - - - s2s_model = Column(String, nullable=True) # e.g., "gpt-4o-transcribe", speech-to-speech model - s2s_config = Column(JSON, nullable=True) # Additional S2S configuration (extensible) - - # Additional configuration for extensibility - extra_metadata = Column(JSON, nullable=True) # For future extensions (renamed from 'metadata' to avoid SQLAlchemy conflict) - - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -# Enums moved to enums.py - - -class TestAgentConversation(Base): - """Test Agent Conversation - Records conversations between test AI agent and voice AI agent.""" - __tablename__ = "test_agent_conversations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every playground conversation belongs to a - # workspace within its org. Stamped from the X-Workspace-Id header - # (falling back to the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Configuration - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) - scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=False) - voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) - - # Conversation data - status = Column(String, nullable=False, default=TestAgentConversationStatus.INITIALIZING.value) - - - - live_transcription = Column(JSON, nullable=True) # Array of conversation turns with timestamps - conversation_audio_key = Column(String, nullable=True) # S3 key for recorded conversation audio - full_transcript = Column(String, nullable=True) # Full conversation transcript - - # Metadata - started_at = Column(DateTime(timezone=True), server_default=func.now()) - ended_at = Column(DateTime(timezone=True), nullable=True) - duration_seconds = Column(Float, nullable=True) - - # Additional metadata - conversation_metadata = Column(JSON, nullable=True) # Additional conversation metadata - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class EvaluatorSuite(Base): - """Evaluator suite — one agent + one persona + N scenario combinations.""" - - __tablename__ = "evaluator_suites" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - name = Column(String, nullable=True) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) - metric_ids = Column(JSON, nullable=True) - llm_provider = Column(String, nullable=True) - llm_model = Column(String, nullable=True) - llm_config = Column(JSON, nullable=True) - tags = Column(JSON, nullable=True) - default_runs_per_combination = Column(Integer, nullable=False, default=1) - round_robin_index = Column(Integer, nullable=False, default=0) - is_active = Column(Boolean, nullable=False, default=False) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class Evaluator(Base): - """Evaluator - Configuration for testing agents with specific persona and scenario combinations, or custom prompt evaluators.""" - __tablename__ = "evaluators" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluator_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every evaluator belongs to a workspace within - # its org. Stamped from the X-Workspace-Id header (falling back to - # the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Display name (required for custom evaluators, optional for standard) - name = Column(String, nullable=True) - - # Parent suite (nullable for legacy/custom evaluators) - suite_id = Column( - UUID(as_uuid=True), - ForeignKey("evaluator_suites.id", ondelete="CASCADE"), - nullable=True, - index=True, - ) - - # Standard evaluator configuration (nullable for custom evaluators) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) - scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) - - # Custom evaluator prompt (used instead of agent/persona/scenario) - custom_prompt = Column(Text, nullable=True) - - # Custom evaluator metric selection. When set, the worker filters the - # enabled-org metrics down to only these IDs (list of metric UUID strings). - # Standard evaluators leave this NULL and use all enabled agent metrics. - metric_ids = Column(JSON, nullable=True) - - # LLM configuration for evaluation (overrides hardcoded defaults) - llm_provider = Column(String, nullable=True) # e.g. "openai", "anthropic", "google" - llm_model = Column(String, nullable=True) # e.g. "gpt-4.1", "claude-sonnet-4-20250514" - llm_config = Column(JSON, nullable=True) - - # Tags for categorization - tags = Column(JSON, nullable=True) # Array of tag strings - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -# Enums moved to enums.py - - -class Metric(Base): - """Metric - Configuration for evaluation metrics. - - Supports a 2-level hierarchy via ``parent_metric_id``: a "category" - parent metric (e.g. "Call Outcome") owns N child sub-metric labels - (e.g. "happy_completion", "angry_hangup"). ``selection_mode`` is set - only on parents and controls how the LLM scores children together - (``single_choice`` = pick exactly one; ``multi_label`` = independent - yes/no with logical consistency). Children are always boolean. - """ - __tablename__ = "metrics" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: two-shape column. - # - # * ``workspace_id = `` — workspace-scoped metric. Only - # visible inside that workspace (the default behavior; existing - # rows all look like this). - # * ``workspace_id IS NULL`` — org-shared metric. Surfaces in - # every workspace's listing under this org so users don't have - # to recreate the same metric per workspace. - # - # Children always inherit their parent's ``workspace_id`` (including - # NULL) so a category metric's whole subtree shares one scope; the - # add-child / promote-discovered endpoints enforce this. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=True, - index=True, - ) - - # Basic information - name = Column(String, nullable=False) - description = Column(String, nullable=True) - # Free-form illustrative example used to sharpen the LLM judge's - # rubric. Today this is consumed by child sub-labels of a - # categorization parent metric so each label can carry "what does - # this look like in a transcript?" text alongside the rubric in - # ``description``. The column lives on every Metric row for - # forward-compat: a standalone metric could later surface its own - # example without another migration. - example = Column(Text, nullable=True) - - # Configuration - metric_type = Column(String, nullable=False, default=MetricType.RATING.value) - metric_category = Column( - String(30), - nullable=False, - default=MetricCategory.QUALITY.value, - server_default=MetricCategory.QUALITY.value, - ) - trigger = Column(String, nullable=False, default=MetricTrigger.ALWAYS.value) - metric_origin = Column(String(30), nullable=False, default="default") - supported_surfaces = Column(JSON, nullable=False, default=list) # ["agent", "voice_playground", "blind_test"] - enabled_surfaces = Column(JSON, nullable=False, default=list) # subset of supported_surfaces - custom_data_type = Column(String(30), nullable=True) # "boolean" | "enum" | "number_range" - custom_config = Column(JSON, nullable=True) # enum options / number range config - tags = Column(JSON, nullable=True) # ["tone", "latency", ...] - - # Hierarchy: NULL = standalone or parent. When set, this row is a - # child sub-metric of the referenced parent. ON DELETE CASCADE so - # deleting a category removes its children atomically. - parent_metric_id = Column( - UUID(as_uuid=True), - ForeignKey("metrics.id", ondelete="CASCADE"), - nullable=True, - index=True, - ) - # Set only on parent rows (``parent_metric_id IS NULL``). Either - # ``single_choice`` or ``multi_label``. NULL = legacy / non-hierarchical - # metric (no children). - selection_mode = Column(String(20), nullable=True) - - # When true on a parent metric (any selection_mode), the LLM is - # invited during call-import evaluation to emit additional - # candidate sub-labels beyond the user-defined children. The - # candidates surface in a "Discovered labels" panel where the user - # manually promotes them into real child Metric rows. For - # ``single_choice`` parents the discovered entries are - # supplemental — the chosen child is still picked from the - # predefined children so the exactly-one-true invariant holds. - # The validator rejects this flag on standalone / child metrics. - allow_discovery = Column( - Boolean, nullable=False, default=False, server_default="false" - ) - - # When True, this metric is a "transcript-compare judge": the - # call-import evaluator feeds BOTH the production transcript - # (``call_import_rows.transcript``, CSV-supplied) and the diarised - # transcript (``call_import_rows.diarised_transcript``, worker- - # produced by the STT/diarisation pipeline) to the LLM as a - # labeled pair instead of feeding one transcript. The parent - # evaluation's ``CallImportEvaluation.transcript_source`` is - # ignored for these metrics — they always read both columns. - # Rows where either transcript is missing are skipped per-metric - # with ``skipped="comparison_missing_transcript"`` so the rest of - # the row's metrics still produce scores. The Pydantic validator - # rejects ``compare_transcripts`` combined with ``parent_metric_id`` - # or ``selection_mode`` (i.e. it can't simultaneously be part of - # a parent/child hierarchy). The call-import worker also - # auto-promotes a metric to comparison mode when its description - # references the production / diarised transcripts in well-known - # phrases (see ``_metric_text_references_production`` in - # ``app.workers.tasks.evaluate_call_import_row``). - compare_transcripts = Column( - Boolean, nullable=False, default=False, server_default="false" - ) - - parent = relationship( - "Metric", - remote_side=[id], - backref="children", - ) - - # When true, the LLM-judge is asked to also return a short free-form - # rationale alongside the value (stored under ``metric_scores[id].rationale``). - # Adds a second " - LLM Rationale" column in the call-import CSV export. - capture_rationale = Column(Boolean, nullable=False, default=False) - - enabled = Column(Boolean, nullable=False, default=True) - - # Metadata - is_default = Column(Boolean, nullable=False, default=False) # Pre-defined metrics - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class EvaluatorResult(Base): - """EvaluatorResult - Results from running an evaluator with transcription and metric evaluations.""" - __tablename__ = "evaluator_results" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - result_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every evaluator result belongs to a workspace - # within its org. Stamped from the active workspace at creation time - # (either the X-Workspace-Id header or the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # References - evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True, index=True) # Optional - can be None for test calls without persona/scenario - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Nullable for custom evaluators - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) # Optional - can be None for test calls - scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) # Optional - can be None for test calls - - # Result data - name = Column(String, nullable=True) # Scenario name or test call name (optional) - timestamp = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) - duration_seconds = Column(Float, nullable=True) # Call duration - status = Column(String(20), nullable=False, default=EvaluatorResultStatus.QUEUED.value) - - # Audio and transcription - audio_s3_key = Column(String, nullable=True) # S3 key for audio file - transcription = Column(String, nullable=True) # Full transcription - speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] - - # Metric scores - JSON object with metric_id as key and score as value - # Format: {"metric_id_1": {"value": 85, "type": "rating"}, "metric_id_2": {"value": true, "type": "boolean"}} - metric_scores = Column(JSON, nullable=True) - - # Celery task tracking - celery_task_id = Column(String, nullable=True, index=True) # Celery task ID for tracking - - # Error information - error_message = Column(String, nullable=True) - - # Call event tracking (similar to CallRecording) - call_event = Column(String, nullable=True, index=True) # Latest call event (e.g., call_started, call_ended) - provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) - provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" - call_data = Column(JSON, nullable=True) # Full call details from provider (like CallRecording) - - # Data-plane shard routing (payload rows on shard DBs when sharding enabled) - shard_id = Column(String(64), nullable=True, index=True) - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -# Enums moved to enums.py - - -class CallRecordingSource(str, enum.Enum): - """Source of the call recording data.""" - - PLAYGROUND = "playground" - WEBHOOK = "webhook" - - -class CallRecording(Base): - """Call Recording model for tracking voice provider calls.""" - __tablename__ = "call_recordings" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every recording belongs to a workspace within - # its org. For playground-origin rows this is stamped from the active - # workspace at creation time; for webhook-origin rows the worker - # looks up the recording's agent and inherits its workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - call_short_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID - status = Column(Enum(CallRecordingStatus), nullable=False, default=CallRecordingStatus.PENDING, index=True) - call_event = Column(String, nullable=True, index=True) # Latest webhook event (e.g., call_started, call_ended) - source = Column(Enum(CallRecordingSource), nullable=False, default=CallRecordingSource.PLAYGROUND, index=True) - call_data = Column(JSON, nullable=True) # JSON blob for provider response - provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) - provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Reference to our agent - - # Link to EvaluatorResult for metric evaluations - evaluator_result_id = Column( - UUID(as_uuid=True), - ForeignKey("evaluator_results.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - - shard_id = Column(String(64), nullable=True, index=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class EvaluatorResultPayload(Base): - """Heavy evaluator result fields stored on data shards when sharding is enabled.""" - - __tablename__ = "evaluator_result_payloads" - - evaluator_result_id = Column(UUID(as_uuid=True), primary_key=True) - workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) - audio_s3_key = Column(String, nullable=True) - transcription = Column(String, nullable=True) - speaker_segments = Column(JSON, nullable=True) - metric_scores = Column(JSON, nullable=True) - call_data = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class CallRecordingPayload(Base): - """Heavy call recording fields stored on data shards when sharding is enabled.""" - - __tablename__ = "call_recording_payloads" - - call_recording_id = Column(UUID(as_uuid=True), primary_key=True) - workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) - call_data = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class Alert(Base): - """Alert model for configuring monitoring alerts.""" - __tablename__ = "alerts" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - - # Basic information - name = Column(String(255), nullable=False) - description = Column(String, nullable=True) - - # Metric condition configuration - metric_type = Column(String, nullable=False, default=AlertMetricType.NUMBER_OF_CALLS.value) - aggregation = Column(String, nullable=False, default=AlertAggregation.SUM.value) - operator = Column(String, nullable=False, default=AlertOperator.GREATER_THAN.value) - threshold_value = Column(Float, nullable=False) - time_window_minutes = Column(Integer, nullable=False, default=60) # Time window for aggregation - - # Agent selection (JSON array of agent UUIDs, null means all agents) - agent_ids = Column(JSON, nullable=True) - - # Notification configuration - notify_frequency = Column(String, nullable=False, default=AlertNotifyFrequency.IMMEDIATE.value) - notify_emails = Column(JSON, nullable=True) # Array of email addresses - notify_webhooks = Column(JSON, nullable=True) # Array of webhook URLs (Slack, etc.) - - # Status - status = Column(String, nullable=False, default=AlertStatus.ACTIVE.value) - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - # Relationships - alert_history = relationship("AlertHistory", back_populates="alert", cascade="all, delete-orphan") - - -class AlertHistory(Base): - """Alert history model for tracking triggered alerts.""" - __tablename__ = "alert_history" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - alert_id = Column(UUID(as_uuid=True), ForeignKey("alerts.id"), nullable=False, index=True) - - # Trigger information - triggered_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) - triggered_value = Column(Float, nullable=False) # The actual value that triggered the alert - threshold_value = Column(Float, nullable=False) # The threshold at time of trigger - - # Status tracking - status = Column(String, nullable=False, default=AlertHistoryStatus.TRIGGERED.value) - - # Notification tracking - notified_at = Column(DateTime(timezone=True), nullable=True) - notification_details = Column(JSON, nullable=True) # Details of sent notifications - - # Resolution - acknowledged_at = Column(DateTime(timezone=True), nullable=True) - acknowledged_by = Column(String, nullable=True) - resolved_at = Column(DateTime(timezone=True), nullable=True) - resolved_by = Column(String, nullable=True) - resolution_notes = Column(String, nullable=True) - - # Additional context - context_data = Column(JSON, nullable=True) # Additional data about the trigger - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - # Relationships - alert = relationship("Alert", back_populates="alert_history") - - -class CronJob(Base): - """Cron job model for scheduling automated evaluator runs.""" - __tablename__ = "cron_jobs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - - # Basic information - name = Column(String(255), nullable=False) - cron_expression = Column(String(100), nullable=False) # e.g., "0 9 * * 1-5" - timezone = Column(String(100), nullable=False, default="UTC") - - # Run configuration - max_runs = Column(Integer, nullable=False, default=10) - current_runs = Column(Integer, nullable=False, default=0) - - # Evaluators to trigger (JSON array of evaluator UUIDs) - evaluator_ids = Column(JSON, nullable=False) - - # Status - status = Column(String, nullable=False, default=CronJobStatus.ACTIVE.value) - - # Run tracking - next_run_at = Column(DateTime(timezone=True), nullable=True) - last_run_at = Column(DateTime(timezone=True), nullable=True) - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class TTSComparisonStatus(str, enum.Enum): - PENDING = "pending" - GENERATING = "generating" - EVALUATING = "evaluating" - COMPLETED = "completed" - FAILED = "failed" - - -class TTSSampleStatus(str, enum.Enum): - PENDING = "pending" - GENERATING = "generating" - COMPLETED = "completed" - FAILED = "failed" - - -class TTSReportJobStatus(str, enum.Enum): - PENDING = "pending" - PROCESSING = "processing" - COMPLETED = "completed" - FAILED = "failed" - - -class TTSComparison(Base): - """TTS Comparison session for A/B testing voice providers.""" - __tablename__ = "tts_comparisons" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every voice playground comparison belongs to - # a workspace within its org. Children (samples, report jobs, blind - # test shares) inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - simulation_id = Column(String(6), unique=True, index=True, nullable=True) - - name = Column(String(255), nullable=True) - status = Column(String(50), nullable=False, default=TTSComparisonStatus.PENDING.value) - - # 'benchmark' = traditional TTS A/B benchmark (provider-generated audio). - # 'blind_test_only' = standalone blind test built from existing recordings - # / uploads / past TTS samples; no TTS generation happens. - mode = Column(String(32), nullable=False, default="benchmark") - - provider_a = Column(String(100), nullable=True) - model_a = Column(String(100), nullable=True) - voices_a = Column(JSON, nullable=True) - - provider_b = Column(String(100), nullable=True) - model_b = Column(String(100), nullable=True) - voices_b = Column(JSON, nullable=True) - - sample_texts = Column(JSON, nullable=False) - num_runs = Column(Integer, nullable=False, default=1) - - blind_test_results = Column(JSON, nullable=True) - evaluation_summary = Column(JSON, nullable=True) - - eval_stt_provider = Column(String(100), nullable=True) - eval_stt_model = Column(String(100), nullable=True) - - celery_task_id = Column(String, nullable=True, index=True) - error_message = Column(String, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - samples = relationship("TTSSample", back_populates="comparison", cascade="all, delete-orphan") - - -class TTSSample(Base): - """Individual TTS audio sample within a comparison.""" - __tablename__ = "tts_samples" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: mirrors the parent TTSComparison's workspace. - # Denormalized for fast filter-by-workspace listings without a join. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - provider = Column(String(100), nullable=True) - model = Column(String(100), nullable=True) - voice_id = Column(String(255), nullable=True) - voice_name = Column(String(255), nullable=True) - side = Column(String(1), nullable=True) # "A" or "B" - sample_index = Column(Integer, nullable=False) - run_index = Column(Integer, nullable=False, default=0) - - # 'tts' (default, audio is synthesized by a provider), 'recording' (audio - # is reused from a CallImportRow recording), or 'upload' (audio was - # uploaded by the user). Non-tts samples are marked completed up-front - # by the API and skipped by the generation worker. - source_type = Column(String(32), nullable=False, default="tts") - # When source_type == 'recording', references CallImportRow.id (no FK - # constraint to keep cascading deletes simple if a call import is later - # removed; the audio_s3_key is what's actually used). - source_ref_id = Column(UUID(as_uuid=True), nullable=True) - - text = Column(String, nullable=False) - audio_s3_key = Column(String(512), nullable=True) - duration_seconds = Column(Float, nullable=True) - latency_ms = Column(Float, nullable=True) - ttfb_ms = Column(Float, nullable=True) - - evaluation_metrics = Column(JSON, nullable=True) - status = Column(String(50), nullable=False, default=TTSSampleStatus.PENDING.value) - error_message = Column(String, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - comparison = relationship("TTSComparison", back_populates="samples") - - -class TTSReportJob(Base): - """Asynchronous PDF report generation jobs for Voice Playground.""" - __tablename__ = "tts_report_jobs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: mirrors the parent TTSComparison's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) - - status = Column(String(50), nullable=False, default=TTSReportJobStatus.PENDING.value) - format = Column(String(20), nullable=False, default="pdf") - filename = Column(String(255), nullable=True) - s3_key = Column(String(512), nullable=True) - error_message = Column(String, nullable=True) - celery_task_id = Column(String, nullable=True, index=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - comparison = relationship("TTSComparison") - - -class TTSBlindTestShareStatus(str, enum.Enum): - OPEN = "open" - CLOSED = "closed" - - -class TTSBlindTestShare(Base): - """A publicly sharable blind test for a TTSComparison. - - The share_token is the capability: anyone holding it can open the public - form and submit a response. Each comparison has at most one share row. - """ - __tablename__ = "tts_blind_test_shares" - __table_args__ = ( - UniqueConstraint("comparison_id", name="uq_blind_test_shares_comparison"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - comparison_id = Column( - UUID(as_uuid=True), - ForeignKey("tts_comparisons.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: mirrors the parent TTSComparison's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - share_token = Column(String(64), unique=True, nullable=False, index=True) - - title = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - - # Internal notes visible only to the share creator (e.g. which voice - # corresponds to which side, source notes for standalone blind tests). - # Never exposed via the public blind test payload. - creator_notes = Column(Text, nullable=True) - - # JSON list: [{ "key": str, "label": str, "type": "rating"|"comment", "scale": int? }] - custom_metrics = Column(JSON, nullable=False) - - status = Column(String(20), nullable=False, default=TTSBlindTestShareStatus.OPEN.value) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - closed_at = Column(DateTime(timezone=True), nullable=True) - created_by = Column(String, nullable=True) - - comparison = relationship("TTSComparison") - responses = relationship( - "TTSBlindTestResponse", - back_populates="share", - cascade="all, delete-orphan", - ) - - -class TTSBlindTestResponse(Base): - """A single rater's submission against a TTSBlindTestShare.""" - __tablename__ = "tts_blind_test_responses" - __table_args__ = ( - UniqueConstraint("share_id", "rater_email", name="uq_blind_test_response_share_email"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - share_id = Column( - UUID(as_uuid=True), - ForeignKey("tts_blind_test_shares.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - # Workspace isolation: mirrors the parent TTSBlindTestShare's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - rater_name = Column(String(255), nullable=False) - rater_email = Column(String(320), nullable=False, index=True) - - # JSON list keyed by sample_index. Server stores in TRUE A/B orientation - # (already de-flipped from whatever the rater's UI showed): - # [{ - # "sample_index": int, - # "preferred": "A" | "B", - # "ratings_a": { metric_key: number }, - # "ratings_b": { metric_key: number }, - # "comment": str? - # }] - responses = Column(JSON, nullable=False) - - ip = Column(String(64), nullable=True) - user_agent = Column(String(512), nullable=True) - - submitted_at = Column(DateTime(timezone=True), server_default=func.now()) - - share = relationship("TTSBlindTestShare", back_populates="responses") - - -class PromptPartial(Base): - """Prompt Partial - Reusable prompt templates with version history.""" - __tablename__ = "prompt_partials" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every prompt partial belongs to a workspace - # within its org. Versions inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - description = Column(String, nullable=True) - content = Column(Text, nullable=False) - tags = Column(JSON, nullable=True) - current_version = Column(Integer, nullable=False, default=1) - # Cached LLM-generated flowchart for imported production agent prompts. - # Shape: AgentFlowGraph JSON (nodes[], edges[]). - agent_flowchart = Column(JSON, nullable=True) - agent_flowchart_status = Column(String(20), nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - versions = relationship("PromptPartialVersion", back_populates="prompt_partial", cascade="all, delete-orphan", order_by="PromptPartialVersion.version.desc()") - - -class PromptPartialVersion(Base): - """Version history for a prompt partial.""" - __tablename__ = "prompt_partial_versions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - prompt_partial_id = Column(UUID(as_uuid=True), ForeignKey("prompt_partials.id", ondelete="CASCADE"), nullable=False, index=True) - # Workspace isolation: mirrors the parent PromptPartial's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - version = Column(Integer, nullable=False) - content = Column(Text, nullable=False) - change_summary = Column(String, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - created_by = Column(String, nullable=True) - - prompt_partial = relationship("PromptPartial", back_populates="versions") - - __table_args__ = ( - UniqueConstraint('prompt_partial_id', 'version', name='uq_prompt_partial_version'), - ) - - -class CustomTTSVoice(Base): - """Organization-scoped custom TTS voice metadata.""" - __tablename__ = "custom_tts_voices" - __table_args__ = ( - UniqueConstraint("organization_id", "provider", "voice_id", name="uq_custom_tts_voice_org_provider_voice_id"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider = Column(String(100), nullable=False, index=True) - voice_id = Column(String(255), nullable=False) - name = Column(String(255), nullable=False) - gender = Column(String(50), nullable=True) - accent = Column(String(100), nullable=True) - description = Column(Text, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - -class PromptOptimizationRun(Base): - """A single GEPA prompt optimization run for an agent.""" - __tablename__ = "prompt_optimization_runs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every optimization run belongs to a workspace - # within its org. Candidates inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) - evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True) - voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) - - seed_prompt = Column(Text, nullable=False) - best_prompt = Column(Text, nullable=True) - best_score = Column(Float, nullable=True) - - status = Column(String(20), nullable=False, default=PromptOptimizationStatus.PENDING.value) - config = Column(JSON, nullable=True) - reflection_trace = Column(JSON, nullable=True) - metric_history = Column(JSON, nullable=True) - - num_iterations = Column(Integer, nullable=True) - num_metric_calls = Column(Integer, nullable=True) - - celery_task_id = Column(String, nullable=True, index=True) - error_message = Column(Text, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - candidates = relationship("PromptOptimizationCandidate", back_populates="optimization_run", cascade="all, delete-orphan") - - -class PromptOptimizationCandidate(Base): - """A candidate prompt generated during an optimization run.""" - __tablename__ = "prompt_optimization_candidates" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - optimization_run_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_runs.id", ondelete="CASCADE"), nullable=False, index=True) - # Workspace isolation: mirrors the parent PromptOptimizationRun's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - prompt_text = Column(Text, nullable=False) - score = Column(Float, nullable=True) - metric_breakdown = Column(JSON, nullable=True) - reflection_summary = Column(Text, nullable=True) - - parent_candidate_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_candidates.id"), nullable=True) - - is_accepted = Column(Boolean, nullable=False, default=False) - pushed_to_provider_at = Column(DateTime(timezone=True), nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - optimization_run = relationship("PromptOptimizationRun", back_populates="candidates") - - -class TelephonyIntegration(Base): - """Per-organization telephony provider credentials and configuration. - - Multiple rows per (organization_id, provider) are allowed so that an - organization can keep several Plivo / Exotel accounts side-by-side. - A partial unique index in migration 028 enforces at most one row with - is_default = TRUE per (org, provider); resolution falls back to that - default row when the caller does not pin a specific credential. - """ - - __tablename__ = "telephony_integrations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider = Column(String(50), nullable=False, default="plivo") - name = Column(String(255), nullable=True) # Optional friendly name to disambiguate multiple credentials - - auth_id = Column(String(255), nullable=False) - auth_token = Column(String(512), nullable=False) - - verify_app_uuid = Column(String(255), nullable=True) - voice_app_id = Column(String(255), nullable=True) - sip_domain = Column(String(255), nullable=True) - masking_config = Column(JSON, nullable=True) - - is_active = Column(Boolean, default=True, nullable=False) - is_default = Column(Boolean, default=False, nullable=False) - last_tested_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyPhoneNumber(Base): - """Inventory of telephony phone numbers owned by an organization.""" - - __tablename__ = "telephony_phone_numbers" - __table_args__ = ( - UniqueConstraint("organization_id", "phone_number", name="uq_telephony_number_org_phone"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - telephony_integration_id = Column( - UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=True, index=True - ) - - phone_number = Column(String(20), nullable=False, index=True) - country_iso2 = Column(String(2), nullable=True) - region = Column(String(100), nullable=True) - number_type = Column(String(20), nullable=True) - capabilities = Column(JSON, nullable=True) - provider_app_id = Column(String(255), nullable=True) - - is_masking_pool = Column(Boolean, default=False, nullable=False) - inbound_enabled = Column(Boolean, default=True, nullable=False) - outbound_enabled = Column(Boolean, default=True, nullable=False) - source = Column(String(20), nullable=False, default="imported") - agent_id = Column( - UUID(as_uuid=True), - ForeignKey( - "agents.id", - ondelete="SET NULL", - use_alter=True, - name="fk_telephony_phone_numbers_agent_id", - ), - nullable=True, - index=True, - ) - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyDialTarget(Base): - """Org-scoped saved destination numbers for outbound test calls.""" - - __tablename__ = "telephony_dial_targets" - __table_args__ = ( - UniqueConstraint("organization_id", "phone_number", name="uq_telephony_dial_target_org_phone"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - phone_number = Column(String(20), nullable=False, index=True) - label = Column(String(255), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyVerifySession(Base): - """Tracks voice OTP verification sessions via telephony provider.""" - - __tablename__ = "telephony_verify_sessions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider_session_uuid = Column(String(255), nullable=False, unique=True, index=True) - recipient_number = Column(String(20), nullable=False) - channel = Column(String(10), nullable=False, default="voice") - status = Column(String(20), nullable=False, default="pending") - initiated_by = Column(String(255), nullable=True) - verify_app_uuid = Column(String(255), nullable=True) - verified_at = Column(DateTime(timezone=True), nullable=True) - expires_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyMaskedSession(Base): - """Number-masking session between two parties through a middle number.""" - - __tablename__ = "telephony_masked_sessions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - telephony_integration_id = Column(UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=False) - masked_number_id = Column( - UUID(as_uuid=True), ForeignKey("telephony_phone_numbers.id"), nullable=False, index=True - ) - masked_number = Column(String(20), nullable=False) - party_a_number = Column(String(20), nullable=False) - party_b_number = Column(String(20), nullable=False) - status = Column(String(20), nullable=False, default="active") - expires_at = Column(DateTime(timezone=True), nullable=True) - ended_at = Column(DateTime(timezone=True), nullable=True) - session_metadata = Column("metadata", JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class CallImportSchema(Base): - """Reusable Input Parameter schema for the call-uploads flow. - - A schema is workspace-scoped: users define a named bundle of typed - Input Parameters once (e.g. "Standard Voice QA" with conversation_id + - recording_url + transcript + agent_name) and then map those parameters - to CSV/Excel headers each time they upload a new batch. - - Every schema MUST contain exactly one parameter with - ``type='conversation_id'`` and ``is_required=True`` - that's the - mandatory identity field every imported row needs. The invariant is - enforced in app code on create/update (no DB-level CHECK because the - parent + children are written across two tables in one transaction). - """ - - __tablename__ = "call_import_schemas" - __table_args__ = ( - # Case-insensitive uniqueness is enforced via the matching partial - # index on ``LOWER(name)`` in the migration; this constraint here - # would be case-sensitive and is intentionally omitted to avoid - # confusing the user. - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - created_by_user_id = Column( - UUID(as_uuid=True), - ForeignKey("users.id", ondelete="SET NULL"), - nullable=True, - ) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - parameters = relationship( - "CallImportSchemaParameter", - back_populates="schema", - cascade="all, delete-orphan", - order_by="CallImportSchemaParameter.ordering", - ) - - -class CallImportSchemaParameter(Base): - """A single typed parameter inside a :class:`CallImportSchema`. - - ``type`` is one of the strings tracked by - :data:`app.models.enums.CallImportParameterType`. ``conversation_id`` - is reserved for the mandatory identity parameter every schema must - contain. - """ - - __tablename__ = "call_import_schema_parameters" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - schema_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_schemas.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - type = Column(String(32), nullable=False) - description = Column(Text, nullable=True) - is_required = Column(Boolean, nullable=False, default=False) - # Stable ordering so the UI renders parameters in the order the - # schema author defined them (matters when conversation_id is pinned - # first and the user re-orders the rest). - ordering = Column(Integer, nullable=False, default=0) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - schema = relationship("CallImportSchema", back_populates="parameters") - - -class CallImport(Base): - """Batch record for a CSV-driven call import job.""" - - __tablename__ = "call_imports" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every imported batch belongs to a workspace - # within its org. The /upload endpoint stamps it from the active - # workspace header (or the org's Default if absent). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - created_by_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) - last_updated_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True - ) - - # Telephony provider key (e.g. ``'exotel'``, ``'plivo'``). In the - # legacy one-shot ``POST /upload`` endpoint this is supplied with the - # file; in the three-stage flow (UPLOAD -> MAP -> IMPORT) the value - # isn't known until the IMPORT stage, so the column is nullable for - # ``uploaded`` / ``mapped`` batches. - provider = Column(String(50), nullable=True, default="exotel") - # Pin a specific telephony credential for this batch so the worker - # downloads recordings using *that* row instead of the org default. - # NULL preserves legacy behavior (resolve by provider + default). - telephony_integration_id = Column( - UUID(as_uuid=True), - ForeignKey("telephony_integrations.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - original_filename = Column(String(512), nullable=True) - # When the source file was a multi-sheet Excel workbook, this records - # the worksheet the rows came from (one batch per sheet). NULL for CSV - # uploads since CSV has no sheet concept. - sheet_name = Column(String(255), nullable=True) - - # --- Source-file staging (UPLOAD stage) --------------------------- - # The raw CSV / Excel file is stored in S3 between stages so the - # user can come back later to MAP and IMPORT without re-uploading. - # ``source_s3_key`` is NULL on legacy batches that were imported via - # the one-shot endpoint (those batches stay read-only post-import). - source_s3_key = Column(Text, nullable=True) - source_format = Column(String(16), nullable=True) - source_size_bytes = Column(BigInteger, nullable=True) - source_content_type = Column(String(255), nullable=True) - - # Snapshot of the file's sheets + headers captured at UPLOAD time - # so the MAP UI doesn't need to re-fetch the source bytes from S3. - # Shape: ``[{"name": str, "headers": [str, ...], "row_count": int}, ...]``. - available_sheets = Column(JSON, nullable=True) - - # User's explicit "drop these columns" decision captured at MAP - # time. Was validation-only and ephemeral in the legacy flow; now - # persisted so the IMPORT stage can re-parse the file with the same - # mapping/skip intent. - skipped_columns = Column(JSON, nullable=False, default=list) - # Rows skipped at parse time (missing/invalid conversation_id or URL). - # Shape: ``[{"source_row": int, "reason": str, "message": str}, ...]``. - source_row_skips = Column(JSON, nullable=False, default=list) - - # Free-text high-level segregation label. Powers the "Dataset" filter - # at the top of the imports page; multiple imports can share a value. - dataset = Column(String(255), nullable=True, index=True) - - # Reusable Input Parameter schema this batch was uploaded against. - # NULL on legacy batches uploaded before the schema-driven flow - # shipped; those still render via ``column_mapping`` + ``extra_columns`` - # + ``custom_column_mapping`` below. - schema_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_schemas.id", ondelete="RESTRICT"), - nullable=True, - index=True, - ) - # New schema-driven mapping: ``{schema_parameter_name: csv_header}``. - # Populated for new uploads; empty dict on legacy batches. - parameter_mapping = Column(JSON, nullable=False, default=dict) - - # Legacy free-form mapping (pre-schema-flow). Kept on the model so - # batches that were uploaded before the schema feature shipped still - # render correctly on the detail page; new uploads stop writing here. - # Keys: external_call_id (required), transcript, recording_url. - # (DB column ``external_call_id`` is now ``conversation_id``; this - # JSON key stays as-is for historical batches.) - # Values: original CSV header strings (preserve user casing for export). - column_mapping = Column(JSON, nullable=False, default=dict) - # Ordered list of additional CSV header strings the uploader wants - # preserved verbatim into the evaluation export CSV. - extra_columns = Column(JSON, nullable=False, default=list) - # User-defined ``{custom_field_name: csv_header}`` mappings on top of - # the three system fields above. Cells from the mapped CSV columns are - # preserved per row (keyed by the CSV header in ``raw_columns``) and - # surface in the evaluation export under the uploader-chosen name. - custom_column_mapping = Column(JSON, nullable=False, default=dict) - - total_rows = Column(Integer, nullable=False, default=0) - completed_rows = Column(Integer, nullable=False, default=0) - failed_rows = Column(Integer, nullable=False, default=0) - - status = Column( - Enum(CallImportStatus, values_callable=get_enum_values), - nullable=False, - default=CallImportStatus.PENDING, - index=True, - ) - error_message = Column(Text, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - rows = relationship( - "CallImportRow", - back_populates="call_import", - cascade="all, delete-orphan", - order_by="CallImportRow.row_index", - ) - tags = relationship( - "CallImportTag", - secondary="call_import_tag_assignments", - backref="call_imports", - lazy="selectin", - ) - evaluations = relationship( - "CallImportEvaluation", - back_populates="call_import", - cascade="all, delete-orphan", - ) - - -class CallImportShardSlice(Base): - """Registry row: which shard stores a slice of rows for an import.""" - - __tablename__ = "call_import_shard_slices" - - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - primary_key=True, - ) - slice_id = Column(Integer, primary_key=True) - shard_id = Column(String(64), nullable=False, index=True) - row_index_min = Column(Integer, nullable=False) - row_index_max = Column(Integer, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - -class CallImportRow(Base): - """A single row within a CallImport batch (one CSV line / one external call).""" - - __tablename__ = "call_import_rows" - __table_args__ = ( - UniqueConstraint("call_import_id", "row_index", name="uq_call_import_row_index"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - row_index = Column(Integer, nullable=False) - # Was historically named ``external_call_id``; renamed to - # ``conversation_id`` so the new schema-driven upload flow can refer - # to it by a single canonical name across the schema definition, - # exports, and downstream evaluation tables. - conversation_id = Column(String(255), nullable=False, index=True) - # Supplied via CSV for Exotel credentialed imports (required per row). - # Nullable in the schema for legacy rows imported before recording_url - # was mandatory on every Exotel upload. - recording_url = Column(Text, nullable=True) - # Date-only call recording date supplied by the import schema. Used - # for historical report comparisons without timezone/time ambiguity. - recording_date = Column(Date, nullable=True, index=True) - # The "production" transcript: the value supplied via the CSV - # upload mapping. Never overwritten by the diarisation worker — - # the worker writes its output into ``diarised_transcript`` so - # the user keeps both versions side by side. - transcript = Column(Text, nullable=True) - # Snapshot of the original CSV row keyed by the user's headers so the - # evaluation export can reproduce every column the uploader supplied - # (mapped + extra). NULL on legacy rows imported before this column. - raw_columns = Column(JSON, nullable=True) - - # Where the value in ``transcript`` came from. ``csv`` = supplied via - # the upload mapping, ``edited`` = manually changed in the UI. NULL - # on rows that have never had a production transcript. - # (Worker-produced transcripts now live in ``diarised_transcript`` - # and are tracked via ``diarised_transcript_*`` metadata below.) - transcript_source = Column(String(20), nullable=True) - # Provider/model recorded by the (legacy) post-hoc transcription - # worker. New worker runs leave these NULL and write into the - # ``diarised_transcript_*`` columns instead; kept on the model for - # backwards compatibility with pre-split rows that still carry the - # original transcription metadata here. - transcript_provider = Column(String(50), nullable=True) - transcript_model = Column(String(100), nullable=True) - # Lifecycle status for the legacy transcription workflow itself, - # independent of the row's recording-fetch ``status``. ``idle`` = - # no transcribe task has touched this column. New diarisation runs - # update ``diarised_transcript_status`` instead. - transcript_status = Column( - String(20), - nullable=False, - default="idle", - ) - transcript_error = Column(Text, nullable=True) - transcribed_at = Column(DateTime(timezone=True), nullable=True) - - # The "diarised" transcript: produced by the post-hoc - # transcription/diarisation worker. Stored separately so a manual - # diarisation run never clobbers the production transcript above. - # Evaluations can be configured to score against either column - # (see ``CallImportEvaluation.transcript_source``). - diarised_transcript = Column(Text, nullable=True) - # Provider/model the diarisation worker used. Surfaced in the UI - # as "Diarised via deepgram/nova-2" next to the diarised - # transcript section. - diarised_transcript_provider = Column(String(50), nullable=True) - diarised_transcript_model = Column(String(100), nullable=True) - # Lifecycle status for the diarisation workflow. - # ``idle`` = no diarisation task has run; ``pending``/``running`` = - # a Celery task is queued or in flight; ``completed``/``failed`` = - # terminal. Independent of ``transcript_status`` so the two - # transcripts can be in different lifecycle states. - diarised_transcript_status = Column( - String(20), - nullable=False, - default="idle", - server_default="idle", - ) - diarised_transcript_error = Column(Text, nullable=True) - diarised_at = Column(DateTime(timezone=True), nullable=True) - - # Structured speaker turns produced by the diarisation worker — - # ``[{ "speaker": "agent"|"user"|"speaker_3", "text": "...", - # "start": float, "end": float, "raw_speaker": "Speaker 1" }, ...]`` - # The plain-text ``diarised_transcript`` above is a rendered view - # of this list (``: `` per line). When the worker - # cannot recover structured turns (no pyannote token / single- - # speaker recording / provider that doesn't surface segments) this - # column stays NULL and the plain-text path is still populated. - diarised_segments = Column(JSON, nullable=True) - # When True the ``agent`` <-> ``user`` mapping inside - # ``diarised_segments`` is inverted at render / export time. The - # worker writes the canonical mapping using the "first speaker is - # the agent" heuristic; reviewers can flip the toggle from the row - # detail panel without re-running diarisation. - diarised_speaker_swap = Column( - Boolean, - nullable=False, - default=False, - server_default="false", - ) - # LLM that turned the STT plain-text output into structured - # ``diarised_segments``. The legacy diarisation worker used - # pyannote and left these NULL; the current path always runs an - # LLM with the operator-supplied (or default) ``diarised_prompt`` - # below, and records exactly which model + prompt produced each - # row so reviewers can reproduce a specific run. - diarised_llm_provider = Column(String(50), nullable=True) - diarised_llm_model = Column(String(100), nullable=True) - diarised_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) - diarised_prompt = Column(Text, nullable=True) - # Which diarisation pipeline produced this row's turns. - # * ``"stt_llm"`` (default) — two-stage: STT then LLM diariser. - # ``diarised_transcript_provider``/``_model`` describe the STT - # side; ``diarised_llm_provider``/``_model`` the LLM side. - # * ``"llm_only"`` — single-stage: audio fed straight to a - # multimodal LLM. ``diarised_transcript_provider`` is stamped - # with the sentinel ``"llm_only"``; the real model is on - # ``diarised_llm_*``. - # Persisting it on the row (not just the run) lets the row detail - # panel render the right "Diarised via …" label even for ad-hoc - # standalone transcribes (no parent evaluation). - transcribe_mode = Column( - String(20), - nullable=False, - default="stt_llm", - server_default="stt_llm", - ) - - status = Column( - Enum(CallImportRowStatus, values_callable=get_enum_values), - nullable=False, - default=CallImportRowStatus.PENDING, - index=True, - ) - - recording_s3_key = Column(String(1024), nullable=True) - recording_content_type = Column(String(128), nullable=True) - recording_size_bytes = Column(Integer, nullable=True) - - error_message = Column(Text, nullable=True) - attempts = Column(Integer, nullable=False, default=0) - celery_task_id = Column(String(255), nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - call_import = relationship("CallImport", back_populates="rows") - - -@event.listens_for(CallImportRow, "before_insert") -def _call_import_row_fill_workspace_id(_mapper, connection, target): - """Denormalize workspace_id from the parent import when omitted.""" - if target.workspace_id is not None or target.call_import_id is None: - return - workspace_id = connection.execute( - select(CallImport.workspace_id).where( - CallImport.id == target.call_import_id - ) - ).scalar_one_or_none() - if workspace_id is not None: - target.workspace_id = workspace_id - - -class CallImportTag(Base): - """User-defined tag that can be attached to one or more call imports. - - Tags coexist with the free-text ``CallImport.dataset`` column: dataset - is the primary high-level segregation, tags are an optional secondary - classification (an import can have many tags). - """ - - __tablename__ = "call_import_tags" - __table_args__ = ( - UniqueConstraint("organization_id", "name", name="uq_call_import_tag_org_name"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - name = Column(String(255), nullable=False) - color = Column(String(32), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - -class CallImportTagAssignment(Base): - """Many-to-many join table between CallImport and CallImportTag.""" - - __tablename__ = "call_import_tag_assignments" - - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - primary_key=True, - ) - tag_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_tags.id", ondelete="CASCADE"), - primary_key=True, - index=True, - ) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - -class CallImportEvaluation(Base): - """Parent record for an evaluation run over a CallImport batch. - - A user picks a subset of org ``Metric`` rows and triggers an evaluation; - we fan out one ``CallImportEvaluationRow`` per source row and roll up - counters as workers finish. Status mirrors ``CallImportStatus`` plus a - ``RUNNING`` value so the UI can distinguish "queued" from "in flight". - """ - - __tablename__ = "call_import_evaluations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id"), - nullable=False, - index=True, - ) - # Workspace isolation: mirrors the parent CallImport's workspace. - # Denormalized for fast filter-by-workspace listings without a join. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - created_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id"), nullable=True - ) - last_updated_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True - ) - - # Optional user-supplied label for this run. Lets the UI surface - # something more meaningful than the UUID prefix (e.g. "March QA pass"). - name = Column(String(255), nullable=True) - - # JSON list of Metric UUID strings selected for this run. Stored as text - # in JSON so we don't have to deal with PG arrays of UUIDs / cascade - # delete policies when metrics are removed; the loader filters for - # still-existing org metrics at run time. - selected_metric_ids = Column(JSON, nullable=False, default=list) - # Hierarchy grouping snapshot: ``{parent_id_str: [child_id_str, ...]}``. - # Captures which children belong to which parent for THIS run so the UI - # / aggregator can reconstruct the tree even when the user selected - # only a subset of children, or after metrics are deleted / renamed. - # NULL on legacy rows means "no hierarchy" → fall back to flat - # ``selected_metric_ids`` semantics. - selected_metric_groups = Column(JSON, nullable=True) - # User-driven merges of LLM-discovered candidate sub-labels for - # ``allow_discovery`` parents. Shape: - # ``{"": {"": "", ...}}``. - # Populated via ``POST .../discovered-labels/merge``; consulted by - # the discovered-labels aggregator, the flow graph builder, and the - # worker so that rows finishing AFTER a merge cannot reintroduce - # the merged-away slug. Empty dict on fresh rows. - discovered_label_aliases = Column( - JSON, nullable=False, default=dict, server_default="{}" - ) - - # Per-run opt-in for top-level metric discovery. When True, the LLM - # is asked to propose brand-new top-level metrics (boolean / rating / - # category) observed in the transcripts in addition to scoring the - # ``selected_metric_ids`` for the row. Candidates surface in a - # "Discovered metrics" panel on the evaluation's Flow tab and can - # be promoted into real standalone ``Metric`` rows via - # ``POST /metrics/from-discovered``. Defaults to False so existing - # evaluation creation payloads keep their previous behaviour. - discover_new_metrics = Column( - Boolean, nullable=False, default=False, server_default="false" - ) - # Flat slug-to-slug redirect map for user merges + tombstones of - # discovered top-level metric candidates. Mirrors - # ``discovered_label_aliases`` but is NOT nested per parent — - # top-level metric discovery is not scoped to any parent. Shape:: - # - # {"": "", ...} - # - # An empty-string value tombstones the slug so workers finishing - # later can't re-introduce it. - discovered_metric_aliases = Column( - JSON, nullable=False, default=dict, server_default="{}" - ) - - # Run-level LLM config picked from the Run Evaluation modal. NULL on - # legacy rows means "use the historical OpenAI/gpt-4o default" — the - # worker checks for this and falls back accordingly. ``llm_credential_id`` - # pins a specific AIProvider row when the org has multiple credentials - # for the same provider. - llm_provider = Column(String(50), nullable=True) - llm_model = Column(String(100), nullable=True) - llm_credential_id = Column( - UUID(as_uuid=True), - ForeignKey("aiproviders.id", ondelete="SET NULL"), - nullable=True, - ) - llm_config = Column(JSON, nullable=True) - # Optional per-metric LLM override: - # ``{"": {"provider": "...", "model": "...", "credential_id": "..."}}``. - # Each entry overrides the run-level default for that metric only; - # missing keys = use run-level default. Stored as JSON so the UI can - # round-trip arbitrary {provider, model} pairs without migrations. - metric_llm_overrides = Column(JSON, nullable=True) - - # When ``auto_transcribe`` was set on the create payload, record the - # STT provider/model used so the UI can show "Auto-transcribed via - # deepgram/nova-2" on the evaluation header. ``stt_credential_id`` is - # untyped (no FK) because STT keys may live in either ``aiproviders`` - # (OpenAI) or ``integrations`` (Deepgram, ElevenLabs) — the - # transcription service handles the lookup. - stt_provider = Column(String(50), nullable=True) - stt_model = Column(String(100), nullable=True) - stt_credential_id = Column(UUID(as_uuid=True), nullable=True) - - # Run-level LLM diariser config. Used when the create-run / - # retry-run paths chain a ``transcribe_call_import_row_task`` - # because the row is missing a diarised transcript. Persisted on - # the run so a retry uses the same diariser the original create - # call picked (unless the retry payload explicitly overrides). - diarisation_llm_provider = Column(String(50), nullable=True) - diarisation_llm_model = Column(String(100), nullable=True) - diarisation_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) - diarisation_prompt = Column(Text, nullable=True) - # Mode the run was *created* with for its auto-transcribe step. - # Retry chains read this to decide whether to enqueue an STT+LLM - # transcribe or a single-stage multimodal LLM transcribe — without - # it we'd have to infer the mode from "stt_provider is NULL", which - # would silently break legacy rows that simply never configured - # auto-transcribe. See migration 041 for the column DDL. - transcribe_mode = Column( - String(20), - nullable=False, - default="stt_llm", - server_default="stt_llm", - ) - - # Which of the two transcripts on each ``CallImportRow`` this run - # scored against. ``'production'`` reads ``CallImportRow.transcript`` - # (the CSV-supplied value); ``'diarised'`` reads - # ``CallImportRow.diarised_transcript`` (the worker output). When - # the user ticks both checkboxes in the Run Evaluation modal we - # create two ``CallImportEvaluation`` rows — one per source — so - # the two scorings can be compared side-by-side. Defaults to - # ``'production'`` so legacy runs (which always read the single - # historical ``transcript`` column) keep their semantics. - transcript_source = Column( - String(20), - nullable=False, - default="production", - server_default="production", - ) - - # Cached LLM-generated TLDR rendered above the Visualizations charts. - # Populated lazily by ``POST /evaluations/{eval_id}/insights`` so we - # never auto-burn LLM tokens on page load. Shape:: - # {"narrative": str, "patterns": [str, ...], - # "generated_at": iso8601, "generated_at_completed_rows": int, - # "provider": str, "model": str} - # NULL on rows that have never been summarised. - tldr_summary = Column(JSON, nullable=True) - - # Cached LLM-generated user insights for External Audit PDF section 03. - # Populated by a background Celery job triggered alongside TLDR generation. - # Shape: EvaluationUserInsightsState JSON (status, insights[], progress, …). - user_insights = Column(JSON, nullable=True) - - # Cached per-metric failure clustering for internal diagnostics PDF/UI. - # Shape: EvaluationMetricClustersState JSON (status, groups[], …). - metric_clusters = Column(JSON, nullable=True) - - # Cached LLM-generated prompt improvement suggestions keyed to an - # imported agent (PromptPartial tagged __imported_agent__). - # Shape: EvaluationPromptImprovementsState JSON. - prompt_improvements = Column(JSON, nullable=True) - - # Cached LLM explanations for week-over-week metric deltas keyed by - # baseline evaluation id + completed row counts. - period_delta_explanations = Column(JSON, nullable=True) - - status = Column(String(20), nullable=False, default="pending", index=True) - - total_rows = Column(Integer, nullable=False, default=0) - completed_rows = Column(Integer, nullable=False, default=0) - failed_rows = Column(Integer, nullable=False, default=0) - # Flexprice pass-level delta billing watermark: rows already emitted - # on ``call_import.evaluation_completed`` for this evaluation run. - billed_completed_rows = Column( - Integer, nullable=False, default=0, server_default="0" - ) - error_message = Column(Text, nullable=True) - celery_group_id = Column(String(255), nullable=True) - - started_at = Column(DateTime(timezone=True), nullable=True) - finished_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - call_import = relationship("CallImport", back_populates="evaluations") - row_results = relationship( - "CallImportEvaluationRow", - back_populates="evaluation", - cascade="all, delete-orphan", - ) - - -class CallImportEvaluationRow(Base): - """Per-source-row scoring output for a CallImportEvaluation parent.""" - - __tablename__ = "call_import_evaluation_rows" - __table_args__ = ( - UniqueConstraint( - "evaluation_id", "call_import_row_id", name="uq_call_import_evaluation_row" - ), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - call_import_row_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_rows.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - status = Column(String(20), nullable=False, default="pending", index=True) - # Same shape as EvaluatorResult.metric_scores: {metric_id_str: {value, type, metric_name, ...}} - metric_scores = Column(JSON, nullable=False, default=dict) - error_message = Column(Text, nullable=True) - celery_task_id = Column(String(255), nullable=True) - - started_at = Column(DateTime(timezone=True), nullable=True) - finished_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - evaluation = relationship("CallImportEvaluation", back_populates="row_results") - source_row = relationship("CallImportRow") - - -@event.listens_for(CallImportEvaluationRow, "before_insert") -def _call_import_evaluation_row_fill_workspace_id(_mapper, connection, target): - """Denormalize workspace_id from the parent evaluation when omitted.""" - if target.workspace_id is not None or target.evaluation_id is None: - return - workspace_id = connection.execute( - select(CallImportEvaluation.workspace_id).where( - CallImportEvaluation.id == target.evaluation_id - ) - ).scalar_one_or_none() - if workspace_id is not None: - target.workspace_id = workspace_id - - -class CallImportEvaluationReportSnapshot(Base): - """Persisted PDF-report aggregate used for period-over-period deltas.""" - - __tablename__ = "call_import_evaluation_report_snapshots" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - period_label = Column(String(64), nullable=True, index=True) - period_start = Column(Date, nullable=True, index=True) - period_end = Column(Date, nullable=True, index=True) - report_config = Column(JSON, nullable=False, default=dict, server_default="{}") - selected_metric_ids = Column(JSON, nullable=False, default=list, server_default="[]") - metric_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") - insight_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") - narrative = Column(JSON, nullable=True) - total_calls = Column(Integer, nullable=False, default=0) - selected_metric_count = Column(Integer, nullable=False, default=0) - total_metric_count = Column(Integer, nullable=False, default=0) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - -class CallImportEvaluationPdfReport(Base): - """Stored PDF artifact for a call import evaluation report generation.""" - - __tablename__ = "call_import_evaluation_pdf_reports" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - snapshot_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_evaluation_report_snapshots.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - vendor_name = Column(String(120), nullable=False) - report_type = Column(String(20), nullable=False, default="external") - filename = Column(String(255), nullable=True) - s3_key = Column(String(512), nullable=True) - report_config = Column(JSON, nullable=False, default=dict, server_default="{}") - cache_fingerprint = Column(String(64), nullable=True) - created_by = Column(String, nullable=True) - created_by_user_id = Column(UUID(as_uuid=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - -# --------------------------------------------------------------------------- -# Judge Alignment (AlignEval-style hybrid integration) -# -# Three tables back the "Judge Alignment" surface: -# - JudgeDataset: a labeled dataset materialised from one of three sources -# (voice transcripts, existing Metric/Evaluator outputs, -# or a generic CSV upload). Holds the dataset's source -# config + which fields play the role of input/output. -# - JudgeSample: one row in a dataset (input/output pair plus an -# optional binary pass/fail human label). -# - JudgeRun: a single run of an LLM-judge (existing Evaluator) over -# a subset of samples, with computed alignment metrics -# (precision/recall/F1/Cohen's kappa) and per-sample -# predictions. Optionally links to a GEPA optimization -# run when the user kicks off prompt tuning from a -# dataset. -# --------------------------------------------------------------------------- - - -class JudgeDataset(Base): - """Container for binary-labeled samples used to calibrate an LLM-judge.""" - - __tablename__ = "judge_datasets" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - # Workspace isolation: every judge dataset belongs to a workspace - # within its org. Samples and runs inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - name = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - - # One of: "transcript", "metric_output", "csv" - source_type = Column(String(32), nullable=False, index=True) - # Source-specific config. Examples: - # transcript: {"transcription_ids": [...]} or {"agent_id": "..."} - # metric_output: {"metric_id": "...", "evaluator_id": "..."} - # csv: {"s3_key": "...", "filename": "..."} - source_config = Column(JSON, nullable=False, default=dict) - - # Field roles - which textual content is "input" vs "output" for the judge. - # For voice transcripts both default to the transcript text but can be - # tightened (e.g. agent-only turns vs full conversation). - input_field = Column(String(64), nullable=False, default="input") - output_field = Column(String(64), nullable=False, default="output") - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - samples = relationship( - "JudgeSample", - back_populates="dataset", - cascade="all, delete-orphan", - order_by="JudgeSample.created_at", - ) - runs = relationship( - "JudgeRun", - back_populates="dataset", - cascade="all, delete-orphan", - order_by="JudgeRun.created_at.desc()", - ) - - -class JudgeSample(Base): - """One labelable input/output pair within a JudgeDataset.""" - - __tablename__ = "judge_samples" - __table_args__ = ( - UniqueConstraint("dataset_id", "external_id", name="uq_judge_samples_dataset_external"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - dataset_id = Column( - UUID(as_uuid=True), - ForeignKey("judge_datasets.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - # Workspace isolation: mirrors the parent JudgeDataset's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Stable identifier within the source (e.g. transcription UUID, CSV row id). - # Used to dedupe re-imports and link back to the originating record. - external_id = Column(String(128), nullable=True, index=True) - - input_text = Column(Text, nullable=False) - output_text = Column(Text, nullable=False) - - # Binary human label: "pass" | "fail" | null (unlabeled). - # Stored as string (rather than enum) so it stays trivially extendable. - label = Column(String(16), nullable=True, index=True) - labeled_by = Column(String(255), nullable=True) - labeled_at = Column(DateTime(timezone=True), nullable=True) - - # Source-specific context (e.g. agent_id, original metric value, csv row). - extra = Column(JSON, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - dataset = relationship("JudgeDataset", back_populates="samples") - - -class JudgeRun(Base): - """One execution of an LLM-judge against a JudgeDataset, with alignment metrics.""" - - __tablename__ = "judge_runs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - dataset_id = Column( - UUID(as_uuid=True), - ForeignKey("judge_datasets.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - # Workspace isolation: mirrors the parent JudgeDataset's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Reuses the existing Evaluator row (its custom_prompt + llm_provider + llm_model - # define the judge under test). Nullable so a run may target an inline prompt - # in the future without inflating the Evaluator table. - evaluator_id = Column( - UUID(as_uuid=True), ForeignKey("evaluators.id", ondelete="SET NULL"), nullable=True, index=True - ) - - # Which subset was scored: "all" | "dev" | "test" - split = Column(String(16), nullable=False, default="all") - - # Snapshot of the model used (so a later Evaluator edit doesn't rewrite history). - llm_provider = Column(String(64), nullable=True) - llm_model = Column(String(128), nullable=True) - - # Computed alignment metrics: - # {"precision": float, "recall": float, "f1": float, "kappa": float, - # "tp": int, "fp": int, "tn": int, "fn": int, "n": int} - metrics = Column(JSON, nullable=True) - - # Per-sample predictions, keyed by sample_id (UUID string): - # {sample_id: {"prediction": "pass"|"fail", "explanation": str, "raw": str}} - predictions = Column(JSON, nullable=True) - - # Run lifecycle. - status = Column(String(20), nullable=False, default="pending", index=True) - error_message = Column(Text, nullable=True) - celery_task_id = Column(String, nullable=True, index=True) - - # Optional link to a GEPA optimization run kicked off from this dataset. - gepa_optimization_id = Column( - UUID(as_uuid=True), - ForeignKey("prompt_optimization_runs.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - dataset = relationship("JudgeDataset", back_populates="runs") +"""SQLAlchemy database models.""" + +from sqlalchemy import ( + BigInteger, + Boolean, + Column, + Date, + DateTime, + DDL, + Enum, + event, + Float, + ForeignKey, + Integer, + JSON, + String, + Text, + UniqueConstraint, + select, + text, +) +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import relationship +from sqlalchemy.sql import func +import uuid +import enum +from app.models.enums import ( + EvaluationType, EvaluationStatus, EvaluatorResultStatus, RoleEnum, InvitationStatus, + LanguageEnum, CallTypeEnum, CallMediumEnum, GenderEnum, AccentEnum, BackgroundNoiseEnum, + IntegrationPlatform, ModelProvider, VoiceBundleType, TestAgentConversationStatus, + MetricType, MetricCategory, MetricTrigger, CallRecordingStatus, AlertMetricType, AlertAggregation, + AlertOperator, AlertNotifyFrequency, AlertStatus, AlertHistoryStatus, CronJobStatus, + PromptOptimizationStatus, CallImportStatus, CallImportRowStatus, +) + +def get_enum_values(enum_class): + """Helper to get values from enum class for SQLAlchemy.""" + return [e.value for e in enum_class] + +from app.database import Base + + +# Enums moved to enums.py + + +class Organization(Base): + """Organization model for multi-tenancy.""" + + __tablename__ = "organizations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + name = Column(String(255), nullable=False) + voice_playground_threshold_overrides = Column(JSON, nullable=True) + # AlignEval-style judge alignment thresholds. + # Shape: {"min_labels_to_evaluate": int, "min_labels_to_optimize": int} + # Falls back to system defaults (20 / 50) when null. + judge_alignment_settings = Column(JSON, nullable=True) + # Per-org LLM gateway overrides (enabled, gateway_type, base_url, keys). + llm_gateway_settings = Column(JSON, nullable=True) + is_active = Column(Boolean, default=True, nullable=False, server_default=text("true"), index=True) + disabled_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + # Relationships + api_keys = relationship("APIKey", back_populates="organization") + members = relationship("OrganizationMember", back_populates="organization", cascade="all, delete-orphan") + invitations = relationship("Invitation", back_populates="organization", cascade="all, delete-orphan") + workspaces = relationship( + "Workspace", + back_populates="organization", + cascade="all, delete-orphan", + ) + workspace_roles = relationship( + "WorkspaceRole", + back_populates="organization", + cascade="all, delete-orphan", + ) + + +class Workspace(Base): + """Workspace - in-org isolation boundary for call imports and metrics. + + Every organization has at least one workspace (``is_default = True``, + seeded by migration 033). Users pick an "active workspace" in the UI; + list endpoints filter by it so users only see calls/metrics from the + project they're currently working in. Access is governed by + ``workspace_members`` and org-scoped ``workspace_roles`` (capability + bundles); org admins implicitly access all workspaces. + """ + + __tablename__ = "workspaces" + __table_args__ = ( + UniqueConstraint("organization_id", "slug", name="uq_workspaces_org_slug"), + ) + + # ``server_default`` is required so that raw-SQL INSERTs (e.g. the + # per-org Default seed in migration 033) can omit ``id`` and let the + # database fill it in. Without it, ``create_all`` produces a column + # with NOT NULL but no DEFAULT, and the migration crashes with + # ``null value in column "id"``. + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + server_default=text("gen_random_uuid()"), + ) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + slug = Column(String(255), nullable=False) + # At most one default per org. Enforced on Postgres by the partial + # unique index attached via the after_create event below; on + # SQLite (test runs) we rely on the route-level _check_slug_unique + # check + the Default-workspace conftest fixture instead, because + # SQLite doesn't support partial indexes the same way. + is_default = Column(Boolean, nullable=False, default=False, server_default="false") + is_active = Column(Boolean, nullable=False, default=True, server_default="true") + # Reusable PDF/report branding metadata scoped to this workspace. Images + # live in S3. Shape: {"heading": str|null, "images": [{id, s3_key, + # content_type, filename, size_bytes, updated_at}, ...]}. + report_branding = Column(JSON, nullable=True) + created_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + organization = relationship("Organization", back_populates="workspaces") + members = relationship( + "WorkspaceMember", + back_populates="workspace", + cascade="all, delete-orphan", + ) + + +class WorkspaceRole(Base): + """Org-scoped workspace role (system or custom) as a capability bundle.""" + + __tablename__ = "workspace_roles" + __table_args__ = ( + UniqueConstraint("organization_id", "name", name="uq_workspace_roles_org_name"), + ) + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + server_default=text("gen_random_uuid()"), + ) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + capabilities = Column(JSON, nullable=False, default=list) + is_system = Column(Boolean, nullable=False, default=False, server_default="false") + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + organization = relationship("Organization", back_populates="workspace_roles") + members = relationship("WorkspaceMember", back_populates="role") + + +class WorkspaceMember(Base): + """User membership in a workspace with an assigned workspace role.""" + + __tablename__ = "workspace_members" + __table_args__ = ( + UniqueConstraint("workspace_id", "user_id", name="uq_workspace_members_ws_user"), + ) + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + server_default=text("gen_random_uuid()"), + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + user_id = Column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + role_id = Column( + UUID(as_uuid=True), + ForeignKey("workspace_roles.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + added_by_user_id = Column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + workspace = relationship("Workspace", back_populates="members") + user = relationship("User", foreign_keys=[user_id]) + role = relationship("WorkspaceRole", back_populates="members") + added_by = relationship("User", foreign_keys=[added_by_user_id]) + + +# Partial unique index: "at most one default workspace per org". This +# is attached as an after_create event (rather than declared in +# ``__table_args__``) because SQLAlchemy's ``Index(..., +# postgresql_where=...)`` silently degrades to a *full* unique index on +# SQLite - which then forbids any second workspace per org and breaks +# the test suite. ``execute_if(dialect="postgresql")`` makes this DDL +# a no-op on SQLite while still emitting it on Postgres (prod, CI). +event.listen( + Workspace.__table__, + "after_create", + DDL( + "CREATE UNIQUE INDEX IF NOT EXISTS uq_workspaces_org_default " + "ON workspaces (organization_id) WHERE is_default" + ).execute_if(dialect="postgresql"), +) + + +class User(Base): + """User model for authentication and profile management.""" + + __tablename__ = "users" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + email = Column(String(255), unique=True, nullable=False, index=True) + name = Column(String(255), nullable=True) + first_name = Column(String(255), nullable=True) + last_name = Column(String(255), nullable=True) + password_hash = Column(String(255), nullable=True) # Nullable for users created via invitation + external_id = Column(String(255), unique=True, nullable=True, index=True) + auth_provider = Column(String(50), nullable=True) + mfa_enabled = Column(Boolean, default=False, nullable=False) + last_login_at = Column(DateTime(timezone=True), nullable=True) + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + # Relationships + organization_memberships = relationship("OrganizationMember", back_populates="user", cascade="all, delete-orphan") + api_keys = relationship("APIKey", back_populates="user") + invitations = relationship("Invitation", back_populates="invited_user", foreign_keys="Invitation.invited_user_id") + refresh_tokens = relationship("RefreshToken", back_populates="user", cascade="all, delete-orphan") + + +class PlatformAdmin(Base): + """Platform-level administrator (separate from org-scoped users).""" + + __tablename__ = "platform_admins" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + email = Column(String(255), unique=True, nullable=False, index=True) + password_hash = Column(String(255), nullable=False) + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + last_login_at = Column(DateTime(timezone=True), nullable=True) + + signup_reference_codes = relationship( + "SignupReferenceCode", + back_populates="created_by_admin", + foreign_keys="SignupReferenceCode.created_by", + ) + + +class SignupReferenceCode(Base): + """Single- or multi-use reference code required for gated self-service signup.""" + + __tablename__ = "signup_reference_codes" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + code_hash = Column(String(64), unique=True, nullable=False) + label = Column(String(255), nullable=True) + max_uses = Column(Integer, nullable=True) + use_count = Column(Integer, default=0, nullable=False) + expires_at = Column(DateTime(timezone=True), nullable=True) + is_active = Column(Boolean, default=True, nullable=False, index=True) + created_by = Column(UUID(as_uuid=True), ForeignKey("platform_admins.id"), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + created_by_admin = relationship( + "PlatformAdmin", + back_populates="signup_reference_codes", + foreign_keys=[created_by], + ) + + +class RefreshToken(Base): + """Opaque refresh token for extending local-password sessions.""" + + __tablename__ = "refresh_tokens" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True) + token_hash = Column(String(64), unique=True, nullable=False, index=True) + expires_at = Column(DateTime(timezone=True), nullable=False) + revoked_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + user = relationship("User", back_populates="refresh_tokens") + + +class OrganizationMember(Base): + """Organization membership with role.""" + + __tablename__ = "organization_members" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False, index=True) + role = Column(String, nullable=False, default=RoleEnum.READER.value) + + # User preferences for this organization + default_agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) + + joined_at = Column(DateTime(timezone=True), server_default=func.now()) + + # Unique constraint: one membership per user per organization + __table_args__ = ( + UniqueConstraint('organization_id', 'user_id', name='uq_org_user'), + ) + + # Relationships + organization = relationship("Organization", back_populates="members") + user = relationship("User", back_populates="organization_memberships") + default_agent = relationship("Agent", foreign_keys=[default_agent_id]) + + +class Invitation(Base): + """Invitation model for inviting users to organizations.""" + + __tablename__ = "invitations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + invited_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Null if user doesn't exist yet + invited_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False) + email = Column(String(255), nullable=False) # Email of invited user + role = Column(String, nullable=False, default=RoleEnum.READER.value) + status = Column(String, nullable=False, default=InvitationStatus.PENDING.value) + + + + token = Column(String(255), unique=True, nullable=False, index=True) # Invitation token + expires_at = Column(DateTime(timezone=True), nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + accepted_at = Column(DateTime(timezone=True), nullable=True) + + # Relationships + organization = relationship("Organization", back_populates="invitations") + invited_user = relationship("User", foreign_keys=[invited_user_id], back_populates="invitations") + invited_by = relationship("User", foreign_keys=[invited_by_id]) + + +class APIKey(Base): + """API Key model for authentication.""" + + __tablename__ = "api_keys" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + key = Column(String(255), unique=True, nullable=False, index=True) + name = Column(String(255), nullable=True) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Optional: link to user + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + last_used = Column(DateTime(timezone=True), nullable=True) + + # Relationships + organization = relationship("Organization", back_populates="api_keys") + user = relationship("User", back_populates="api_keys") + + +class AudioFile(Base): + """Audio file model.""" + + __tablename__ = "audio_files" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + filename = Column(String(255), nullable=False) + file_path = Column(String(512), nullable=False) + file_size = Column(Integer, nullable=False) # Size in bytes + duration = Column(Float, nullable=True) # Duration in seconds + sample_rate = Column(Integer, nullable=True) + channels = Column(Integer, nullable=True) + format = Column(String(10), nullable=False) # wav, mp3, flac, etc. + uploaded_at = Column(DateTime(timezone=True), server_default=func.now()) + + # Relationships + evaluations = relationship("Evaluation", back_populates="audio_file") + + +class Evaluation(Base): + """Evaluation job model.""" + + __tablename__ = "evaluations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every legacy audio evaluation belongs to a + # workspace within its org. Stamped from the X-Workspace-Id header + # (falling back to the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + audio_id = Column(UUID(as_uuid=True), ForeignKey("audio_files.id"), nullable=False) + reference_text = Column(String, nullable=True) # For WER calculation + evaluation_type = Column(String, nullable=False) + model_name = Column(String(100), nullable=True) + status = Column(String, default=EvaluationStatus.PENDING.value, nullable=False) + + + + metrics_requested = Column(JSON, nullable=True) # List of requested metrics + created_at = Column(DateTime(timezone=True), server_default=func.now()) + started_at = Column(DateTime(timezone=True), nullable=True) + completed_at = Column(DateTime(timezone=True), nullable=True) + error_message = Column(String, nullable=True) + + # Relationships + audio_file = relationship("AudioFile", back_populates="evaluations") + result = relationship("EvaluationResult", back_populates="evaluation", uselist=False) + + +class EvaluationResult(Base): + """Evaluation result model.""" + + __tablename__ = "evaluation_results" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column(UUID(as_uuid=True), ForeignKey("evaluations.id"), nullable=False, unique=True) + # Workspace isolation: mirrors the parent Evaluation's workspace. + # Denormalized for fast filter-by-workspace listings without a join. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + transcript = Column(String, nullable=True) + metrics = Column(JSON, nullable=False) # {"wer": 0.05, "latency_ms": 1250, ...} + raw_output = Column(JSON, nullable=True) # Full model output + processing_time = Column(Float, nullable=True) # Processing time in seconds + model_used = Column(String(100), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + # Relationships + evaluation = relationship("Evaluation", back_populates="result") + + +# ============================================ +# VAIOPS MODELS - Voice AI Ops +# ============================================ + +# Enums moved to enums.py + + +class Agent(Base): + """Test Agent - The voice AI agent being evaluated""" + __tablename__ = "agents" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + agent_id = Column(String(6), unique=True, nullable=True, index=True) # 6-digit ID + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every agent belongs to a workspace within its + # org. Stamped from the X-Workspace-Id header (falling back to the + # org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String, nullable=False) + phone_number = Column(String, nullable=True) # Optional, required only for phone_call + language = Column(String, nullable=False, default=LanguageEnum.ENGLISH.value) + description = Column(String) + provider_prompt = Column(Text, nullable=True) + provider_prompt_synced_at = Column(DateTime(timezone=True), nullable=True) + call_type = Column(String, nullable=False, default=CallTypeEnum.OUTBOUND.value) + call_medium = Column(String, nullable=False, default=CallMediumEnum.PHONE_CALL.value) + telephony_phone_number_id = Column( + UUID(as_uuid=True), + ForeignKey("telephony_phone_numbers.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + + + + # Voice configuration - either voice_bundle_id OR ai_provider_id OR voice_ai_integration_id (mutually exclusive) + voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True, index=True) + ai_provider_id = Column(UUID(as_uuid=True), ForeignKey("aiproviders.id"), nullable=True, index=True) + + # Voice AI agent integration (Retell, Vapi, etc.) + voice_ai_integration_id = Column(UUID(as_uuid=True), ForeignKey("integrations.id"), nullable=True, index=True) + voice_ai_agent_id = Column(String, nullable=True) # Agent ID from the external provider (Retell/Vapi) + prompt_variables = Column(JSON, nullable=True) + silence_hangup_secs = Column(Integer, nullable=False, server_default="15") + + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + created_by = Column(String) + + +class Persona(Base): + """Persona - TTS provider-tied voice identity for testing""" + __tablename__ = "personas" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every persona belongs to a workspace within + # its org. Stamped from the X-Workspace-Id header (falling back to + # the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String, nullable=False) + gender = Column(String, nullable=False, default=GenderEnum.NEUTRAL.value) + tts_provider = Column(String(100), nullable=True) + tts_voice_id = Column(String(255), nullable=True) + tts_voice_name = Column(String(255), nullable=True) + is_custom = Column(Boolean, default=False) + description = Column(Text, nullable=True) + tts_config = Column(JSON, nullable=True) + llm_temperature = Column(Float, nullable=True) + llm_max_tokens = Column(Integer, nullable=True) + response_delay_ms = Column(Integer, nullable=True) + max_turns = Column(Integer, nullable=True) + allow_interruptions = Column(Boolean, nullable=True) + + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + created_by = Column(String) + + +class Scenario(Base): + """Scenario - The conversation scenario/test case""" + __tablename__ = "scenarios" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every scenario belongs to a workspace within + # its org. Stamped from the X-Workspace-Id header (falling back to + # the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) + name = Column(String, nullable=False) + description = Column(String) + required_info = Column(JSON) + + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + created_by = Column(String) + + +# Enums moved to enums.py + + +class Integration(Base): + """Integration model for connecting with external voice AI platforms.""" + __tablename__ = "integrations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + platform = Column(String, nullable=False) + + + + name = Column(String, nullable=True) # Optional friendly name + api_key = Column(String, nullable=False) # Encrypted Private API key for the platform + public_key = Column(String, nullable=True) # Optional Public API key (e.g. for Vapi) + is_active = Column(Boolean, default=True, nullable=False) + # Multiple credentials per (org, platform) are allowed. is_default marks + # the row used when a caller does not explicitly select a credential. + # A partial unique index in migration 028 enforces at most one default + # per (org, platform) at the DB level. + is_default = Column(Boolean, default=False, nullable=False) + # inherit | gateway | direct — per-credential LLM routing override + routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated + + +class ManualTranscription(Base): + """Manual transcription model for storing transcriptions from S3 audio files.""" + + __tablename__ = "manual_transcriptions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + name = Column(String(255), nullable=True) # User-friendly name for the transcription + audio_file_key = Column(String(512), nullable=False) # S3 key or file path + transcript = Column(String, nullable=False) # Full transcript text + speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] + stt_model = Column(String(100), nullable=True) # STT model used (e.g., "whisper-1", "google-speech-v2") + stt_provider = Column(String, nullable=True) # Provider used + + + + language = Column(String(10), nullable=True) # Detected or specified language + processing_time = Column(Float, nullable=True) # Processing time in seconds + raw_output = Column(JSON, nullable=True) # Full model output for reference + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class ConversationEvaluation(Base): + """Conversation evaluation model for evaluating manual transcriptions against agent objectives.""" + + __tablename__ = "conversation_evaluations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + transcription_id = Column(UUID(as_uuid=True), ForeignKey("manual_transcriptions.id"), nullable=False, index=True) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) + + # Evaluation results + objective_achieved = Column(Boolean, nullable=False) # Binary: was the conversation objective achieved? + objective_achieved_reason = Column(String, nullable=True) # Explanation for the binary result + additional_metrics = Column(JSON, nullable=True) # Additional evaluation metrics (e.g., professionalism, clarity, etc.) + overall_score = Column(Float, nullable=True) # Overall score (0.0 to 1.0) + + # LLM metadata + llm_provider = Column(Enum(ModelProvider, native_enum=False), nullable=True) + + llm_model = Column(String(100), nullable=True) + llm_response = Column(JSON, nullable=True) # Full LLM response for reference + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class AIProvider(Base): + """AI Provider - Stores API keys for different AI platforms.""" + __tablename__ = "aiproviders" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider = Column(String, nullable=False) + + + + api_key = Column(String, nullable=False) # Encrypted API key + name = Column(String, nullable=True) # Optional friendly name + # Azure OpenAI resource endpoint (e.g. https://my-resource.openai.azure.com). + # Only used when provider is azure; other providers ignore this column. + endpoint_url = Column(String, nullable=True) + is_active = Column(Boolean, default=True, nullable=False) + # Multiple AIProvider rows per (org, provider) are allowed. is_default + # marks the row resolved when no explicit credential id is selected. + # A partial unique index in migration 028 enforces at most one default. + is_default = Column(Boolean, default=False, nullable=False) + # inherit | gateway | direct — per-credential LLM routing override + routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") + # Bifrost custom model ID used when routing via gateway + gateway_model = Column(String(255), nullable=True) + # inherit | litellm_shim | native_openai — Bifrost API surface override + gateway_interface = Column(String(20), nullable=False, default="inherit", server_default="inherit") + # Optional per-credential Bifrost/gateway base URL override + gateway_base_url = Column(String(512), nullable=True) + # Optional auth header for Bifrost (e.g. x-bf-vk, Authorization, x-api-key) + gateway_auth_header = Column(String(64), nullable=True) + # Env var name whose value is sent as the gateway auth secret + gateway_auth_secret_env = Column(String(128), nullable=True) + # Encrypted inline gateway auth secret (alternative to env var) + gateway_auth_secret = Column(String, nullable=True) + # Arbitrary HTTP headers sent with gateway-routed LiteLLM calls + gateway_extra_headers = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated + + +# Enums moved to enums.py + + +class VoiceBundle(Base): + """VoiceBundle - Composable unit combining STT, LLM, and TTS for voice AI testing, or S2S models.""" + __tablename__ = "voicebundles" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + name = Column(String, nullable=False) + description = Column(String, nullable=True) + + # Bundle type: either STT+LLM+TTS or S2S + # Using String instead of Enum to avoid SQLAlchemy enum conversion issues + # The enum conversion is handled in the Pydantic schemas + bundle_type = Column(String(50), nullable=False, default=VoiceBundleType.STT_LLM_TTS.value) + + # STT Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S + stt_provider = Column(String, nullable=True) + # Optional explicit credential row (aiproviders.id or integrations.id). + # When NULL the credential resolver picks the default row for the + # provider. No FK is set because the target table varies by provider. + stt_credential_id = Column(UUID(as_uuid=True), nullable=True) + + stt_model = Column(String, nullable=True) # e.g., "whisper-1", "google-speech-v2" + + # LLM Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S + llm_provider = Column(String, nullable=True) + llm_credential_id = Column(UUID(as_uuid=True), nullable=True) + + llm_model = Column(String, nullable=True) # e.g., "gpt-4", "claude-3-opus" + llm_temperature = Column(Float, nullable=True, default=0.7) + llm_max_tokens = Column(Integer, nullable=True) + llm_config = Column(JSON, nullable=True) # Additional LLM configuration (extensible) + + # TTS Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S + tts_provider = Column(String, nullable=True) + tts_credential_id = Column(UUID(as_uuid=True), nullable=True) + + tts_model = Column(String, nullable=True) # e.g., "tts-1", "neural-voice" + tts_voice = Column(String, nullable=True) # Voice selection if applicable + tts_config = Column(JSON, nullable=True) # Additional TTS configuration (extensible) + + # S2S Configuration - required for S2S type, optional for STT_LLM_TTS + s2s_provider = Column(String, nullable=True) + s2s_credential_id = Column(UUID(as_uuid=True), nullable=True) + + + + s2s_model = Column(String, nullable=True) # e.g., "gpt-4o-transcribe", speech-to-speech model + s2s_config = Column(JSON, nullable=True) # Additional S2S configuration (extensible) + + # Additional configuration for extensibility + extra_metadata = Column(JSON, nullable=True) # For future extensions (renamed from 'metadata' to avoid SQLAlchemy conflict) + + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +# Enums moved to enums.py + + +class TestAgentConversation(Base): + """Test Agent Conversation - Records conversations between test AI agent and voice AI agent.""" + __tablename__ = "test_agent_conversations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every playground conversation belongs to a + # workspace within its org. Stamped from the X-Workspace-Id header + # (falling back to the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Configuration + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=False) + voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) + + # Conversation data + status = Column(String, nullable=False, default=TestAgentConversationStatus.INITIALIZING.value) + + + + live_transcription = Column(JSON, nullable=True) # Array of conversation turns with timestamps + conversation_audio_key = Column(String, nullable=True) # S3 key for recorded conversation audio + full_transcript = Column(String, nullable=True) # Full conversation transcript + + # Metadata + started_at = Column(DateTime(timezone=True), server_default=func.now()) + ended_at = Column(DateTime(timezone=True), nullable=True) + duration_seconds = Column(Float, nullable=True) + + # Additional metadata + conversation_metadata = Column(JSON, nullable=True) # Additional conversation metadata + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class EvaluatorSuite(Base): + """Evaluator suite — one agent + one persona + N scenario combinations.""" + + __tablename__ = "evaluator_suites" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + name = Column(String, nullable=True) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) + metric_ids = Column(JSON, nullable=True) + llm_provider = Column(String, nullable=True) + llm_model = Column(String, nullable=True) + llm_config = Column(JSON, nullable=True) + tags = Column(JSON, nullable=True) + default_runs_per_combination = Column(Integer, nullable=False, default=1) + round_robin_index = Column(Integer, nullable=False, default=0) + is_active = Column(Boolean, nullable=False, default=False) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class Evaluator(Base): + """Evaluator - Configuration for testing agents with specific persona and scenario combinations, or custom prompt evaluators.""" + __tablename__ = "evaluators" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluator_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every evaluator belongs to a workspace within + # its org. Stamped from the X-Workspace-Id header (falling back to + # the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Display name (required for custom evaluators, optional for standard) + name = Column(String, nullable=True) + + # Parent suite (nullable for legacy/custom evaluators) + suite_id = Column( + UUID(as_uuid=True), + ForeignKey("evaluator_suites.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + + # Standard evaluator configuration (nullable for custom evaluators) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) + + # Custom evaluator prompt (used instead of agent/persona/scenario) + custom_prompt = Column(Text, nullable=True) + + # Custom evaluator metric selection. When set, the worker filters the + # enabled-org metrics down to only these IDs (list of metric UUID strings). + # Standard evaluators leave this NULL and use all enabled agent metrics. + metric_ids = Column(JSON, nullable=True) + + # LLM configuration for evaluation (overrides hardcoded defaults) + llm_provider = Column(String, nullable=True) # e.g. "openai", "anthropic", "google" + llm_model = Column(String, nullable=True) # e.g. "gpt-4.1", "claude-sonnet-4-20250514" + llm_config = Column(JSON, nullable=True) + + # Tags for categorization + tags = Column(JSON, nullable=True) # Array of tag strings + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +# Enums moved to enums.py + + +class Metric(Base): + """Metric - Configuration for evaluation metrics. + + Supports a 2-level hierarchy via ``parent_metric_id``: a "category" + parent metric (e.g. "Call Outcome") owns N child sub-metric labels + (e.g. "happy_completion", "angry_hangup"). ``selection_mode`` is set + only on parents and controls how the LLM scores children together + (``single_choice`` = pick exactly one; ``multi_label`` = independent + yes/no with logical consistency). Children are always boolean. + """ + __tablename__ = "metrics" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: two-shape column. + # + # * ``workspace_id = `` — workspace-scoped metric. Only + # visible inside that workspace (the default behavior; existing + # rows all look like this). + # * ``workspace_id IS NULL`` — org-shared metric. Surfaces in + # every workspace's listing under this org so users don't have + # to recreate the same metric per workspace. + # + # Children always inherit their parent's ``workspace_id`` (including + # NULL) so a category metric's whole subtree shares one scope; the + # add-child / promote-discovered endpoints enforce this. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=True, + index=True, + ) + + # Basic information + name = Column(String, nullable=False) + description = Column(String, nullable=True) + # Free-form illustrative example used to sharpen the LLM judge's + # rubric. Today this is consumed by child sub-labels of a + # categorization parent metric so each label can carry "what does + # this look like in a transcript?" text alongside the rubric in + # ``description``. The column lives on every Metric row for + # forward-compat: a standalone metric could later surface its own + # example without another migration. + example = Column(Text, nullable=True) + + # Configuration + metric_type = Column(String, nullable=False, default=MetricType.RATING.value) + metric_category = Column( + String(30), + nullable=False, + default=MetricCategory.QUALITY.value, + server_default=MetricCategory.QUALITY.value, + ) + trigger = Column(String, nullable=False, default=MetricTrigger.ALWAYS.value) + metric_origin = Column(String(30), nullable=False, default="default") + supported_surfaces = Column(JSON, nullable=False, default=list) # ["agent", "voice_playground", "blind_test"] + enabled_surfaces = Column(JSON, nullable=False, default=list) # subset of supported_surfaces + custom_data_type = Column(String(30), nullable=True) # "boolean" | "enum" | "number_range" + custom_config = Column(JSON, nullable=True) # enum options / number range config + tags = Column(JSON, nullable=True) # ["tone", "latency", ...] + + # Hierarchy: NULL = standalone or parent. When set, this row is a + # child sub-metric of the referenced parent. ON DELETE CASCADE so + # deleting a category removes its children atomically. + parent_metric_id = Column( + UUID(as_uuid=True), + ForeignKey("metrics.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + # Set only on parent rows (``parent_metric_id IS NULL``). Either + # ``single_choice`` or ``multi_label``. NULL = legacy / non-hierarchical + # metric (no children). + selection_mode = Column(String(20), nullable=True) + + # When true on a parent metric (any selection_mode), the LLM is + # invited during call-import evaluation to emit additional + # candidate sub-labels beyond the user-defined children. The + # candidates surface in a "Discovered labels" panel where the user + # manually promotes them into real child Metric rows. For + # ``single_choice`` parents the discovered entries are + # supplemental — the chosen child is still picked from the + # predefined children so the exactly-one-true invariant holds. + # The validator rejects this flag on standalone / child metrics. + allow_discovery = Column( + Boolean, nullable=False, default=False, server_default="false" + ) + + # When True, this metric is a "transcript-compare judge": the + # call-import evaluator feeds BOTH the production transcript + # (``call_import_rows.transcript``, CSV-supplied) and the diarised + # transcript (``call_import_rows.diarised_transcript``, worker- + # produced by the STT/diarisation pipeline) to the LLM as a + # labeled pair instead of feeding one transcript. The parent + # evaluation's ``CallImportEvaluation.transcript_source`` is + # ignored for these metrics — they always read both columns. + # Rows where either transcript is missing are skipped per-metric + # with ``skipped="comparison_missing_transcript"`` so the rest of + # the row's metrics still produce scores. The Pydantic validator + # rejects ``compare_transcripts`` combined with ``parent_metric_id`` + # or ``selection_mode`` (i.e. it can't simultaneously be part of + # a parent/child hierarchy). The call-import worker also + # auto-promotes a metric to comparison mode when its description + # references the production / diarised transcripts in well-known + # phrases (see ``_metric_text_references_production`` in + # ``app.workers.tasks.evaluate_call_import_row``). + compare_transcripts = Column( + Boolean, nullable=False, default=False, server_default="false" + ) + + parent = relationship( + "Metric", + remote_side=[id], + backref="children", + ) + + # When true, the LLM-judge is asked to also return a short free-form + # rationale alongside the value (stored under ``metric_scores[id].rationale``). + # Adds a second " - LLM Rationale" column in the call-import CSV export. + capture_rationale = Column(Boolean, nullable=False, default=False) + + enabled = Column(Boolean, nullable=False, default=True) + + # Studio draft lifecycle: ``draft`` metrics are visible only in Metrics + # Studio until promoted to ``active``. + lifecycle = Column( + String(20), + nullable=False, + default="active", + server_default="active", + ) + promoted_from_draft_at = Column(DateTime(timezone=True), nullable=True) + studio_notes = Column(Text, nullable=True) + + # Metadata + is_default = Column(Boolean, nullable=False, default=False) # Pre-defined metrics + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class EvaluatorResult(Base): + """EvaluatorResult - Results from running an evaluator with transcription and metric evaluations.""" + __tablename__ = "evaluator_results" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + result_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every evaluator result belongs to a workspace + # within its org. Stamped from the active workspace at creation time + # (either the X-Workspace-Id header or the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # References + evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True, index=True) # Optional - can be None for test calls without persona/scenario + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Nullable for custom evaluators + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) # Optional - can be None for test calls + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) # Optional - can be None for test calls + + # Result data + name = Column(String, nullable=True) # Scenario name or test call name (optional) + timestamp = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + duration_seconds = Column(Float, nullable=True) # Call duration + status = Column(String(20), nullable=False, default=EvaluatorResultStatus.QUEUED.value) + + # Audio and transcription + audio_s3_key = Column(String, nullable=True) # S3 key for audio file + transcription = Column(String, nullable=True) # Full transcription + speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] + + # Metric scores - JSON object with metric_id as key and score as value + # Format: {"metric_id_1": {"value": 85, "type": "rating"}, "metric_id_2": {"value": true, "type": "boolean"}} + metric_scores = Column(JSON, nullable=True) + + # Celery task tracking + celery_task_id = Column(String, nullable=True, index=True) # Celery task ID for tracking + + # Error information + error_message = Column(String, nullable=True) + + # Call event tracking (similar to CallRecording) + call_event = Column(String, nullable=True, index=True) # Latest call event (e.g., call_started, call_ended) + provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) + provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" + call_data = Column(JSON, nullable=True) # Full call details from provider (like CallRecording) + + # Data-plane shard routing (payload rows on shard DBs when sharding enabled) + shard_id = Column(String(64), nullable=True, index=True) + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +# Enums moved to enums.py + + +class CallRecordingSource(str, enum.Enum): + """Source of the call recording data.""" + + PLAYGROUND = "playground" + WEBHOOK = "webhook" + + +class CallRecording(Base): + """Call Recording model for tracking voice provider calls.""" + __tablename__ = "call_recordings" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every recording belongs to a workspace within + # its org. For playground-origin rows this is stamped from the active + # workspace at creation time; for webhook-origin rows the worker + # looks up the recording's agent and inherits its workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + call_short_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID + status = Column(Enum(CallRecordingStatus), nullable=False, default=CallRecordingStatus.PENDING, index=True) + call_event = Column(String, nullable=True, index=True) # Latest webhook event (e.g., call_started, call_ended) + source = Column(Enum(CallRecordingSource), nullable=False, default=CallRecordingSource.PLAYGROUND, index=True) + call_data = Column(JSON, nullable=True) # JSON blob for provider response + provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) + provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Reference to our agent + + # Link to EvaluatorResult for metric evaluations + evaluator_result_id = Column( + UUID(as_uuid=True), + ForeignKey("evaluator_results.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + shard_id = Column(String(64), nullable=True, index=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class EvaluatorResultPayload(Base): + """Heavy evaluator result fields stored on data shards when sharding is enabled.""" + + __tablename__ = "evaluator_result_payloads" + + evaluator_result_id = Column(UUID(as_uuid=True), primary_key=True) + workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) + audio_s3_key = Column(String, nullable=True) + transcription = Column(String, nullable=True) + speaker_segments = Column(JSON, nullable=True) + metric_scores = Column(JSON, nullable=True) + call_data = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class CallRecordingPayload(Base): + """Heavy call recording fields stored on data shards when sharding is enabled.""" + + __tablename__ = "call_recording_payloads" + + call_recording_id = Column(UUID(as_uuid=True), primary_key=True) + workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) + call_data = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class Alert(Base): + """Alert model for configuring monitoring alerts.""" + __tablename__ = "alerts" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + + # Basic information + name = Column(String(255), nullable=False) + description = Column(String, nullable=True) + + # Metric condition configuration + metric_type = Column(String, nullable=False, default=AlertMetricType.NUMBER_OF_CALLS.value) + aggregation = Column(String, nullable=False, default=AlertAggregation.SUM.value) + operator = Column(String, nullable=False, default=AlertOperator.GREATER_THAN.value) + threshold_value = Column(Float, nullable=False) + time_window_minutes = Column(Integer, nullable=False, default=60) # Time window for aggregation + + # Agent selection (JSON array of agent UUIDs, null means all agents) + agent_ids = Column(JSON, nullable=True) + + # Notification configuration + notify_frequency = Column(String, nullable=False, default=AlertNotifyFrequency.IMMEDIATE.value) + notify_emails = Column(JSON, nullable=True) # Array of email addresses + notify_webhooks = Column(JSON, nullable=True) # Array of webhook URLs (Slack, etc.) + + # Status + status = Column(String, nullable=False, default=AlertStatus.ACTIVE.value) + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + # Relationships + alert_history = relationship("AlertHistory", back_populates="alert", cascade="all, delete-orphan") + + +class AlertHistory(Base): + """Alert history model for tracking triggered alerts.""" + __tablename__ = "alert_history" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + alert_id = Column(UUID(as_uuid=True), ForeignKey("alerts.id"), nullable=False, index=True) + + # Trigger information + triggered_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + triggered_value = Column(Float, nullable=False) # The actual value that triggered the alert + threshold_value = Column(Float, nullable=False) # The threshold at time of trigger + + # Status tracking + status = Column(String, nullable=False, default=AlertHistoryStatus.TRIGGERED.value) + + # Notification tracking + notified_at = Column(DateTime(timezone=True), nullable=True) + notification_details = Column(JSON, nullable=True) # Details of sent notifications + + # Resolution + acknowledged_at = Column(DateTime(timezone=True), nullable=True) + acknowledged_by = Column(String, nullable=True) + resolved_at = Column(DateTime(timezone=True), nullable=True) + resolved_by = Column(String, nullable=True) + resolution_notes = Column(String, nullable=True) + + # Additional context + context_data = Column(JSON, nullable=True) # Additional data about the trigger + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + # Relationships + alert = relationship("Alert", back_populates="alert_history") + + +class CronJob(Base): + """Cron job model for scheduling automated evaluator runs.""" + __tablename__ = "cron_jobs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + + # Basic information + name = Column(String(255), nullable=False) + cron_expression = Column(String(100), nullable=False) # e.g., "0 9 * * 1-5" + timezone = Column(String(100), nullable=False, default="UTC") + + # Run configuration + max_runs = Column(Integer, nullable=False, default=10) + current_runs = Column(Integer, nullable=False, default=0) + + # Evaluators to trigger (JSON array of evaluator UUIDs) + evaluator_ids = Column(JSON, nullable=False) + + # Status + status = Column(String, nullable=False, default=CronJobStatus.ACTIVE.value) + + # Run tracking + next_run_at = Column(DateTime(timezone=True), nullable=True) + last_run_at = Column(DateTime(timezone=True), nullable=True) + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class TTSComparisonStatus(str, enum.Enum): + PENDING = "pending" + GENERATING = "generating" + EVALUATING = "evaluating" + COMPLETED = "completed" + FAILED = "failed" + + +class TTSSampleStatus(str, enum.Enum): + PENDING = "pending" + GENERATING = "generating" + COMPLETED = "completed" + FAILED = "failed" + + +class TTSReportJobStatus(str, enum.Enum): + PENDING = "pending" + PROCESSING = "processing" + COMPLETED = "completed" + FAILED = "failed" + + +class TTSComparison(Base): + """TTS Comparison session for A/B testing voice providers.""" + __tablename__ = "tts_comparisons" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every voice playground comparison belongs to + # a workspace within its org. Children (samples, report jobs, blind + # test shares) inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + simulation_id = Column(String(6), unique=True, index=True, nullable=True) + + name = Column(String(255), nullable=True) + status = Column(String(50), nullable=False, default=TTSComparisonStatus.PENDING.value) + + # 'benchmark' = traditional TTS A/B benchmark (provider-generated audio). + # 'blind_test_only' = standalone blind test built from existing recordings + # / uploads / past TTS samples; no TTS generation happens. + mode = Column(String(32), nullable=False, default="benchmark") + + provider_a = Column(String(100), nullable=True) + model_a = Column(String(100), nullable=True) + voices_a = Column(JSON, nullable=True) + + provider_b = Column(String(100), nullable=True) + model_b = Column(String(100), nullable=True) + voices_b = Column(JSON, nullable=True) + + sample_texts = Column(JSON, nullable=False) + num_runs = Column(Integer, nullable=False, default=1) + + blind_test_results = Column(JSON, nullable=True) + evaluation_summary = Column(JSON, nullable=True) + + eval_stt_provider = Column(String(100), nullable=True) + eval_stt_model = Column(String(100), nullable=True) + + celery_task_id = Column(String, nullable=True, index=True) + error_message = Column(String, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + samples = relationship("TTSSample", back_populates="comparison", cascade="all, delete-orphan") + + +class TTSSample(Base): + """Individual TTS audio sample within a comparison.""" + __tablename__ = "tts_samples" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: mirrors the parent TTSComparison's workspace. + # Denormalized for fast filter-by-workspace listings without a join. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + provider = Column(String(100), nullable=True) + model = Column(String(100), nullable=True) + voice_id = Column(String(255), nullable=True) + voice_name = Column(String(255), nullable=True) + side = Column(String(1), nullable=True) # "A" or "B" + sample_index = Column(Integer, nullable=False) + run_index = Column(Integer, nullable=False, default=0) + + # 'tts' (default, audio is synthesized by a provider), 'recording' (audio + # is reused from a CallImportRow recording), or 'upload' (audio was + # uploaded by the user). Non-tts samples are marked completed up-front + # by the API and skipped by the generation worker. + source_type = Column(String(32), nullable=False, default="tts") + # When source_type == 'recording', references CallImportRow.id (no FK + # constraint to keep cascading deletes simple if a call import is later + # removed; the audio_s3_key is what's actually used). + source_ref_id = Column(UUID(as_uuid=True), nullable=True) + + text = Column(String, nullable=False) + audio_s3_key = Column(String(512), nullable=True) + duration_seconds = Column(Float, nullable=True) + latency_ms = Column(Float, nullable=True) + ttfb_ms = Column(Float, nullable=True) + + evaluation_metrics = Column(JSON, nullable=True) + status = Column(String(50), nullable=False, default=TTSSampleStatus.PENDING.value) + error_message = Column(String, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + comparison = relationship("TTSComparison", back_populates="samples") + + +class TTSReportJob(Base): + """Asynchronous PDF report generation jobs for Voice Playground.""" + __tablename__ = "tts_report_jobs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: mirrors the parent TTSComparison's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) + + status = Column(String(50), nullable=False, default=TTSReportJobStatus.PENDING.value) + format = Column(String(20), nullable=False, default="pdf") + filename = Column(String(255), nullable=True) + s3_key = Column(String(512), nullable=True) + error_message = Column(String, nullable=True) + celery_task_id = Column(String, nullable=True, index=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + comparison = relationship("TTSComparison") + + +class TTSBlindTestShareStatus(str, enum.Enum): + OPEN = "open" + CLOSED = "closed" + + +class TTSBlindTestShare(Base): + """A publicly sharable blind test for a TTSComparison. + + The share_token is the capability: anyone holding it can open the public + form and submit a response. Each comparison has at most one share row. + """ + __tablename__ = "tts_blind_test_shares" + __table_args__ = ( + UniqueConstraint("comparison_id", name="uq_blind_test_shares_comparison"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + comparison_id = Column( + UUID(as_uuid=True), + ForeignKey("tts_comparisons.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: mirrors the parent TTSComparison's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + share_token = Column(String(64), unique=True, nullable=False, index=True) + + title = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + + # Internal notes visible only to the share creator (e.g. which voice + # corresponds to which side, source notes for standalone blind tests). + # Never exposed via the public blind test payload. + creator_notes = Column(Text, nullable=True) + + # JSON list: [{ "key": str, "label": str, "type": "rating"|"comment", "scale": int? }] + custom_metrics = Column(JSON, nullable=False) + + status = Column(String(20), nullable=False, default=TTSBlindTestShareStatus.OPEN.value) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + closed_at = Column(DateTime(timezone=True), nullable=True) + created_by = Column(String, nullable=True) + + comparison = relationship("TTSComparison") + responses = relationship( + "TTSBlindTestResponse", + back_populates="share", + cascade="all, delete-orphan", + ) + + +class TTSBlindTestResponse(Base): + """A single rater's submission against a TTSBlindTestShare.""" + __tablename__ = "tts_blind_test_responses" + __table_args__ = ( + UniqueConstraint("share_id", "rater_email", name="uq_blind_test_response_share_email"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + share_id = Column( + UUID(as_uuid=True), + ForeignKey("tts_blind_test_shares.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + # Workspace isolation: mirrors the parent TTSBlindTestShare's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + rater_name = Column(String(255), nullable=False) + rater_email = Column(String(320), nullable=False, index=True) + + # JSON list keyed by sample_index. Server stores in TRUE A/B orientation + # (already de-flipped from whatever the rater's UI showed): + # [{ + # "sample_index": int, + # "preferred": "A" | "B", + # "ratings_a": { metric_key: number }, + # "ratings_b": { metric_key: number }, + # "comment": str? + # }] + responses = Column(JSON, nullable=False) + + ip = Column(String(64), nullable=True) + user_agent = Column(String(512), nullable=True) + + submitted_at = Column(DateTime(timezone=True), server_default=func.now()) + + share = relationship("TTSBlindTestShare", back_populates="responses") + + +class PromptPartial(Base): + """Prompt Partial - Reusable prompt templates with version history.""" + __tablename__ = "prompt_partials" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every prompt partial belongs to a workspace + # within its org. Versions inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + description = Column(String, nullable=True) + content = Column(Text, nullable=False) + tags = Column(JSON, nullable=True) + current_version = Column(Integer, nullable=False, default=1) + # Cached LLM-generated flowchart for imported production agent prompts. + # Shape: AgentFlowGraph JSON (nodes[], edges[]). + agent_flowchart = Column(JSON, nullable=True) + agent_flowchart_status = Column(String(20), nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + versions = relationship("PromptPartialVersion", back_populates="prompt_partial", cascade="all, delete-orphan", order_by="PromptPartialVersion.version.desc()") + + +class PromptPartialVersion(Base): + """Version history for a prompt partial.""" + __tablename__ = "prompt_partial_versions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + prompt_partial_id = Column(UUID(as_uuid=True), ForeignKey("prompt_partials.id", ondelete="CASCADE"), nullable=False, index=True) + # Workspace isolation: mirrors the parent PromptPartial's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + version = Column(Integer, nullable=False) + content = Column(Text, nullable=False) + change_summary = Column(String, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + created_by = Column(String, nullable=True) + + prompt_partial = relationship("PromptPartial", back_populates="versions") + + __table_args__ = ( + UniqueConstraint('prompt_partial_id', 'version', name='uq_prompt_partial_version'), + ) + + +class CustomTTSVoice(Base): + """Organization-scoped custom TTS voice metadata.""" + __tablename__ = "custom_tts_voices" + __table_args__ = ( + UniqueConstraint("organization_id", "provider", "voice_id", name="uq_custom_tts_voice_org_provider_voice_id"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider = Column(String(100), nullable=False, index=True) + voice_id = Column(String(255), nullable=False) + name = Column(String(255), nullable=False) + gender = Column(String(50), nullable=True) + accent = Column(String(100), nullable=True) + description = Column(Text, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + +class PromptOptimizationRun(Base): + """A single GEPA prompt optimization run for an agent.""" + __tablename__ = "prompt_optimization_runs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every optimization run belongs to a workspace + # within its org. Candidates inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) + evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True) + voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) + + seed_prompt = Column(Text, nullable=False) + best_prompt = Column(Text, nullable=True) + best_score = Column(Float, nullable=True) + + status = Column(String(20), nullable=False, default=PromptOptimizationStatus.PENDING.value) + config = Column(JSON, nullable=True) + reflection_trace = Column(JSON, nullable=True) + metric_history = Column(JSON, nullable=True) + + num_iterations = Column(Integer, nullable=True) + num_metric_calls = Column(Integer, nullable=True) + + celery_task_id = Column(String, nullable=True, index=True) + error_message = Column(Text, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + candidates = relationship("PromptOptimizationCandidate", back_populates="optimization_run", cascade="all, delete-orphan") + + +class PromptOptimizationCandidate(Base): + """A candidate prompt generated during an optimization run.""" + __tablename__ = "prompt_optimization_candidates" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + optimization_run_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_runs.id", ondelete="CASCADE"), nullable=False, index=True) + # Workspace isolation: mirrors the parent PromptOptimizationRun's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + prompt_text = Column(Text, nullable=False) + score = Column(Float, nullable=True) + metric_breakdown = Column(JSON, nullable=True) + reflection_summary = Column(Text, nullable=True) + + parent_candidate_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_candidates.id"), nullable=True) + + is_accepted = Column(Boolean, nullable=False, default=False) + pushed_to_provider_at = Column(DateTime(timezone=True), nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + optimization_run = relationship("PromptOptimizationRun", back_populates="candidates") + + +class TelephonyIntegration(Base): + """Per-organization telephony provider credentials and configuration. + + Multiple rows per (organization_id, provider) are allowed so that an + organization can keep several Plivo / Exotel accounts side-by-side. + A partial unique index in migration 028 enforces at most one row with + is_default = TRUE per (org, provider); resolution falls back to that + default row when the caller does not pin a specific credential. + """ + + __tablename__ = "telephony_integrations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider = Column(String(50), nullable=False, default="plivo") + name = Column(String(255), nullable=True) # Optional friendly name to disambiguate multiple credentials + + auth_id = Column(String(255), nullable=False) + auth_token = Column(String(512), nullable=False) + + verify_app_uuid = Column(String(255), nullable=True) + voice_app_id = Column(String(255), nullable=True) + sip_domain = Column(String(255), nullable=True) + masking_config = Column(JSON, nullable=True) + + is_active = Column(Boolean, default=True, nullable=False) + is_default = Column(Boolean, default=False, nullable=False) + last_tested_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyPhoneNumber(Base): + """Inventory of telephony phone numbers owned by an organization.""" + + __tablename__ = "telephony_phone_numbers" + __table_args__ = ( + UniqueConstraint("organization_id", "phone_number", name="uq_telephony_number_org_phone"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + telephony_integration_id = Column( + UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=True, index=True + ) + + phone_number = Column(String(20), nullable=False, index=True) + country_iso2 = Column(String(2), nullable=True) + region = Column(String(100), nullable=True) + number_type = Column(String(20), nullable=True) + capabilities = Column(JSON, nullable=True) + provider_app_id = Column(String(255), nullable=True) + + is_masking_pool = Column(Boolean, default=False, nullable=False) + inbound_enabled = Column(Boolean, default=True, nullable=False) + outbound_enabled = Column(Boolean, default=True, nullable=False) + source = Column(String(20), nullable=False, default="imported") + agent_id = Column( + UUID(as_uuid=True), + ForeignKey( + "agents.id", + ondelete="SET NULL", + use_alter=True, + name="fk_telephony_phone_numbers_agent_id", + ), + nullable=True, + index=True, + ) + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyDialTarget(Base): + """Org-scoped saved destination numbers for outbound test calls.""" + + __tablename__ = "telephony_dial_targets" + __table_args__ = ( + UniqueConstraint("organization_id", "phone_number", name="uq_telephony_dial_target_org_phone"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + phone_number = Column(String(20), nullable=False, index=True) + label = Column(String(255), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyVerifySession(Base): + """Tracks voice OTP verification sessions via telephony provider.""" + + __tablename__ = "telephony_verify_sessions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider_session_uuid = Column(String(255), nullable=False, unique=True, index=True) + recipient_number = Column(String(20), nullable=False) + channel = Column(String(10), nullable=False, default="voice") + status = Column(String(20), nullable=False, default="pending") + initiated_by = Column(String(255), nullable=True) + verify_app_uuid = Column(String(255), nullable=True) + verified_at = Column(DateTime(timezone=True), nullable=True) + expires_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyMaskedSession(Base): + """Number-masking session between two parties through a middle number.""" + + __tablename__ = "telephony_masked_sessions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + telephony_integration_id = Column(UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=False) + masked_number_id = Column( + UUID(as_uuid=True), ForeignKey("telephony_phone_numbers.id"), nullable=False, index=True + ) + masked_number = Column(String(20), nullable=False) + party_a_number = Column(String(20), nullable=False) + party_b_number = Column(String(20), nullable=False) + status = Column(String(20), nullable=False, default="active") + expires_at = Column(DateTime(timezone=True), nullable=True) + ended_at = Column(DateTime(timezone=True), nullable=True) + session_metadata = Column("metadata", JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class CallImportSchema(Base): + """Reusable Input Parameter schema for the call-uploads flow. + + A schema is workspace-scoped: users define a named bundle of typed + Input Parameters once (e.g. "Standard Voice QA" with conversation_id + + recording_url + transcript + agent_name) and then map those parameters + to CSV/Excel headers each time they upload a new batch. + + Every schema MUST contain exactly one parameter with + ``type='conversation_id'`` and ``is_required=True`` - that's the + mandatory identity field every imported row needs. A schema may + optionally include at most one ``recording_url`` parameter. The + invariant is enforced in app code on create/update (no DB-level + CHECK because the parent + children are written across two tables in + one transaction). + """ + + __tablename__ = "call_import_schemas" + __table_args__ = ( + # Case-insensitive uniqueness is enforced via the matching partial + # index on ``LOWER(name)`` in the migration; this constraint here + # would be case-sensitive and is intentionally omitted to avoid + # confusing the user. + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + created_by_user_id = Column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + parameters = relationship( + "CallImportSchemaParameter", + back_populates="schema", + cascade="all, delete-orphan", + order_by="CallImportSchemaParameter.ordering", + ) + + +class CallImportSchemaParameter(Base): + """A single typed parameter inside a :class:`CallImportSchema`. + + ``type`` is one of the strings tracked by + :data:`app.models.enums.CallImportParameterType`. ``conversation_id`` + is reserved for the mandatory identity parameter every schema must + contain. + """ + + __tablename__ = "call_import_schema_parameters" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + schema_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_schemas.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + type = Column(String(32), nullable=False) + description = Column(Text, nullable=True) + is_required = Column(Boolean, nullable=False, default=False) + # Stable ordering so the UI renders parameters in the order the + # schema author defined them (matters when conversation_id is pinned + # first and the user re-orders the rest). + ordering = Column(Integer, nullable=False, default=0) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + schema = relationship("CallImportSchema", back_populates="parameters") + + +class CallImport(Base): + """Batch record for a CSV-driven call import job.""" + + __tablename__ = "call_imports" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every imported batch belongs to a workspace + # within its org. The /upload endpoint stamps it from the active + # workspace header (or the org's Default if absent). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + created_by_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + last_updated_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + + # Telephony provider key (e.g. ``'exotel'``, ``'plivo'``). In the + # legacy one-shot ``POST /upload`` endpoint this is supplied with the + # file; in the three-stage flow (UPLOAD -> MAP -> IMPORT) the value + # isn't known until the IMPORT stage, so the column is nullable for + # ``uploaded`` / ``mapped`` batches. + provider = Column(String(50), nullable=True, default="exotel") + # Pin a specific telephony credential for this batch so the worker + # downloads recordings using *that* row instead of the org default. + # NULL preserves legacy behavior (resolve by provider + default). + telephony_integration_id = Column( + UUID(as_uuid=True), + ForeignKey("telephony_integrations.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + original_filename = Column(String(512), nullable=True) + # When the source file was a multi-sheet Excel workbook, this records + # the worksheet the rows came from (one batch per sheet). NULL for CSV + # uploads since CSV has no sheet concept. + sheet_name = Column(String(255), nullable=True) + + # --- Source-file staging (UPLOAD stage) --------------------------- + # The raw CSV / Excel file is stored in S3 between stages so the + # user can come back later to MAP and IMPORT without re-uploading. + # ``source_s3_key`` is NULL on legacy batches that were imported via + # the one-shot endpoint (those batches stay read-only post-import). + source_s3_key = Column(Text, nullable=True) + source_format = Column(String(16), nullable=True) + source_size_bytes = Column(BigInteger, nullable=True) + source_content_type = Column(String(255), nullable=True) + + # Snapshot of the file's sheets + headers captured at UPLOAD time + # so the MAP UI doesn't need to re-fetch the source bytes from S3. + # Shape: ``[{"name": str, "headers": [str, ...], "row_count": int}, ...]``. + available_sheets = Column(JSON, nullable=True) + + # User's explicit "drop these columns" decision captured at MAP + # time. Was validation-only and ephemeral in the legacy flow; now + # persisted so the IMPORT stage can re-parse the file with the same + # mapping/skip intent. + skipped_columns = Column(JSON, nullable=False, default=list) + # Rows skipped at parse time (missing/invalid conversation_id or URL). + # Shape: ``[{"source_row": int, "reason": str, "message": str}, ...]``. + source_row_skips = Column(JSON, nullable=False, default=list) + + # Free-text high-level segregation label. Powers the "Dataset" filter + # at the top of the imports page; multiple imports can share a value. + dataset = Column(String(255), nullable=True, index=True) + + # Reusable Input Parameter schema this batch was uploaded against. + # NULL on legacy batches uploaded before the schema-driven flow + # shipped; those still render via ``column_mapping`` + ``extra_columns`` + # + ``custom_column_mapping`` below. + schema_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_schemas.id", ondelete="RESTRICT"), + nullable=True, + index=True, + ) + # New schema-driven mapping: ``{schema_parameter_name: csv_header}``. + # Populated for new uploads; empty dict on legacy batches. + parameter_mapping = Column(JSON, nullable=False, default=dict) + + # Legacy free-form mapping (pre-schema-flow). Kept on the model so + # batches that were uploaded before the schema feature shipped still + # render correctly on the detail page; new uploads stop writing here. + # Keys: external_call_id (required), transcript, recording_url. + # (DB column ``external_call_id`` is now ``conversation_id``; this + # JSON key stays as-is for historical batches.) + # Values: original CSV header strings (preserve user casing for export). + column_mapping = Column(JSON, nullable=False, default=dict) + # Ordered list of additional CSV header strings the uploader wants + # preserved verbatim into the evaluation export CSV. + extra_columns = Column(JSON, nullable=False, default=list) + # User-defined ``{custom_field_name: csv_header}`` mappings on top of + # the three system fields above. Cells from the mapped CSV columns are + # preserved per row (keyed by the CSV header in ``raw_columns``) and + # surface in the evaluation export under the uploader-chosen name. + custom_column_mapping = Column(JSON, nullable=False, default=dict) + + total_rows = Column(Integer, nullable=False, default=0) + completed_rows = Column(Integer, nullable=False, default=0) + failed_rows = Column(Integer, nullable=False, default=0) + + status = Column( + Enum(CallImportStatus, values_callable=get_enum_values), + nullable=False, + default=CallImportStatus.PENDING, + index=True, + ) + error_message = Column(Text, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + rows = relationship( + "CallImportRow", + back_populates="call_import", + cascade="all, delete-orphan", + order_by="CallImportRow.row_index", + ) + tags = relationship( + "CallImportTag", + secondary="call_import_tag_assignments", + backref="call_imports", + lazy="selectin", + ) + evaluations = relationship( + "CallImportEvaluation", + back_populates="call_import", + cascade="all, delete-orphan", + ) + + +class CallImportShardSlice(Base): + """Registry row: which shard stores a slice of rows for an import.""" + + __tablename__ = "call_import_shard_slices" + + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + primary_key=True, + ) + slice_id = Column(Integer, primary_key=True) + shard_id = Column(String(64), nullable=False, index=True) + row_index_min = Column(Integer, nullable=False) + row_index_max = Column(Integer, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class CallImportRow(Base): + """A single row within a CallImport batch (one CSV line / one external call).""" + + __tablename__ = "call_import_rows" + __table_args__ = ( + UniqueConstraint("call_import_id", "row_index", name="uq_call_import_row_index"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + row_index = Column(Integer, nullable=False) + # Was historically named ``external_call_id``; renamed to + # ``conversation_id`` so the new schema-driven upload flow can refer + # to it by a single canonical name across the schema definition, + # exports, and downstream evaluation tables. + conversation_id = Column(String(255), nullable=False, index=True) + # Supplied via CSV for Exotel credentialed imports (required per row). + # Nullable in the schema for legacy rows imported before recording_url + # was mandatory on every Exotel upload. + recording_url = Column(Text, nullable=True) + # Date-only call recording date supplied by the import schema. Used + # for historical report comparisons without timezone/time ambiguity. + recording_date = Column(Date, nullable=True, index=True) + # The "production" transcript: the value supplied via the CSV + # upload mapping. Never overwritten by the diarisation worker — + # the worker writes its output into ``diarised_transcript`` so + # the user keeps both versions side by side. + transcript = Column(Text, nullable=True) + # Snapshot of the original CSV row keyed by the user's headers so the + # evaluation export can reproduce every column the uploader supplied + # (mapped + extra). NULL on legacy rows imported before this column. + raw_columns = Column(JSON, nullable=True) + + # Where the value in ``transcript`` came from. ``csv`` = supplied via + # the upload mapping, ``edited`` = manually changed in the UI. NULL + # on rows that have never had a production transcript. + # (Worker-produced transcripts now live in ``diarised_transcript`` + # and are tracked via ``diarised_transcript_*`` metadata below.) + transcript_source = Column(String(20), nullable=True) + # Provider/model recorded by the (legacy) post-hoc transcription + # worker. New worker runs leave these NULL and write into the + # ``diarised_transcript_*`` columns instead; kept on the model for + # backwards compatibility with pre-split rows that still carry the + # original transcription metadata here. + transcript_provider = Column(String(50), nullable=True) + transcript_model = Column(String(100), nullable=True) + # Lifecycle status for the legacy transcription workflow itself, + # independent of the row's recording-fetch ``status``. ``idle`` = + # no transcribe task has touched this column. New diarisation runs + # update ``diarised_transcript_status`` instead. + transcript_status = Column( + String(20), + nullable=False, + default="idle", + ) + transcript_error = Column(Text, nullable=True) + transcribed_at = Column(DateTime(timezone=True), nullable=True) + + # The "diarised" transcript: produced by the post-hoc + # transcription/diarisation worker. Stored separately so a manual + # diarisation run never clobbers the production transcript above. + # Evaluations can be configured to score against either column + # (see ``CallImportEvaluation.transcript_source``). + diarised_transcript = Column(Text, nullable=True) + # Provider/model the diarisation worker used. Surfaced in the UI + # as "Diarised via deepgram/nova-2" next to the diarised + # transcript section. + diarised_transcript_provider = Column(String(50), nullable=True) + diarised_transcript_model = Column(String(100), nullable=True) + # Lifecycle status for the diarisation workflow. + # ``idle`` = no diarisation task has run; ``pending``/``running`` = + # a Celery task is queued or in flight; ``completed``/``failed`` = + # terminal. Independent of ``transcript_status`` so the two + # transcripts can be in different lifecycle states. + diarised_transcript_status = Column( + String(20), + nullable=False, + default="idle", + server_default="idle", + ) + diarised_transcript_error = Column(Text, nullable=True) + diarised_at = Column(DateTime(timezone=True), nullable=True) + + # Structured speaker turns produced by the diarisation worker — + # ``[{ "speaker": "agent"|"user"|"speaker_3", "text": "...", + # "start": float, "end": float, "raw_speaker": "Speaker 1" }, ...]`` + # The plain-text ``diarised_transcript`` above is a rendered view + # of this list (``: `` per line). When the worker + # cannot recover structured turns (no pyannote token / single- + # speaker recording / provider that doesn't surface segments) this + # column stays NULL and the plain-text path is still populated. + diarised_segments = Column(JSON, nullable=True) + # When True the ``agent`` <-> ``user`` mapping inside + # ``diarised_segments`` is inverted at render / export time. The + # worker writes the canonical mapping using the "first speaker is + # the agent" heuristic; reviewers can flip the toggle from the row + # detail panel without re-running diarisation. + diarised_speaker_swap = Column( + Boolean, + nullable=False, + default=False, + server_default="false", + ) + # LLM that turned the STT plain-text output into structured + # ``diarised_segments``. The legacy diarisation worker used + # pyannote and left these NULL; the current path always runs an + # LLM with the operator-supplied (or default) ``diarised_prompt`` + # below, and records exactly which model + prompt produced each + # row so reviewers can reproduce a specific run. + diarised_llm_provider = Column(String(50), nullable=True) + diarised_llm_model = Column(String(100), nullable=True) + diarised_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) + diarised_prompt = Column(Text, nullable=True) + # Which diarisation pipeline produced this row's turns. + # * ``"stt_llm"`` (default) — two-stage: STT then LLM diariser. + # ``diarised_transcript_provider``/``_model`` describe the STT + # side; ``diarised_llm_provider``/``_model`` the LLM side. + # * ``"llm_only"`` — single-stage: audio fed straight to a + # multimodal LLM. ``diarised_transcript_provider`` is stamped + # with the sentinel ``"llm_only"``; the real model is on + # ``diarised_llm_*``. + # Persisting it on the row (not just the run) lets the row detail + # panel render the right "Diarised via …" label even for ad-hoc + # standalone transcribes (no parent evaluation). + transcribe_mode = Column( + String(20), + nullable=False, + default="stt_llm", + server_default="stt_llm", + ) + + status = Column( + Enum(CallImportRowStatus, values_callable=get_enum_values), + nullable=False, + default=CallImportRowStatus.PENDING, + index=True, + ) + + recording_s3_key = Column(String(1024), nullable=True) + recording_content_type = Column(String(128), nullable=True) + recording_size_bytes = Column(Integer, nullable=True) + + error_message = Column(Text, nullable=True) + attempts = Column(Integer, nullable=False, default=0) + celery_task_id = Column(String(255), nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + call_import = relationship("CallImport", back_populates="rows") + + +@event.listens_for(CallImportRow, "before_insert") +def _call_import_row_fill_workspace_id(_mapper, connection, target): + """Denormalize workspace_id from the parent import when omitted.""" + if target.workspace_id is not None or target.call_import_id is None: + return + workspace_id = connection.execute( + select(CallImport.workspace_id).where( + CallImport.id == target.call_import_id + ) + ).scalar_one_or_none() + if workspace_id is not None: + target.workspace_id = workspace_id + + +class CallImportTag(Base): + """User-defined tag that can be attached to one or more call imports. + + Tags coexist with the free-text ``CallImport.dataset`` column: dataset + is the primary high-level segregation, tags are an optional secondary + classification (an import can have many tags). + """ + + __tablename__ = "call_import_tags" + __table_args__ = ( + UniqueConstraint("organization_id", "name", name="uq_call_import_tag_org_name"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + name = Column(String(255), nullable=False) + color = Column(String(32), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +class CallImportTagAssignment(Base): + """Many-to-many join table between CallImport and CallImportTag.""" + + __tablename__ = "call_import_tag_assignments" + + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + primary_key=True, + ) + tag_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_tags.id", ondelete="CASCADE"), + primary_key=True, + index=True, + ) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class CallImportEvaluation(Base): + """Parent record for an evaluation run over a CallImport batch. + + A user picks a subset of org ``Metric`` rows and triggers an evaluation; + we fan out one ``CallImportEvaluationRow`` per source row and roll up + counters as workers finish. Status mirrors ``CallImportStatus`` plus a + ``RUNNING`` value so the UI can distinguish "queued" from "in flight". + """ + + __tablename__ = "call_import_evaluations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id"), + nullable=False, + index=True, + ) + # Workspace isolation: mirrors the parent CallImport's workspace. + # Denormalized for fast filter-by-workspace listings without a join. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + created_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id"), nullable=True + ) + last_updated_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + + # Optional user-supplied label for this run. Lets the UI surface + # something more meaningful than the UUID prefix (e.g. "March QA pass"). + name = Column(String(255), nullable=True) + + # JSON list of Metric UUID strings selected for this run. Stored as text + # in JSON so we don't have to deal with PG arrays of UUIDs / cascade + # delete policies when metrics are removed; the loader filters for + # still-existing org metrics at run time. + selected_metric_ids = Column(JSON, nullable=False, default=list) + # Hierarchy grouping snapshot: ``{parent_id_str: [child_id_str, ...]}``. + # Captures which children belong to which parent for THIS run so the UI + # / aggregator can reconstruct the tree even when the user selected + # only a subset of children, or after metrics are deleted / renamed. + # NULL on legacy rows means "no hierarchy" → fall back to flat + # ``selected_metric_ids`` semantics. + selected_metric_groups = Column(JSON, nullable=True) + # User-driven merges of LLM-discovered candidate sub-labels for + # ``allow_discovery`` parents. Shape: + # ``{"": {"": "", ...}}``. + # Populated via ``POST .../discovered-labels/merge``; consulted by + # the discovered-labels aggregator, the flow graph builder, and the + # worker so that rows finishing AFTER a merge cannot reintroduce + # the merged-away slug. Empty dict on fresh rows. + discovered_label_aliases = Column( + JSON, nullable=False, default=dict, server_default="{}" + ) + + # Per-run opt-in for top-level metric discovery. When True, the LLM + # is asked to propose brand-new top-level metrics (boolean / rating / + # category) observed in the transcripts in addition to scoring the + # ``selected_metric_ids`` for the row. Candidates surface in a + # "Discovered metrics" panel on the evaluation's Flow tab and can + # be promoted into real standalone ``Metric`` rows via + # ``POST /metrics/from-discovered``. Defaults to False so existing + # evaluation creation payloads keep their previous behaviour. + discover_new_metrics = Column( + Boolean, nullable=False, default=False, server_default="false" + ) + # Flat slug-to-slug redirect map for user merges + tombstones of + # discovered top-level metric candidates. Mirrors + # ``discovered_label_aliases`` but is NOT nested per parent — + # top-level metric discovery is not scoped to any parent. Shape:: + # + # {"": "", ...} + # + # An empty-string value tombstones the slug so workers finishing + # later can't re-introduce it. + discovered_metric_aliases = Column( + JSON, nullable=False, default=dict, server_default="{}" + ) + + # Run-level LLM config picked from the Run Evaluation modal. NULL on + # legacy rows means "use the historical OpenAI/gpt-4o default" — the + # worker checks for this and falls back accordingly. ``llm_credential_id`` + # pins a specific AIProvider row when the org has multiple credentials + # for the same provider. + llm_provider = Column(String(50), nullable=True) + llm_model = Column(String(100), nullable=True) + llm_credential_id = Column( + UUID(as_uuid=True), + ForeignKey("aiproviders.id", ondelete="SET NULL"), + nullable=True, + ) + llm_config = Column(JSON, nullable=True) + # Optional per-metric LLM override: + # ``{"": {"provider": "...", "model": "...", "credential_id": "..."}}``. + # Each entry overrides the run-level default for that metric only; + # missing keys = use run-level default. Stored as JSON so the UI can + # round-trip arbitrary {provider, model} pairs without migrations. + metric_llm_overrides = Column(JSON, nullable=True) + + # When ``auto_transcribe`` was set on the create payload, record the + # STT provider/model used so the UI can show "Auto-transcribed via + # deepgram/nova-2" on the evaluation header. ``stt_credential_id`` is + # untyped (no FK) because STT keys may live in either ``aiproviders`` + # (OpenAI) or ``integrations`` (Deepgram, ElevenLabs) — the + # transcription service handles the lookup. + stt_provider = Column(String(50), nullable=True) + stt_model = Column(String(100), nullable=True) + stt_credential_id = Column(UUID(as_uuid=True), nullable=True) + + # Run-level LLM diariser config. Used when the create-run / + # retry-run paths chain a ``transcribe_call_import_row_task`` + # because the row is missing a diarised transcript. Persisted on + # the run so a retry uses the same diariser the original create + # call picked (unless the retry payload explicitly overrides). + diarisation_llm_provider = Column(String(50), nullable=True) + diarisation_llm_model = Column(String(100), nullable=True) + diarisation_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) + diarisation_prompt = Column(Text, nullable=True) + # Mode the run was *created* with for its auto-transcribe step. + # Retry chains read this to decide whether to enqueue an STT+LLM + # transcribe or a single-stage multimodal LLM transcribe — without + # it we'd have to infer the mode from "stt_provider is NULL", which + # would silently break legacy rows that simply never configured + # auto-transcribe. See migration 041 for the column DDL. + transcribe_mode = Column( + String(20), + nullable=False, + default="stt_llm", + server_default="stt_llm", + ) + + # Which of the two transcripts on each ``CallImportRow`` this run + # scored against. ``'production'`` reads ``CallImportRow.transcript`` + # (the CSV-supplied value); ``'diarised'`` reads + # ``CallImportRow.diarised_transcript`` (the worker output). When + # the user ticks both checkboxes in the Run Evaluation modal we + # create two ``CallImportEvaluation`` rows — one per source — so + # the two scorings can be compared side-by-side. Defaults to + # ``'production'`` so legacy runs (which always read the single + # historical ``transcript`` column) keep their semantics. + transcript_source = Column( + String(20), + nullable=False, + default="production", + server_default="production", + ) + + # Cached LLM-generated TLDR rendered above the Visualizations charts. + # Populated lazily by ``POST /evaluations/{eval_id}/insights`` so we + # never auto-burn LLM tokens on page load. Shape:: + # {"narrative": str, "patterns": [str, ...], + # "generated_at": iso8601, "generated_at_completed_rows": int, + # "provider": str, "model": str} + # NULL on rows that have never been summarised. + tldr_summary = Column(JSON, nullable=True) + + # Cached LLM-generated user insights for External Audit PDF section 03. + # Populated by a background Celery job triggered alongside TLDR generation. + # Shape: EvaluationUserInsightsState JSON (status, insights[], progress, …). + user_insights = Column(JSON, nullable=True) + + # Cached per-metric failure clustering for internal diagnostics PDF/UI. + # Shape: EvaluationMetricClustersState JSON (status, groups[], …). + metric_clusters = Column(JSON, nullable=True) + + # Cached LLM-generated prompt improvement suggestions keyed to an + # imported agent (PromptPartial tagged __imported_agent__). + # Shape: EvaluationPromptImprovementsState JSON. + prompt_improvements = Column(JSON, nullable=True) + + # Cached LLM explanations for week-over-week metric deltas keyed by + # baseline evaluation id + completed row counts. + period_delta_explanations = Column(JSON, nullable=True) + + status = Column(String(20), nullable=False, default="pending", index=True) + + total_rows = Column(Integer, nullable=False, default=0) + completed_rows = Column(Integer, nullable=False, default=0) + failed_rows = Column(Integer, nullable=False, default=0) + # Flexprice pass-level delta billing watermark: rows already emitted + # on ``call_import.evaluation_completed`` for this evaluation run. + billed_completed_rows = Column( + Integer, nullable=False, default=0, server_default="0" + ) + error_message = Column(Text, nullable=True) + celery_group_id = Column(String(255), nullable=True) + + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + call_import = relationship("CallImport", back_populates="evaluations") + row_results = relationship( + "CallImportEvaluationRow", + back_populates="evaluation", + cascade="all, delete-orphan", + ) + + +class CallImportEvaluationRow(Base): + """Per-source-row scoring output for a CallImportEvaluation parent.""" + + __tablename__ = "call_import_evaluation_rows" + __table_args__ = ( + UniqueConstraint( + "evaluation_id", "call_import_row_id", name="uq_call_import_evaluation_row" + ), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_import_row_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_rows.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + status = Column(String(20), nullable=False, default="pending", index=True) + # Same shape as EvaluatorResult.metric_scores: {metric_id_str: {value, type, metric_name, ...}} + metric_scores = Column(JSON, nullable=False, default=dict) + error_message = Column(Text, nullable=True) + celery_task_id = Column(String(255), nullable=True) + + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + evaluation = relationship("CallImportEvaluation", back_populates="row_results") + source_row = relationship("CallImportRow") + + +@event.listens_for(CallImportEvaluationRow, "before_insert") +def _call_import_evaluation_row_fill_workspace_id(_mapper, connection, target): + """Denormalize workspace_id from the parent evaluation when omitted.""" + if target.workspace_id is not None or target.evaluation_id is None: + return + workspace_id = connection.execute( + select(CallImportEvaluation.workspace_id).where( + CallImportEvaluation.id == target.evaluation_id + ) + ).scalar_one_or_none() + if workspace_id is not None: + target.workspace_id = workspace_id + + +class MetricStudioRun(Base): + """Ad-hoc metric experiment run in Metrics Studio.""" + + __tablename__ = "metric_studio_runs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + created_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id"), nullable=True + ) + + name = Column(String(255), nullable=True) + selected_metric_ids = Column(JSON, nullable=False, default=list) + selected_metric_groups = Column(JSON, nullable=True) + transcript_source = Column( + String(20), + nullable=False, + default="diarised", + server_default="diarised", + ) + + llm_provider = Column(String(50), nullable=True) + llm_model = Column(String(100), nullable=True) + llm_credential_id = Column( + UUID(as_uuid=True), + ForeignKey("aiproviders.id", ondelete="SET NULL"), + nullable=True, + ) + llm_config = Column(JSON, nullable=True) + metric_llm_overrides = Column(JSON, nullable=True) + + status = Column(String(20), nullable=False, default="pending", index=True) + total_items = Column(Integer, nullable=False, default=0) + completed_items = Column(Integer, nullable=False, default=0) + failed_items = Column(Integer, nullable=False, default=0) + error_message = Column(Text, nullable=True) + + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + results = relationship( + "MetricStudioRunResult", + back_populates="run", + cascade="all, delete-orphan", + ) + + +class MetricStudioRunResult(Base): + """Per-source scoring output for a MetricStudioRun.""" + + __tablename__ = "metric_studio_run_results" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + run_id = Column( + UUID(as_uuid=True), + ForeignKey("metric_studio_runs.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + source_kind = Column(String(40), nullable=False) + source_ref = Column(String(255), nullable=False) + display_label = Column(String(512), nullable=True) + source_metadata = Column(JSON, nullable=True) + + status = Column(String(20), nullable=False, default="pending", index=True) + metric_scores = Column(JSON, nullable=False, default=dict) + error_message = Column(Text, nullable=True) + celery_task_id = Column(String(255), nullable=True) + + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + run = relationship("MetricStudioRun", back_populates="results") + + +class CallImportEvaluationReportSnapshot(Base): + """Persisted PDF-report aggregate used for period-over-period deltas.""" + + __tablename__ = "call_import_evaluation_report_snapshots" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + period_label = Column(String(64), nullable=True, index=True) + period_start = Column(Date, nullable=True, index=True) + period_end = Column(Date, nullable=True, index=True) + report_config = Column(JSON, nullable=False, default=dict, server_default="{}") + selected_metric_ids = Column(JSON, nullable=False, default=list, server_default="[]") + metric_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") + insight_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") + narrative = Column(JSON, nullable=True) + total_calls = Column(Integer, nullable=False, default=0) + selected_metric_count = Column(Integer, nullable=False, default=0) + total_metric_count = Column(Integer, nullable=False, default=0) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +class CallImportEvaluationPdfReport(Base): + """Stored PDF artifact for a call import evaluation report generation.""" + + __tablename__ = "call_import_evaluation_pdf_reports" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + snapshot_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluation_report_snapshots.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + vendor_name = Column(String(120), nullable=False) + report_type = Column(String(20), nullable=False, default="external") + filename = Column(String(255), nullable=True) + s3_key = Column(String(512), nullable=True) + report_config = Column(JSON, nullable=False, default=dict, server_default="{}") + cache_fingerprint = Column(String(64), nullable=True) + created_by = Column(String, nullable=True) + created_by_user_id = Column(UUID(as_uuid=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +# --------------------------------------------------------------------------- +# Judge Alignment (AlignEval-style hybrid integration) +# +# Three tables back the "Judge Alignment" surface: +# - JudgeDataset: a labeled dataset materialised from one of three sources +# (voice transcripts, existing Metric/Evaluator outputs, +# or a generic CSV upload). Holds the dataset's source +# config + which fields play the role of input/output. +# - JudgeSample: one row in a dataset (input/output pair plus an +# optional binary pass/fail human label). +# - JudgeRun: a single run of an LLM-judge (existing Evaluator) over +# a subset of samples, with computed alignment metrics +# (precision/recall/F1/Cohen's kappa) and per-sample +# predictions. Optionally links to a GEPA optimization +# run when the user kicks off prompt tuning from a +# dataset. +# --------------------------------------------------------------------------- + + +class JudgeDataset(Base): + """Container for binary-labeled samples used to calibrate an LLM-judge.""" + + __tablename__ = "judge_datasets" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + # Workspace isolation: every judge dataset belongs to a workspace + # within its org. Samples and runs inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + name = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + + # One of: "transcript", "metric_output", "csv" + source_type = Column(String(32), nullable=False, index=True) + # Source-specific config. Examples: + # transcript: {"transcription_ids": [...]} or {"agent_id": "..."} + # metric_output: {"metric_id": "...", "evaluator_id": "..."} + # csv: {"s3_key": "...", "filename": "..."} + source_config = Column(JSON, nullable=False, default=dict) + + # Field roles - which textual content is "input" vs "output" for the judge. + # For voice transcripts both default to the transcript text but can be + # tightened (e.g. agent-only turns vs full conversation). + input_field = Column(String(64), nullable=False, default="input") + output_field = Column(String(64), nullable=False, default="output") + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + samples = relationship( + "JudgeSample", + back_populates="dataset", + cascade="all, delete-orphan", + order_by="JudgeSample.created_at", + ) + runs = relationship( + "JudgeRun", + back_populates="dataset", + cascade="all, delete-orphan", + order_by="JudgeRun.created_at.desc()", + ) + + +class JudgeSample(Base): + """One labelable input/output pair within a JudgeDataset.""" + + __tablename__ = "judge_samples" + __table_args__ = ( + UniqueConstraint("dataset_id", "external_id", name="uq_judge_samples_dataset_external"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + dataset_id = Column( + UUID(as_uuid=True), + ForeignKey("judge_datasets.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + # Workspace isolation: mirrors the parent JudgeDataset's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Stable identifier within the source (e.g. transcription UUID, CSV row id). + # Used to dedupe re-imports and link back to the originating record. + external_id = Column(String(128), nullable=True, index=True) + + input_text = Column(Text, nullable=False) + output_text = Column(Text, nullable=False) + + # Binary human label: "pass" | "fail" | null (unlabeled). + # Stored as string (rather than enum) so it stays trivially extendable. + label = Column(String(16), nullable=True, index=True) + labeled_by = Column(String(255), nullable=True) + labeled_at = Column(DateTime(timezone=True), nullable=True) + + # Source-specific context (e.g. agent_id, original metric value, csv row). + extra = Column(JSON, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + dataset = relationship("JudgeDataset", back_populates="samples") + + +class JudgeRun(Base): + """One execution of an LLM-judge against a JudgeDataset, with alignment metrics.""" + + __tablename__ = "judge_runs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + dataset_id = Column( + UUID(as_uuid=True), + ForeignKey("judge_datasets.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + # Workspace isolation: mirrors the parent JudgeDataset's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Reuses the existing Evaluator row (its custom_prompt + llm_provider + llm_model + # define the judge under test). Nullable so a run may target an inline prompt + # in the future without inflating the Evaluator table. + evaluator_id = Column( + UUID(as_uuid=True), ForeignKey("evaluators.id", ondelete="SET NULL"), nullable=True, index=True + ) + + # Which subset was scored: "all" | "dev" | "test" + split = Column(String(16), nullable=False, default="all") + + # Snapshot of the model used (so a later Evaluator edit doesn't rewrite history). + llm_provider = Column(String(64), nullable=True) + llm_model = Column(String(128), nullable=True) + + # Computed alignment metrics: + # {"precision": float, "recall": float, "f1": float, "kappa": float, + # "tp": int, "fp": int, "tn": int, "fn": int, "n": int} + metrics = Column(JSON, nullable=True) + + # Per-sample predictions, keyed by sample_id (UUID string): + # {sample_id: {"prediction": "pass"|"fail", "explanation": str, "raw": str}} + predictions = Column(JSON, nullable=True) + + # Run lifecycle. + status = Column(String(20), nullable=False, default="pending", index=True) + error_message = Column(Text, nullable=True) + celery_task_id = Column(String, nullable=True, index=True) + + # Optional link to a GEPA optimization run kicked off from this dataset. + gepa_optimization_id = Column( + UUID(as_uuid=True), + ForeignKey("prompt_optimization_runs.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + dataset = relationship("JudgeDataset", back_populates="runs") diff --git a/app/models/enums.py b/app/models/enums.py index 86205f5d..140ffe48 100644 --- a/app/models/enums.py +++ b/app/models/enums.py @@ -182,6 +182,13 @@ class MetricCategory(str, enum.Enum): USER_INSIGHT = "user_insight" +class MetricLifecycle(str, enum.Enum): + """Lifecycle state for metrics — drafts are Studio-only until promoted.""" + + ACTIVE = "active" + DRAFT = "draft" + + class MetricTrigger(str, enum.Enum): """Metric trigger enumeration.""" ALWAYS = "always" diff --git a/app/models/schemas.py b/app/models/schemas.py index de5c203e..25208f30 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -1,5257 +1,5386 @@ -"""Pydantic schemas for request/response validation.""" - -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator, validator -import re -from typing import Optional, List, Dict, Any, Literal -from datetime import date, datetime -from uuid import UUID -from app.models.enums import ( - EvaluationType, EvaluationStatus, EvaluatorResultStatus, RoleEnum, InvitationStatus, - LanguageEnum, CallTypeEnum, CallMediumEnum, GenderEnum, AccentEnum, BackgroundNoiseEnum, - IntegrationPlatform, ModelProvider, CredentialRoutingMode, GatewayInterfaceMode, VoiceBundleType, TestAgentConversationStatus, - MetricType, MetricCategory, MetricTrigger, CallRecordingStatus, AlertMetricType, AlertAggregation, - AlertOperator, AlertNotifyFrequency, AlertStatus, AlertHistoryStatus, CronJobStatus, - CallImportStatus, CallImportRowStatus, CallImportParameterType, -) - - - -# Audio File Schemas -class AudioFileBase(BaseModel): - """Base audio file schema.""" - - filename: str - format: str - - -class AudioFileCreate(AudioFileBase): - """Schema for audio file creation.""" - - file_size: int - duration: Optional[float] = None - sample_rate: Optional[int] = None - channels: Optional[int] = None - - -class AudioFileResponse(AudioFileBase): - """Schema for audio file response.""" - - id: UUID - file_size: int - duration: Optional[float] = None - sample_rate: Optional[int] = None - channels: Optional[int] = None - uploaded_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -# Evaluation Schemas -class EvaluationCreate(BaseModel): - """Schema for creating an evaluation.""" - - audio_id: UUID - reference_text: Optional[str] = None - evaluation_type: EvaluationType - model_name: Optional[str] = Field(None, description="Model to use for evaluation") - metrics: Optional[List[str]] = Field( - default=["wer", "latency"], description="Metrics to calculate" - ) - - @field_validator("metrics") - @classmethod - def validate_metrics(cls, v): - """Validate metrics list.""" - allowed_metrics = ["wer", "cer", "latency", "quality_score", "rtf"] - if v: - invalid = [m for m in v if m not in allowed_metrics] - if invalid: - raise ValueError(f"Invalid metrics: {invalid}") - return v - - -class EvaluationResponse(BaseModel): - """Schema for evaluation response.""" - - id: UUID - audio_id: UUID - reference_text: Optional[str] = None - evaluation_type: EvaluationType - model_name: Optional[str] = None - status: EvaluationStatus - metrics_requested: Optional[List[str]] = None - created_at: datetime - started_at: Optional[datetime] = None - completed_at: Optional[datetime] = None - error_message: Optional[str] = None - - model_config = ConfigDict(from_attributes=True) - - -class EvaluationStatusResponse(BaseModel): - """Schema for evaluation status response.""" - - id: UUID - status: EvaluationStatus - created_at: datetime - completed_at: Optional[datetime] = None - error_message: Optional[str] = None - - -# Evaluation Result Schemas -class EvaluationResultResponse(BaseModel): - """Schema for evaluation result response.""" - - evaluation_id: UUID - status: EvaluationStatus - transcript: Optional[str] = None - metrics: Dict[str, Any] - processing_time: Optional[float] = None - model_used: Optional[str] = None - created_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class MetricsResponse(BaseModel): - """Schema for metrics breakdown.""" - - evaluation_id: UUID - metrics: Dict[str, Any] - processing_time: Optional[float] = None - - -# Comparison Schema -class ComparisonRequest(BaseModel): - """Schema for comparing multiple evaluations.""" - - evaluation_ids: List[UUID] = Field(..., min_length=2, description="At least 2 evaluation IDs to compare") - - -class ComparisonResponse(BaseModel): - """Schema for comparison results.""" - - evaluations: List[EvaluationResultResponse] - comparison_metrics: Dict[str, Any] - - -# API Key Schemas -class APIKeyCreate(BaseModel): - """Schema for creating API key.""" - - name: Optional[str] = None - - -class APIKeyResponse(BaseModel): - """Schema for API key response.""" - - id: UUID - key: str - name: Optional[str] = None - is_active: bool - created_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -# Generic Response Schemas -class MessageResponse(BaseModel): - """Generic message response.""" - - message: str - - -class ErrorResponse(BaseModel): - """Error response schema.""" - - detail: str - -# ============================================ -# VAIOPS SCHEMAS - Voice AI Ops -# ============================================ - -# Enums moved to enums.py - - -# Agent Schemas -class AgentCreate(BaseModel): - """Schema for creating a new agent""" - name: str = Field(..., min_length=1, max_length=255) - phone_number: Optional[str] = None - language: LanguageEnum = LanguageEnum.ENGLISH - description: str = Field(..., min_length=1) - call_type: CallTypeEnum = CallTypeEnum.OUTBOUND - call_medium: CallMediumEnum = CallMediumEnum.PHONE_CALL - telephony_phone_number_id: Optional[UUID] = None - voice_bundle_id: UUID = Field(..., description="Required voice bundle for test agent execution") - ai_provider_id: Optional[UUID] = None - voice_ai_integration_id: Optional[UUID] = None - voice_ai_agent_id: Optional[str] = None - provider_prompt: Optional[str] = None - silence_hangup_secs: int = Field( - default=15, - ge=0, - le=600, - description="End live calls after this many seconds of silence (0 disables)", - ) - - @field_validator('description') - @classmethod - def description_min_words(cls, v: str) -> str: - if len(v.split()) < 10: - raise ValueError('Description must be at least 10 words.') - return v - - @field_validator('phone_number') - @classmethod - def phone_number_format(cls, v: Optional[str]) -> Optional[str]: - if v is not None and v != '': - import re - if not re.fullmatch(r'[\d+]+', v): - raise ValueError('Phone number must contain only digits and the + character.') - return v - - @model_validator(mode='after') - def validate_phone_number(self): - """Ensure phone_number is provided when call_medium is phone_call""" - if self.call_medium == CallMediumEnum.PHONE_CALL and not self.phone_number: - raise ValueError('phone_number is required when call_medium is phone_call') - return self - - model_config = ConfigDict(json_schema_extra={ - "example": { - "name": "Customer Support Bot", - "phone_number": "+1234567890", - "language": "en", - "description": "A customer support bot that handles inquiries about orders, returns, and general questions", - "call_type": "outbound", - "voice_bundle_id": "123e4567-e89b-12d3-a456-426614174000", - "voice_ai_integration_id": "123e4567-e89b-12d3-a456-426614174001", - "voice_ai_agent_id": "agent_abc123" - } - }) - - -class AgentUpdate(BaseModel): - """Schema for updating an agent""" - name: Optional[str] = None - phone_number: Optional[str] = None - language: Optional[LanguageEnum] = None - description: Optional[str] = None - call_type: Optional[CallTypeEnum] = None - call_medium: Optional[CallMediumEnum] = None - telephony_phone_number_id: Optional[UUID] = None - voice_bundle_id: Optional[UUID] = None - voice_ai_integration_id: Optional[UUID] = None - voice_ai_agent_id: Optional[str] = None - provider_prompt: Optional[str] = None - prompt_variables: Optional[Dict[str, str]] = None - silence_hangup_secs: Optional[int] = Field(default=None, ge=0, le=600) - - @model_validator(mode='after') - def validate_voice_config(self): - """Validate voice configuration - both voice_bundle_id and voice_ai_integration_id can be provided independently""" - voice_bundle = self.voice_bundle_id - voice_ai_integration = self.voice_ai_integration_id - - # If voice_ai_integration_id is provided, voice_ai_agent_id must also be provided - if voice_ai_integration and not self.voice_ai_agent_id: - raise ValueError('voice_ai_agent_id is required when voice_ai_integration_id is provided.') - - return self - - @model_validator(mode='after') - def validate_phone_number(self): - """Ensure phone_number is provided when call_medium is phone_call""" - # Only validate if call_medium is being set to phone_call - if self.call_medium == CallMediumEnum.PHONE_CALL and not self.phone_number: - # If phone_number is not being updated, we need to check existing value - # This will be handled in the route - pass - return self - - - -class PreviewIntegrationAgentPromptRequest(BaseModel): - """Fetch a provider agent prompt before an EfficientAI agent exists.""" - voice_ai_agent_id: str = Field(..., min_length=1) - - - - -class PreviewIntegrationAgentPromptResponse(BaseModel): - provider_prompt: str - - - - -class AgentPhoneAssignmentConflict(BaseModel): - """Another agent already owns this phone number.""" - agent_id: UUID - agent_name: str - phone_number: str - - - - -class AgentPhoneAssignmentCheckResponse(BaseModel): - """Result of checking whether a phone number is free to assign.""" - available: bool - phone_number: Optional[str] = None - conflict: Optional[AgentPhoneAssignmentConflict] = None - - - - -class TestPromptSectionResponse(BaseModel): - """One canonical section of a generated test agent prompt.""" - key: str - title: str - content: str - - - - -class GeneratedScenarioDraftResponse(BaseModel): - """LLM-generated scenario draft before persistence.""" - name: str - description: str - goal: Optional[str] = None - - - - -class GenerateTestPromptRequest(BaseModel): - """Stage 1: generate foundational test agent prompt from production prompt.""" - production_prompt: str = Field(..., min_length=1) - agent_name: str = Field(..., min_length=1, max_length=255) - language: Optional[str] = None - call_type: Optional[str] = None - provider: Optional[str] = None - model: Optional[str] = None - credential_id: Optional[UUID] = None - llm_config: Optional[Dict[str, Any]] = None - additional_context: Optional[str] = None - - - - -class GenerateTestPromptResponse(BaseModel): - sections: List[TestPromptSectionResponse] - test_agent_prompt: str - provider: str - model: str - - - - -class GenerateScenariosFromPromptRequest(BaseModel): - """Stage 2: generate scenario drafts from test agent prompt.""" - test_agent_prompt: str = Field(..., min_length=1) - agent_name: str = Field(..., min_length=1, max_length=255) - scenario_count: int = Field(default=5, ge=1, le=10) - language: Optional[str] = None - call_type: Optional[str] = None - provider: Optional[str] = None - model: Optional[str] = None - credential_id: Optional[UUID] = None - llm_config: Optional[Dict[str, Any]] = None - additional_context: Optional[str] = None - - - - -class GenerateScenariosFromPromptResponse(BaseModel): - scenarios: List[GeneratedScenarioDraftResponse] - provider: str - model: str - - - - -class GenerateTestSetupRequest(BaseModel): - """Convenience: run stage 1 then stage 2 sequentially.""" - production_prompt: str = Field(..., min_length=1) - agent_name: str = Field(..., min_length=1, max_length=255) - scenario_count: int = Field(default=5, ge=1, le=10) - language: Optional[str] = None - call_type: Optional[str] = None - provider: Optional[str] = None - model: Optional[str] = None - credential_id: Optional[UUID] = None - llm_config: Optional[Dict[str, Any]] = None - additional_context: Optional[str] = None - - - - -class GenerateTestSetupResponse(BaseModel): - sections: List[TestPromptSectionResponse] - test_agent_prompt: str - scenarios: List[GeneratedScenarioDraftResponse] - provider: str - model: str - - - -class AgentResponse(BaseModel): - """Schema for agent response""" - id: UUID - agent_id: Optional[str] = None - name: str - phone_number: Optional[str] = None - language: LanguageEnum - description: Optional[str] - call_type: CallTypeEnum - call_medium: CallMediumEnum - telephony_phone_number_id: Optional[UUID] = None - voice_bundle_id: Optional[UUID] - ai_provider_id: Optional[UUID] - voice_ai_integration_id: Optional[UUID] - voice_ai_agent_id: Optional[str] - provider_prompt: Optional[str] = None - provider_prompt_synced_at: Optional[datetime] = None - created_at: datetime - updated_at: datetime - - @field_validator('language', mode='before') - @classmethod - def convert_language(cls, v): - """Convert string to LanguageEnum (handles uppercase DB values like ENGLISH -> en).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - # Map old uppercase names to new values - language_map = {'english': 'en', 'spanish': 'es', 'french': 'fr', 'german': 'de', - 'chinese': 'zh', 'japanese': 'ja', 'hindi': 'hi', 'arabic': 'ar'} - if v_lower in language_map: - return LanguageEnum(language_map[v_lower]) - try: - return LanguageEnum(v_lower) - except ValueError: - for enum_member in LanguageEnum: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid LanguageEnum value: {v}") - return v - - @field_validator('call_type', mode='before') - @classmethod - def convert_call_type(cls, v): - """Convert string to CallTypeEnum (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return CallTypeEnum(v_lower) - except ValueError: - for enum_member in CallTypeEnum: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid CallTypeEnum value: {v}") - return v - - @field_validator('call_medium', mode='before') - @classmethod - def convert_call_medium(cls, v): - """Convert string to CallMediumEnum (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return CallMediumEnum(v_lower) - except ValueError: - for enum_member in CallMediumEnum: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid CallMediumEnum value: {v}") - return v - - model_config = ConfigDict(from_attributes=True) - - -# Persona Schemas -class PersonaCreate(BaseModel): - """Schema for creating a new persona (TTS provider-tied voice identity)""" - name: str = Field(..., min_length=1, max_length=255) - gender: GenderEnum = GenderEnum.NEUTRAL - tts_provider: Optional[str] = None - tts_voice_id: Optional[str] = None - tts_voice_name: Optional[str] = None - is_custom: bool = False - description: Optional[str] = None - tts_config: Optional[Dict[str, Any]] = None - llm_temperature: Optional[float] = Field(None, ge=0.0, le=2.0) - llm_max_tokens: Optional[int] = Field(None, gt=0, le=8192) - response_delay_ms: Optional[int] = Field(None, ge=0, le=10000) - max_turns: Optional[int] = Field(None, ge=1, le=100) - allow_interruptions: Optional[bool] = None - - @model_validator(mode="after") - def validate_tts_config(self): - from app.services.personas.persona_tts_config import validate_persona_tts_config - - validate_persona_tts_config(self.tts_provider, self.tts_config) - return self - - -class PersonaUpdate(BaseModel): - """Schema for updating a persona""" - name: Optional[str] = None - gender: Optional[GenderEnum] = None - tts_provider: Optional[str] = None - tts_voice_id: Optional[str] = None - tts_voice_name: Optional[str] = None - is_custom: Optional[bool] = None - description: Optional[str] = None - tts_config: Optional[Dict[str, Any]] = None - llm_temperature: Optional[float] = Field(None, ge=0.0, le=2.0) - llm_max_tokens: Optional[int] = Field(None, gt=0, le=8192) - response_delay_ms: Optional[int] = Field(None, ge=0, le=10000) - max_turns: Optional[int] = Field(None, ge=1, le=100) - allow_interruptions: Optional[bool] = None - - @model_validator(mode="after") - def validate_tts_config(self): - from app.services.personas.persona_tts_config import validate_persona_tts_config - - if self.tts_config is not None: - validate_persona_tts_config(self.tts_provider, self.tts_config) - return self - - -class PersonaResponse(BaseModel): - """Schema for persona response""" - id: UUID - name: str - gender: str - tts_provider: Optional[str] = None - tts_voice_id: Optional[str] = None - tts_voice_name: Optional[str] = None - is_custom: bool = False - description: Optional[str] = None - tts_config: Optional[Dict[str, Any]] = None - llm_temperature: Optional[float] = None - llm_max_tokens: Optional[int] = None - response_delay_ms: Optional[int] = None - max_turns: Optional[int] = None - allow_interruptions: Optional[bool] = None - created_at: datetime - updated_at: datetime - - @field_validator('gender', mode='before') - @classmethod - def convert_gender(cls, v): - if v is None: - return "neutral" - if isinstance(v, str): - return v.lower() - if hasattr(v, 'value'): - return v.value - return v - - model_config = ConfigDict(from_attributes=True) - - -class PersonaCloneRequest(BaseModel): - """Schema for cloning a persona""" - name: Optional[str] = None - - -# Scenario Schemas - -class AgentPromptSourcesResponse(BaseModel): - """Prompt texts from an agent that can seed a persona description.""" - agent_id: UUID - agent_name: str - test_agent_prompt: str - agent_prompt: str - - - - -class GeneratePersonaPromptRequest(BaseModel): - """Generate a persona caller prompt from an agent prompt via LLM.""" - agent_id: UUID - source: str = Field(default="auto", pattern="^(test_agent|agent|auto)$") - persona_name: Optional[str] = Field(None, max_length=255) - persona_gender: Optional[str] = None - additional_context: Optional[str] = None - provider: Optional[str] = None - model: Optional[str] = None - credential_id: Optional[UUID] = None - llm_config: Optional[Dict[str, Any]] = None - - - - -class GeneratePersonaPromptResponse(BaseModel): - persona_prompt: str - source_used: str - provider: str - model: str - - -# Scenario Schemas - -class ScenarioCreate(BaseModel): - """Schema for creating a new scenario""" - name: str = Field(..., min_length=1, max_length=255) - agent_id: Optional[UUID] = None - description: Optional[str] = None - required_info: Dict[str, str] = Field(default_factory=dict) - - -class ScenarioUpdate(BaseModel): - """Schema for updating a scenario""" - name: Optional[str] = None - agent_id: Optional[UUID] = None - description: Optional[str] = None - required_info: Optional[Dict[str, str]] = None - - -class ScenarioResponse(BaseModel): - """Schema for scenario response""" - id: UUID - name: str - agent_id: Optional[UUID] - description: Optional[str] - required_info: Dict[str, str] - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -# ============================================ -# IAM & USER SCHEMAS -# ============================================ - -# User Schemas -class UserCreate(BaseModel): - """Schema for creating a user.""" - email: str = Field(..., description="User email address") - name: Optional[str] = None - password: Optional[str] = None # Optional for invitation-based signup - - -class UserUpdate(BaseModel): - """Schema for updating user profile.""" - name: Optional[str] = None - first_name: Optional[str] = None - last_name: Optional[str] = None - email: Optional[str] = None - - -class UserResponse(BaseModel): - """Schema for user response.""" - id: UUID - email: str - name: Optional[str] - first_name: Optional[str] - last_name: Optional[str] - is_active: bool - created_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class OrganizationMemberResponse(BaseModel): - """Schema for organization member response.""" - id: UUID - user_id: UUID - organization_id: UUID - role: RoleEnum - joined_at: datetime - user: UserResponse # Include user details - - model_config = ConfigDict(from_attributes=True) - - -# Invitation Schemas -class InvitationCreate(BaseModel): - """Schema for creating an invitation.""" - email: str = Field(..., description="Email address of the user to invite") - role: RoleEnum = RoleEnum.READER - - -class InvitationResponse(BaseModel): - """Schema for invitation response.""" - id: UUID - organization_id: UUID - email: str - role: RoleEnum - status: InvitationStatus - expires_at: datetime - created_at: datetime - organization_name: Optional[str] = None # Include organization name - - @field_validator('role', mode='before') - @classmethod - def convert_role(cls, v): - """Convert string to RoleEnum (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return RoleEnum(v_lower) - except ValueError: - for enum_member in RoleEnum: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid RoleEnum value: {v}") - return v - - @field_validator('status', mode='before') - @classmethod - def convert_status(cls, v): - """Convert string to InvitationStatus (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return InvitationStatus(v_lower) - except ValueError: - for enum_member in InvitationStatus: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid InvitationStatus value: {v}") - return v - - model_config = ConfigDict(from_attributes=True) - - -class InvitationUpdate(BaseModel): - """Schema for updating invitation (accept/decline).""" - token: str - - -class RoleUpdate(BaseModel): - """Schema for updating user role in organization.""" - role: RoleEnum - - -# Profile Schemas -class ProfileResponse(BaseModel): - """Schema for user profile response.""" - id: UUID - email: str - name: Optional[str] - first_name: Optional[str] - last_name: Optional[str] - created_at: datetime - organizations: List[dict] = Field(default_factory=list) # List of org memberships - - model_config = ConfigDict(from_attributes=True) - - -# ============================================ -# INTEGRATION SCHEMAS -# ============================================ - -class IntegrationCreate(BaseModel): - """Schema for creating an integration.""" - platform: IntegrationPlatform - api_key: str = Field(..., description="Private API key for the platform") - public_key: Optional[str] = Field(None, description="Optional public API key (e.g. for Vapi)") - name: Optional[str] = Field(None, description="Optional friendly name for the integration") - routing_mode: CredentialRoutingMode = Field( - CredentialRoutingMode.INHERIT, - description="LLM routing preference: inherit org default, force gateway, or direct API key.", - ) - is_default: Optional[bool] = Field( - None, - description=( - "Mark this credential as the default for the (org, platform). " - "If omitted and no default exists yet, this row becomes the default." - ), - ) - - -class IntegrationUpdate(BaseModel): - """Schema for updating an integration.""" - name: Optional[str] = None - api_key: Optional[str] = None - public_key: Optional[str] = None - is_active: Optional[bool] = None - routing_mode: Optional[CredentialRoutingMode] = None - - -class IntegrationResponse(BaseModel): - """Schema for integration response.""" - id: UUID - organization_id: UUID - platform: IntegrationPlatform - name: Optional[str] - public_key: Optional[str] = None - is_active: bool - is_default: bool = False - routing_mode: CredentialRoutingMode = CredentialRoutingMode.INHERIT - effective_routing: Literal["inherit", "direct", "gateway", "bifrost", "litellm_proxy"] = "inherit" - created_at: datetime - updated_at: datetime - last_tested_at: Optional[datetime] = None - # Note: api_key is NOT included in response for security - - @field_validator('platform', mode='before') - @classmethod - def convert_platform(cls, v): - """Convert string to IntegrationPlatform enum if needed (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - # Try lowercase first (enum value) - v_lower = v.lower() - try: - return IntegrationPlatform(v_lower) - except ValueError: - # Try to find by enum name (uppercase) - for enum_member in IntegrationPlatform: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid IntegrationPlatform value: {v}") - return v - - @field_validator('routing_mode', mode='before') - @classmethod - def convert_routing_mode(cls, v): - if v is None: - return CredentialRoutingMode.INHERIT - if isinstance(v, str): - try: - return CredentialRoutingMode(v.lower()) - except ValueError: - return CredentialRoutingMode.INHERIT - return v - - model_config = ConfigDict(from_attributes=True) - - -# ============================================ -# DATA SOURCES SCHEMAS -# ============================================ - -class S3ConnectionTest(BaseModel): - """Schema for testing S3 connection.""" - bucket_name: str - region: str = "us-east-1" - access_key_id: str - secret_access_key: str - endpoint_url: Optional[str] = None - - -class S3ConnectionTestResponse(BaseModel): - """Schema for S3 connection test response.""" - success: bool - message: str - bucket_name: Optional[str] = None - - -class S3FileInfo(BaseModel): - """Schema for S3 file information.""" - key: str - filename: str - size: int - last_modified: str - - -class S3ListFilesResponse(BaseModel): - """Schema for listing S3 files response.""" - files: List[S3FileInfo] - total: int - prefix: Optional[str] = None - - -class S3FolderInfo(BaseModel): - """Schema for S3 folder information.""" - name: str - path: str - - -class S3BrowseResponse(BaseModel): - """Schema for browsing S3 folders within an organization.""" - folders: List[S3FolderInfo] - files: List[S3FileInfo] - current_path: str - organization_id: str - - -class S3UploadResponse(BaseModel): - """Schema for S3 upload response.""" - key: str - bucket: str - file_id: UUID - message: str - - -# AIProvider Schemas -_MAX_GATEWAY_EXTRA_HEADERS = 20 - - -def _validate_gateway_extra_headers( - value: Optional[Dict[str, Any]], -) -> Optional[Dict[str, str]]: - if value is None: - return None - if not isinstance(value, dict): - raise ValueError("gateway_extra_headers must be a JSON object of string keys and values.") - if len(value) > _MAX_GATEWAY_EXTRA_HEADERS: - raise ValueError( - f"gateway_extra_headers supports at most {_MAX_GATEWAY_EXTRA_HEADERS} headers." - ) - normalized: Dict[str, str] = {} - for raw_key, raw_val in value.items(): - key = str(raw_key).strip() - if not key: - raise ValueError("gateway_extra_headers keys must be non-empty strings.") - if len(key) > 64 or any(ch.isspace() for ch in key): - raise ValueError(f"Invalid gateway header name: {key!r}") - if raw_val is None: - raise ValueError(f"gateway_extra_headers[{key!r}] must be a string value.") - val = str(raw_val).strip() - if not val: - raise ValueError(f"gateway_extra_headers[{key!r}] must be a non-empty string.") - if len(val) > 1024 or "\n" in val or "\r" in val: - raise ValueError(f"gateway_extra_headers[{key!r}] value is invalid.") - normalized[key] = val - return normalized or None - - -class AIProviderCreate(BaseModel): - """Schema for creating an AI Provider.""" - provider: ModelProvider - api_key: Optional[str] = Field( - None, - description=( - "Provider API key. Optional when routing via gateway with " - "gateway-managed credentials (passthrough_provider_keys: false)." - ), - ) - name: Optional[str] = None - routing_mode: CredentialRoutingMode = Field( - CredentialRoutingMode.INHERIT, - description="LLM routing preference: inherit org default, force gateway, or direct API key.", - ) - gateway_model: Optional[str] = Field( - None, - min_length=1, - max_length=255, - description="Bifrost custom model ID sent when routing via gateway.", - ) - gateway_interface: GatewayInterfaceMode = Field( - GatewayInterfaceMode.INHERIT, - description="Bifrost API surface: inherit org default, LiteLLM shim, or native OpenAI-compatible.", - ) - gateway_base_url: Optional[str] = Field( - None, - max_length=512, - description="Optional per-credential Bifrost/gateway base URL override.", - ) - gateway_auth_header: Optional[str] = Field( - None, - max_length=64, - description="Auth header name for Bifrost (default x-bf-vk).", - ) - gateway_auth_secret_env: Optional[str] = Field( - None, - max_length=128, - description="Environment variable name holding the gateway auth secret.", - ) - gateway_auth_secret: Optional[str] = Field( - None, - description="Inline gateway auth secret (encrypted at rest). Alternative to env var.", - ) - gateway_extra_headers: Optional[Dict[str, str]] = Field( - None, - description="Arbitrary HTTP headers sent with gateway-routed LiteLLM calls.", - ) - is_default: Optional[bool] = Field( - None, - description=( - "Mark this credential as the default for the (org, provider). " - "If omitted and no default exists yet, this row becomes the default." - ), - ) - endpoint_url: Optional[str] = Field( - None, - description="Provider endpoint URL (required for Azure OpenAI).", - ) - - @field_validator("api_key") - @classmethod - def validate_api_key(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - return trimmed or None - - @field_validator("endpoint_url") - @classmethod - def validate_endpoint_url(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - return trimmed or None - - @field_validator("gateway_model") - @classmethod - def validate_gateway_model(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - return trimmed or None - - @field_validator("gateway_base_url") - @classmethod - def validate_gateway_base_url(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - return trimmed or None - - @field_validator("gateway_auth_header") - @classmethod - def validate_gateway_auth_header(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - if not trimmed: - return None - if len(trimmed) > 64 or any(ch.isspace() for ch in trimmed): - raise ValueError("gateway_auth_header must be a single non-empty header name.") - return trimmed - - @field_validator("gateway_auth_secret_env") - @classmethod - def validate_gateway_auth_secret_env(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - if not trimmed: - return None - if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", trimmed): - raise ValueError( - "gateway_auth_secret_env must be a valid environment variable name." - ) - return trimmed - - @field_validator("gateway_auth_secret") - @classmethod - def validate_gateway_auth_secret(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - return trimmed or None - - @field_validator("gateway_extra_headers") - @classmethod - def validate_gateway_extra_headers(cls, v: Optional[Dict[str, Any]]) -> Optional[Dict[str, str]]: - return _validate_gateway_extra_headers(v) - - -class AIProviderUpdate(BaseModel): - """Schema for updating an AI Provider.""" - api_key: Optional[str] = Field(None, min_length=1) - name: Optional[str] = None - endpoint_url: Optional[str] = None - is_active: Optional[bool] = None - routing_mode: Optional[CredentialRoutingMode] = None - gateway_model: Optional[str] = Field(None, min_length=1, max_length=255) - gateway_interface: Optional[GatewayInterfaceMode] = None - gateway_base_url: Optional[str] = Field(None, max_length=512) - gateway_auth_header: Optional[str] = Field(None, max_length=64) - gateway_auth_secret_env: Optional[str] = Field(None, max_length=128) - gateway_auth_secret: Optional[str] = None - clear_gateway_auth_secret: bool = False - gateway_extra_headers: Optional[Dict[str, str]] = None - - @field_validator("gateway_model") - @classmethod - def validate_gateway_model(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - return trimmed or None - - @field_validator("gateway_base_url") - @classmethod - def validate_gateway_base_url_update(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - return trimmed or None - - @field_validator("gateway_auth_header") - @classmethod - def validate_gateway_auth_header_update(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - if not trimmed: - return None - if len(trimmed) > 64 or any(ch.isspace() for ch in trimmed): - raise ValueError("gateway_auth_header must be a single non-empty header name.") - return trimmed - - @field_validator("gateway_auth_secret_env") - @classmethod - def validate_gateway_auth_secret_env_update(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - if not trimmed: - return None - if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", trimmed): - raise ValueError( - "gateway_auth_secret_env must be a valid environment variable name." - ) - return trimmed - - @field_validator("gateway_auth_secret") - @classmethod - def validate_gateway_auth_secret_update(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - return trimmed or None - - @field_validator("gateway_extra_headers") - @classmethod - def validate_gateway_extra_headers_update( - cls, v: Optional[Dict[str, Any]] - ) -> Optional[Dict[str, str]]: - return _validate_gateway_extra_headers(v) - - -class AIProviderResponse(BaseModel): - """Schema for AI Provider response.""" - id: UUID - provider: ModelProvider - api_key: Optional[str] = None # Will be None in response for security - name: Optional[str] - endpoint_url: Optional[str] = None - is_active: bool - is_default: bool = False - routing_mode: CredentialRoutingMode = CredentialRoutingMode.INHERIT - gateway_model: Optional[str] = None - gateway_interface: GatewayInterfaceMode = GatewayInterfaceMode.INHERIT - gateway_base_url: Optional[str] = None - gateway_auth_header: Optional[str] = None - gateway_auth_secret_env: Optional[str] = None - has_gateway_auth_secret: bool = False - gateway_extra_headers: Optional[Dict[str, str]] = None - gateway_managed: bool = False - effective_routing: Literal["inherit", "direct", "gateway", "bifrost", "litellm_proxy"] = "inherit" - effective_gateway_interface: Literal["litellm_shim", "native_openai"] = "litellm_shim" - created_at: datetime - updated_at: datetime - last_tested_at: Optional[datetime] - - @field_validator('provider', mode='before') - @classmethod - def convert_provider(cls, v): - """Convert string to ModelProvider (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return ModelProvider(v_lower) - except ValueError: - for enum_member in ModelProvider: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid ModelProvider value: {v}") - return v - - @field_validator('routing_mode', mode='before') - @classmethod - def convert_routing_mode(cls, v): - if v is None: - return CredentialRoutingMode.INHERIT - if isinstance(v, str): - try: - return CredentialRoutingMode(v.lower()) - except ValueError: - return CredentialRoutingMode.INHERIT - return v - - model_config = ConfigDict(from_attributes=True) - - -class LLMGenerationConfig(BaseModel): - """User-tunable LLM sampling / generation parameters.""" - - temperature: Optional[float] = Field(None, ge=0.0, le=2.0) - max_tokens: Optional[int] = Field(None, gt=0) - top_p: Optional[float] = Field(None, ge=0.0, le=1.0) - top_k: Optional[int] = Field(None, ge=0) - frequency_penalty: Optional[float] = Field(None, ge=-2.0, le=2.0) - presence_penalty: Optional[float] = Field(None, ge=-2.0, le=2.0) - seed: Optional[int] = Field(None, ge=0) - - def to_dict(self) -> Dict[str, Any]: - """Return only explicitly set fields.""" - return self.model_dump(exclude_none=True) - - -# VoiceBundle Schemas -class VoiceBundleCreate(BaseModel): - """Schema for creating a VoiceBundle.""" - name: str = Field(..., min_length=1, max_length=255) - description: Optional[str] = None - - # Bundle type: either STT+LLM+TTS or S2S - bundle_type: VoiceBundleType = Field(default=VoiceBundleType.STT_LLM_TTS) - - # STT Configuration - required for STT_LLM_TTS, optional for S2S - stt_provider: Optional[ModelProvider] = None - stt_model: Optional[str] = Field(None, min_length=1) - stt_credential_id: Optional[UUID] = Field( - None, - description=( - "Optional explicit AIProvider/Integration row id to use for STT. " - "When omitted the resolver picks the default credential for stt_provider." - ), - ) - - # LLM Configuration - required for STT_LLM_TTS, optional for S2S - llm_provider: Optional[ModelProvider] = None - llm_model: Optional[str] = Field(None, min_length=1) - llm_temperature: Optional[float] = Field(None, ge=0.0, le=2.0) - llm_max_tokens: Optional[int] = Field(None, gt=0) - llm_config: Optional[Dict[str, Any]] = None - llm_credential_id: Optional[UUID] = None - - # TTS Configuration - required for STT_LLM_TTS, optional for S2S - tts_provider: Optional[ModelProvider] = None - tts_model: Optional[str] = Field(None, min_length=1) - tts_voice: Optional[str] = None - tts_config: Optional[Dict[str, Any]] = None - tts_credential_id: Optional[UUID] = None - - # S2S Configuration - required for S2S, optional for STT_LLM_TTS - s2s_provider: Optional[ModelProvider] = None - s2s_model: Optional[str] = Field(None, min_length=1) - s2s_config: Optional[Dict[str, Any]] = None - s2s_credential_id: Optional[UUID] = None - - # Additional metadata - extra_metadata: Optional[Dict[str, Any]] = None - - @model_validator(mode='after') - def validate_bundle_configuration(self): - """Validate that required fields are provided based on bundle_type.""" - if self.bundle_type == VoiceBundleType.STT_LLM_TTS: - if not self.stt_provider or not self.stt_model: - raise ValueError('STT provider and model are required for STT_LLM_TTS bundle type') - if not self.llm_provider or not self.llm_model: - raise ValueError('LLM provider and model are required for STT_LLM_TTS bundle type') - if not self.tts_provider or not self.tts_model: - raise ValueError('TTS provider and model are required for STT_LLM_TTS bundle type') - elif self.bundle_type == VoiceBundleType.S2S: - if not self.s2s_provider or not self.s2s_model: - raise ValueError('S2S provider and model are required for S2S bundle type') - return self - - -class VoiceBundleUpdate(BaseModel): - """Schema for updating a VoiceBundle.""" - name: Optional[str] = Field(None, min_length=1, max_length=255) - description: Optional[str] = None - - # Bundle type - bundle_type: Optional[VoiceBundleType] = None - - # STT Configuration - stt_provider: Optional[ModelProvider] = None - stt_model: Optional[str] = Field(None, min_length=1) - stt_credential_id: Optional[UUID] = None - - # LLM Configuration - llm_provider: Optional[ModelProvider] = None - llm_model: Optional[str] = Field(None, min_length=1) - llm_temperature: Optional[float] = Field(None, ge=0.0, le=2.0) - llm_max_tokens: Optional[int] = Field(None, gt=0) - llm_config: Optional[Dict[str, Any]] = None - llm_credential_id: Optional[UUID] = None - - # TTS Configuration - tts_provider: Optional[ModelProvider] = None - tts_model: Optional[str] = Field(None, min_length=1) - tts_voice: Optional[str] = None - tts_config: Optional[Dict[str, Any]] = None - tts_credential_id: Optional[UUID] = None - - # S2S Configuration - s2s_provider: Optional[ModelProvider] = None - s2s_model: Optional[str] = Field(None, min_length=1) - s2s_config: Optional[Dict[str, Any]] = None - s2s_credential_id: Optional[UUID] = None - - # Additional metadata - extra_metadata: Optional[Dict[str, Any]] = None - is_active: Optional[bool] = None - - -class VoiceBundleResponse(BaseModel): - """Schema for VoiceBundle response.""" - id: UUID - name: str - description: Optional[str] - - # Bundle type - can be string from DB or enum, validator handles conversion - bundle_type: VoiceBundleType - - @field_validator('bundle_type', mode='before') - @classmethod - def convert_bundle_type(cls, v): - """Convert string to VoiceBundleType enum if needed.""" - if isinstance(v, str): - try: - return VoiceBundleType(v) - except ValueError: - # Try to find by value - for enum_member in VoiceBundleType: - if enum_member.value == v: - return enum_member - raise ValueError(f"Invalid bundle_type value: {v}") - return v - - @field_validator('stt_provider', 'llm_provider', 'tts_provider', 's2s_provider', mode='before') - @classmethod - def convert_model_provider(cls, v): - """Convert string to ModelProvider enum if needed (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - # Try lowercase first (enum value) - v_lower = v.lower() - try: - return ModelProvider(v_lower) - except ValueError: - # Try to find by enum name (uppercase) - for enum_member in ModelProvider: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid ModelProvider value: {v}") - return v - - # STT Configuration - stt_provider: Optional[ModelProvider] - stt_model: Optional[str] - stt_credential_id: Optional[UUID] = None - - # LLM Configuration - llm_provider: Optional[ModelProvider] - llm_model: Optional[str] - llm_temperature: Optional[float] - llm_max_tokens: Optional[int] - llm_config: Optional[Dict[str, Any]] - llm_credential_id: Optional[UUID] = None - - # TTS Configuration - tts_provider: Optional[ModelProvider] - tts_model: Optional[str] - tts_voice: Optional[str] - tts_config: Optional[Dict[str, Any]] - tts_credential_id: Optional[UUID] = None - - # S2S Configuration - s2s_provider: Optional[ModelProvider] - s2s_model: Optional[str] - s2s_config: Optional[Dict[str, Any]] - s2s_credential_id: Optional[UUID] = None - - # Additional metadata - extra_metadata: Optional[Dict[str, Any]] - is_active: bool - created_at: datetime - updated_at: datetime - created_by: Optional[str] - - model_config = ConfigDict(from_attributes=True) - - -# Test Agent Conversation Schemas -class TestAgentConversationCreate(BaseModel): - """Schema for creating a new test agent conversation.""" - agent_id: UUID - persona_id: UUID - scenario_id: UUID - voice_bundle_id: UUID - conversation_metadata: Optional[Dict[str, Any]] = None - - model_config = ConfigDict(json_schema_extra={ - "example": { - "agent_id": "123e4567-e89b-12d3-a456-426614174000", - "persona_id": "123e4567-e89b-12d3-a456-426614174001", - "scenario_id": "123e4567-e89b-12d3-a456-426614174002", - "voice_bundle_id": "123e4567-e89b-12d3-a456-426614174003" - } - }) - - -class TestAgentConversationUpdate(BaseModel): - """Schema for updating a test agent conversation.""" - status: Optional[str] = None - live_transcription: Optional[List[Dict[str, Any]]] = None - full_transcript: Optional[str] = None - conversation_metadata: Optional[Dict[str, Any]] = None - - -class TestAgentConversationResponse(BaseModel): - """Schema for test agent conversation response.""" - id: UUID - organization_id: UUID - agent_id: UUID - persona_id: UUID - scenario_id: UUID - voice_bundle_id: UUID - status: str - live_transcription: Optional[List[Dict[str, Any]]] - conversation_audio_key: Optional[str] - full_transcript: Optional[str] - started_at: datetime - ended_at: Optional[datetime] - duration_seconds: Optional[float] - conversation_metadata: Optional[Dict[str, Any]] - created_at: datetime - updated_at: datetime - created_by: Optional[str] - - model_config = ConfigDict(from_attributes=True) - - -class ConversationTurn(BaseModel): - """Schema for a single conversation turn.""" - speaker: str # "test_agent" or "voice_agent" - text: str - timestamp: float # Time in seconds from start - audio_segment_key: Optional[str] = None # S3 key for this segment's audio - - -# Conversation Evaluation Schemas -class ConversationEvaluationCreate(BaseModel): - """Schema for creating a conversation evaluation.""" - transcription_id: UUID - agent_id: UUID - llm_provider: Optional[ModelProvider] = ModelProvider.OPENAI - llm_model: Optional[str] = "gpt-4o" - - model_config = ConfigDict(json_schema_extra={ - "example": { - "transcription_id": "123e4567-e89b-12d3-a456-426614174000", - "agent_id": "123e4567-e89b-12d3-a456-426614174001", - "llm_provider": "openai", - "llm_model": "gpt-4o" - } - }) - - -class ConversationEvaluationResponse(BaseModel): - """Schema for conversation evaluation response.""" - id: UUID - organization_id: UUID - transcription_id: UUID - agent_id: UUID - objective_achieved: bool - objective_achieved_reason: Optional[str] - additional_metrics: Optional[Dict[str, Any]] - overall_score: Optional[float] - llm_provider: Optional[ModelProvider] - llm_model: Optional[str] - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -# Evaluator Schemas -class EvaluatorCreate(BaseModel): - """Schema for creating an evaluator. Either provide agent_id+persona_id+scenario_id (standard) or metric_ids/custom_prompt (custom).""" - name: Optional[str] = None - agent_id: Optional[UUID] = None - persona_id: Optional[UUID] = None - scenario_id: Optional[UUID] = None - custom_prompt: Optional[str] = None - metric_ids: Optional[List[UUID]] = None - llm_provider: Optional[ModelProvider] = None - llm_model: Optional[str] = None - llm_config: Optional[Dict[str, Any]] = None - tags: Optional[List[str]] = None - - -class EvaluatorUpdate(BaseModel): - """Schema for updating an evaluator.""" - name: Optional[str] = None - agent_id: Optional[UUID] = None - persona_id: Optional[UUID] = None - scenario_id: Optional[UUID] = None - custom_prompt: Optional[str] = None - metric_ids: Optional[List[UUID]] = None - llm_provider: Optional[ModelProvider] = None - llm_model: Optional[str] = None - llm_config: Optional[Dict[str, Any]] = None - tags: Optional[List[str]] = None - - -class EvaluatorResponse(BaseModel): - """Schema for evaluator response.""" - id: UUID - evaluator_id: str - organization_id: UUID - name: Optional[str] = None - agent_id: Optional[UUID] = None - persona_id: Optional[UUID] = None - scenario_id: Optional[UUID] = None - custom_prompt: Optional[str] = None - metric_ids: Optional[List[UUID]] = None - llm_provider: Optional[ModelProvider] = None - llm_model: Optional[str] = None - llm_config: Optional[Dict[str, Any]] = None - tags: Optional[List[str]] - created_at: datetime - updated_at: datetime - created_by: Optional[str] - - @field_validator('llm_provider', mode='before') - @classmethod - def convert_llm_provider(cls, v): - """Convert string to ModelProvider (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return ModelProvider(v_lower) - except ValueError: - for enum_member in ModelProvider: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid ModelProvider value: {v}") - return v - - model_config = ConfigDict(from_attributes=True) - - -class EvaluatorBulkCreate(BaseModel): - """Schema for creating multiple evaluators at once.""" - name: Optional[str] = None - agent_id: UUID - scenario_id: UUID - persona_ids: List[UUID] - tags: Optional[List[str]] = None - - -class RunEvaluatorsRequest(BaseModel): - """Schema for running evaluators.""" - evaluator_ids: List[UUID] = Field(..., description="List of evaluator IDs to run") - - -class RunEvaluatorsResponse(BaseModel): - """Schema for run evaluators response.""" - task_ids: List[str] = Field(..., description="List of Celery task IDs for tracking") - evaluator_results: List["EvaluatorResultResponse"] = Field(default_factory=list, description="List of created evaluator results") - - model_config = ConfigDict( - from_attributes=True, - json_schema_extra={ - "example": { - "agent_id": "123e4567-e89b-12d3-a456-426614174000", - "scenario_id": "123e4567-e89b-12d3-a456-426614174002", - "persona_ids": [ - "123e4567-e89b-12d3-a456-426614174001", - "123e4567-e89b-12d3-a456-426614174003" - ], - "tags": ["test", "production"] - } - }, - ) - - -# Metric Schemas -SelectionMode = Literal["single_choice", "multi_label"] - - -MetricScope = Literal["workspace", "organization"] - -# Max length for metric rubric text (description / example) accepted by -# the API. DB columns are TEXT or unbounded VARCHAR; this cap is validation-only. -METRIC_RUBRIC_TEXT_MAX_LENGTH = 32_000 - - - -class EvaluatorSuiteCombinationResponse(BaseModel): - """One agent+persona+scenario combination inside a suite.""" - id: UUID - evaluator_id: str - scenario_id: Optional[UUID] = None - scenario_name: Optional[str] = None - scenario_description: Optional[str] = None - scenario_required_info: Optional[Any] = None - - - - -class EvaluatorSuiteCreate(BaseModel): - """Schema for creating an evaluator suite.""" - name: Optional[str] = None - agent_id: UUID - persona_id: UUID - scenario_ids: List[UUID] - metric_ids: Optional[List[UUID]] = None - llm_provider: Optional[ModelProvider] = None - llm_model: Optional[str] = None - llm_config: Optional[Dict[str, Any]] = None - tags: Optional[List[str]] = None - default_runs_per_combination: int = 1 - - - - -class EvaluatorSuiteUpdate(BaseModel): - """Schema for updating an evaluator suite.""" - name: Optional[str] = None - tags: Optional[List[str]] = None - default_runs_per_combination: Optional[int] = None - llm_provider: Optional[ModelProvider] = None - llm_model: Optional[str] = None - llm_config: Optional[Dict[str, Any]] = None - metric_ids: Optional[List[UUID]] = None - - - - -class EvaluatorSuiteResponse(BaseModel): - """Schema for evaluator suite response.""" - id: UUID - organization_id: UUID - name: Optional[str] = None - agent_id: UUID - persona_id: UUID - agent_name: Optional[str] = None - persona_name: Optional[str] = None - agent_call_type: Optional[str] = None - agent_call_medium: Optional[str] = None - metric_ids: Optional[List[str]] = None - llm_provider: Optional[str] = None - llm_model: Optional[str] = None - llm_config: Optional[Dict[str, Any]] = None - tags: Optional[List[str]] = None - default_runs_per_combination: int = 1 - round_robin_index: int = 0 - is_active: bool = False - agent_suite_count: int = 1 - combination_count: int = 0 - combinations: List[EvaluatorSuiteCombinationResponse] = Field(default_factory=list) - created_at: datetime - updated_at: datetime - created_by: Optional[str] = None - - - - -class EvaluatorSuiteAddScenariosRequest(BaseModel): - """Schema for adding scenarios to an existing suite.""" - scenario_ids: List[UUID] - - - - -class RunEvaluatorSuiteRequest(BaseModel): - """Schema for running all combinations in a suite.""" - runs_per_combination: Optional[int] = None - to_number: Optional[str] = None - from_number: Optional[str] = None - - - - -class RunEvaluatorSuiteResponse(BaseModel): - """Schema for suite run response.""" - total_runs: int - task_ids: List[str] = Field(default_factory=list) - evaluator_results: List["EvaluatorResultResponse"] = Field(default_factory=list) - phone_call_refs: List[str] = Field(default_factory=list) - - - - -class RunNextCombinationRequest(BaseModel): - """Schema for running the next round-robin combination.""" - from_number: Optional[str] = None - - - - -class RunNextCombinationResponse(BaseModel): - """Schema for round-robin run response.""" - evaluator_id: UUID - scenario_id: Optional[UUID] = None - scenario_name: str - combination_index: int - next_index: int - evaluator_result_id: Optional[UUID] = None - result_id: Optional[str] = None - task_id: Optional[str] = None - phone_call_ref: Optional[str] = None - call_short_id: Optional[str] = None - - - - -class ChooseNextCombinationResponse(BaseModel): - """Advance inbound round-robin without initiating a call or evaluation run.""" - evaluator_id: UUID - scenario_id: Optional[UUID] = None - scenario_name: str - combination_index: int - next_index: int - - -# Metric Schemas -SelectionMode = Literal["single_choice", "multi_label"] - - -MetricScope = Literal["workspace", "organization"] - -# Max length for metric rubric text (description / example) accepted by -# the API. DB columns are TEXT or unbounded VARCHAR; this cap is validation-only. -METRIC_RUBRIC_TEXT_MAX_LENGTH = 32_000 - - - -class MetricCreate(BaseModel): - """Schema for creating a metric. - - Hierarchy: - - ``parent_metric_id`` set => this is a child sub-metric. ``metric_type`` - is forced to ``boolean`` server-side; ``selection_mode`` must be None. - - ``selection_mode`` set => this is a parent category metric. - ``parent_metric_id`` must be None (max depth = 2). - - Scope: - - ``scope="workspace"`` (default) stamps the metric with the active - ``X-Workspace-Id`` so it only shows up inside that workspace. - - ``scope="organization"`` stamps ``workspace_id=NULL`` so the metric - is visible in every workspace of the org. Children always inherit - their parent's scope; setting ``scope`` on a child request body is - ignored server-side. - """ - name: str - description: Optional[str] = None - # Optional illustrative example surfaced alongside ``description`` - # in the LLM judge's rubric. Today this is mainly populated on - # child sub-labels (one example per categorization label) but - # standalone metrics may carry it too without a schema change. - example: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) - metric_type: MetricType = MetricType.RATING - metric_category: MetricCategory = MetricCategory.QUALITY - trigger: MetricTrigger = MetricTrigger.ALWAYS - enabled: bool = True - metric_origin: str = "custom" - supported_surfaces: List[str] = ["agent"] - enabled_surfaces: Optional[List[str]] = None - custom_data_type: Optional[str] = None - custom_config: Optional[Dict[str, Any]] = None - tags: Optional[List[str]] = None - capture_rationale: Optional[bool] = False - parent_metric_id: Optional[UUID] = None - selection_mode: Optional[SelectionMode] = None - # Only meaningful on multi_label parents; ignored everywhere else. - # When true, the LLM is invited during call-import evaluation to emit - # additional candidate sub-labels beyond the user-defined children. - allow_discovery: bool = False - # When true, this metric is a "transcript-compare judge": at - # call-import evaluation time the worker feeds BOTH the production - # transcript (``call_import_rows.transcript``, CSV-supplied) and - # the diarised transcript (``call_import_rows.diarised_transcript``, - # worker-produced) to the LLM as a labeled pair. The parent - # evaluation's ``transcript_source`` is ignored for these metrics. - # Mutually exclusive with ``parent_metric_id`` / ``selection_mode`` - # G�� comparison metrics stay standalone so the LLM grouping logic - # doesn't have to second-guess which prompt template to use within - # a hierarchy. (Parent-level keyword auto-detection in the worker - # still routes a categorisation parent through the comparison - # prompt without setting this flag.) - compare_transcripts: bool = False - # When ``"organization"``, the metric is stored with - # ``workspace_id=NULL`` so it surfaces in every workspace of the - # caller's org. Default ``"workspace"`` preserves the historical - # behavior of stamping the metric with the active ``X-Workspace-Id``. - # Ignored when ``parent_metric_id`` is set (children inherit the - # parent's scope unconditionally). - scope: MetricScope = "workspace" - - @model_validator(mode='after') - def validate_compare_transcripts_exclusions(self): - """Reject body combinations that don't make sense for a - transcript-compare judge. - - The Metric ORM column accepts the value; the validator just - prevents the user from accidentally requesting an incoherent - metric shape (e.g. "compare two transcripts but also live - inside a categorisation hierarchy" � different prompt - templates). - """ - if not self.compare_transcripts: - return self - if self.parent_metric_id is not None: - raise ValueError( - "Transcript-compare metrics must be standalone: " - "they cannot be a child sub-metric in this version." - ) - if self.selection_mode is not None: - raise ValueError( - "Transcript-compare metrics must be standalone: " - "they cannot own children (selection_mode must be " - "unset) in this version." - ) - return self - - model_config = ConfigDict(json_schema_extra={ - "example": { - "name": "Professionalism", - "description": "Measures the professional tone and behavior", - "metric_type": "rating", - "trigger": "always", - "enabled": True - } - }) - - -class MetricChildDraft(BaseModel): - """One child sub-metric in a parent + children atomic create body.""" - - name: str = Field(..., max_length=120) - description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) - # Optional illustrative example for this label. Surfaced alongside - # ``description`` in the LLM judge's rubric so each label can carry - # both its definition AND a "what does this look like?" example. - example: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) - enabled: bool = True - capture_rationale: Optional[bool] = True - tags: Optional[List[str]] = None - - -class MetricCreateWithChildren(BaseModel): - """One-shot create body: a parent metric + N children, atomically. - - Children are persisted as full ``Metric`` rows with - ``parent_metric_id`` set to the new parent. ``metric_type`` on every - child is forced to ``boolean`` server-side regardless of what's - passed in the parent body. - """ - - name: str = Field(..., max_length=120) - description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) - selection_mode: SelectionMode - metric_category: MetricCategory = MetricCategory.QUALITY - enabled: bool = True - supported_surfaces: List[str] = Field(default_factory=lambda: ["agent"]) - enabled_surfaces: Optional[List[str]] = None - tags: Optional[List[str]] = None - # When true on a multi_label parent, allow the LLM to emit candidate - # labels beyond the listed children at evaluation time. Validator - # rejects allow_discovery=True on single_choice parents. - allow_discovery: bool = False - # Parent-level "Enable LLM Rationale" toggle. When true the LLM - # judge emits a single rationale string at the parent level - # (children never carry rationales in hierarchical mode), which the - # table renders as the " - LLM Rationale" column. - capture_rationale: bool = False - children: List[MetricChildDraft] = Field( - default_factory=list, - description="Child sub-metric labels under this parent.", - ) - # See ``MetricCreate.scope``. Same semantics: ``"organization"`` - # creates the parent + all children with ``workspace_id=NULL`` so - # the whole category subtree is shared across every workspace in - # the org. - scope: MetricScope = "workspace" - - -class MetricUpdate(BaseModel): - """Schema for updating a metric.""" - name: Optional[str] = None - description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) - # ``None`` here means "leave unchanged"; pass an empty string to - # clear a previously stored example. - example: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) - metric_type: Optional[MetricType] = None - trigger: Optional[MetricTrigger] = None - enabled: Optional[bool] = None - metric_origin: Optional[str] = None - supported_surfaces: Optional[List[str]] = None - enabled_surfaces: Optional[List[str]] = None - custom_data_type: Optional[str] = None - custom_config: Optional[Dict[str, Any]] = None - tags: Optional[List[str]] = None - metric_category: Optional[MetricCategory] = None - capture_rationale: Optional[bool] = None - selection_mode: Optional[SelectionMode] = None - allow_discovery: Optional[bool] = None - # See ``MetricCreate.compare_transcripts``. ``None`` here means - # "leave unchanged". The route layer enforces mutual exclusion - # against the row's existing ``parent_metric_id`` / - # ``selection_mode`` when this is set to True, because the patch - # body alone doesn't have enough context to validate cross-state. - compare_transcripts: Optional[bool] = None - - @model_validator(mode='after') - def validate_compare_transcripts_exclusions(self): - """Reject patch bodies that flip compare_transcripts on while - ALSO trying to set a conflicting field in the same request. - - Cross-state validation against the persisted row (e.g. "the - existing metric already has a parent") is done in the - update route since the schema doesn't have the row in hand. - """ - if self.compare_transcripts is not True: - return self - if self.selection_mode is not None: - raise ValueError( - "Transcript-compare metrics must be standalone: " - "selection_mode must be cleared before enabling " - "compare_transcripts." - ) - return self - - -class MetricResponse(BaseModel): - """Schema for metric response. - - ``children`` is populated for parent metrics (those with - ``selection_mode`` set) and is otherwise an empty list. The list is - built once at serialization time so callers get a single tree - structure without follow-up requests. - """ - id: UUID - organization_id: UUID - # ``None`` when the metric is org-shared (``scope == "organization"``). - # See the ORM ``Metric.workspace_id`` docstring. - workspace_id: Optional[UUID] = None - # Computed convenience field so the UI doesn't have to do - # ``workspace_id == null`` checks everywhere. Always one of - # ``"workspace"`` or ``"organization"``. - scope: MetricScope = "workspace" - name: str - description: Optional[str] - # Optional illustrative example. Populated mainly on categorization - # child labels but surfaced for every metric so the UI can render - # it uniformly without branching on parent/child shape. - example: Optional[str] = None - metric_type: MetricType - metric_category: MetricCategory = MetricCategory.QUALITY - trigger: MetricTrigger - enabled: bool - is_default: bool - metric_origin: str - supported_surfaces: List[str] - enabled_surfaces: List[str] - custom_data_type: Optional[str] - custom_config: Optional[Dict[str, Any]] - tags: Optional[List[str]] - capture_rationale: bool = False - parent_metric_id: Optional[UUID] = None - selection_mode: Optional[SelectionMode] = None - allow_discovery: bool = False - # See ``MetricCreate.compare_transcripts``. Surfaced so the UI can - # render a "Compare transcripts" badge in the metric picker and - # know to skip the run's transcript_source toggle for this metric. - compare_transcripts: bool = False - children: List["MetricResponse"] = Field(default_factory=list) - created_at: datetime - updated_at: datetime - created_by: Optional[str] - - @field_validator('metric_type', mode='before') - @classmethod - def convert_metric_type(cls, v): - """Convert string to MetricType (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return MetricType(v_lower) - except ValueError: - for enum_member in MetricType: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid MetricType value: {v}") - return v - - @field_validator('trigger', mode='before') - @classmethod - def convert_trigger(cls, v): - """Convert string to MetricTrigger (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return MetricTrigger(v_lower) - except ValueError: - for enum_member in MetricTrigger: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid MetricTrigger value: {v}") - return v - - @validator('supported_surfaces', 'enabled_surfaces', pre=True) - def normalize_surfaces(cls, v): - if v is None: - return [] - if isinstance(v, str): - return [v] - if isinstance(v, (list, tuple)): - return [str(item).lower() for item in v if item] - return [] - - @validator('metric_origin', pre=True) - def normalize_metric_origin(cls, v): - if v is None: - return "custom" - return str(v).lower() - - model_config = ConfigDict(from_attributes=True) - - -MetricResponse.model_rebuild() - - -# Evaluator Result Schemas -class EvaluatorResultCreate(BaseModel): - """Schema for creating an evaluator result.""" - evaluator_id: UUID - agent_id: Optional[UUID] = None - persona_id: Optional[UUID] = None - scenario_id: Optional[UUID] = None - name: Optional[str] = None - duration_seconds: Optional[float] = None - audio_s3_key: Optional[str] = None - - -class EvaluatorResultCreateManual(BaseModel): - """Schema for manually creating an evaluator result from existing audio file.""" - evaluator_id: UUID - audio_s3_key: str - duration_seconds: Optional[float] = None - - -class EvaluatorResultUpdate(BaseModel): - """Schema for updating an evaluator result.""" - status: Optional[EvaluatorResultStatus] = None - transcription: Optional[str] = None - metric_scores: Optional[Dict[str, Any]] = None - error_message: Optional[str] = None - duration_seconds: Optional[float] = None - - - -class EvaluatorResultCounts(BaseModel): - """Rollup counts for evaluator result navigation.""" - - total: int = 0 - completed: int = 0 - failed: int = 0 - in_progress: int = 0 - last_run_at: Optional[datetime] = None - - - - -class EvaluatorResultsScenarioSummary(BaseModel): - scenario_id: UUID - scenario_name: str - counts: EvaluatorResultCounts - - - - -class EvaluatorResultsSuiteSummary(BaseModel): - suite_id: UUID - suite_name: Optional[str] = None - agent_id: UUID - persona_id: Optional[UUID] = None - counts: EvaluatorResultCounts - scenarios: Optional[List["EvaluatorResultsScenarioSummary"]] = None - - - - -class EvaluatorResultsAgentSummary(BaseModel): - agent_id: UUID - agent_name: str - counts: EvaluatorResultCounts - suites: Optional[List[EvaluatorResultsSuiteSummary]] = None - - - - -class EvaluatorResultsUnassignedSummary(BaseModel): - counts: EvaluatorResultCounts - recent_result_ids: List[str] = Field(default_factory=list) - - - - -class EvaluatorResultsOverviewResponse(BaseModel): - workspace_counts: EvaluatorResultCounts - agents: List[EvaluatorResultsAgentSummary] = Field(default_factory=list) - unassigned: EvaluatorResultsUnassignedSummary - - - - -class EvaluatorResultListResponse(BaseModel): - items: List["EvaluatorResultResponse"] - total: int - - - -class EvaluatorResultResponse(BaseModel): - """Schema for evaluator result response.""" - id: UUID - result_id: str - organization_id: UUID - evaluator_id: Optional[UUID] = None # Optional for playground test results - agent_id: Optional[UUID] = None # Nullable for custom evaluators - persona_id: Optional[UUID] = None # Optional for playground test results - scenario_id: Optional[UUID] = None # Optional for playground test results - name: Optional[str] = None # Optional for playground test results - timestamp: datetime - duration_seconds: Optional[float] - status: EvaluatorResultStatus - audio_s3_key: Optional[str] - transcription: Optional[str] - speaker_segments: Optional[List[Dict[str, Any]]] = None # [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] - metric_scores: Optional[Dict[str, Any]] - celery_task_id: Optional[str] - error_message: Optional[str] - - # Call tracking fields (for voice AI integrations) - call_event: Optional[str] = None - provider_call_id: Optional[str] = None - provider_platform: Optional[str] = None - call_data: Optional[Dict[str, Any]] = None # Full call details from provider - - created_at: datetime - updated_at: datetime - created_by: Optional[str] - - # Related entities (optional, populated when requested) - agent: Optional[AgentResponse] = None - persona: Optional[PersonaResponse] = None - scenario: Optional[ScenarioResponse] = None - evaluator: Optional[EvaluatorResponse] = None - - @field_validator('status', mode='before') - @classmethod - def convert_status(cls, v): - """Convert string to EvaluatorResultStatus (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return EvaluatorResultStatus(v_lower) - except ValueError: - for enum_member in EvaluatorResultStatus: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid EvaluatorResultStatus value: {v}") - return v - - model_config = ConfigDict(from_attributes=True) - - -# ============================================ -# ALERTING SCHEMAS -# ============================================ - -class AlertCreate(BaseModel): - """Schema for creating an alert.""" - name: str = Field(..., min_length=1, max_length=255) - description: Optional[str] = None - - # Metric condition - metric_type: AlertMetricType = AlertMetricType.NUMBER_OF_CALLS - aggregation: AlertAggregation = AlertAggregation.SUM - operator: AlertOperator = AlertOperator.GREATER_THAN - threshold_value: float = Field(..., description="Threshold value for the alert") - time_window_minutes: int = Field(default=60, ge=1, description="Time window in minutes for aggregation") - - # Agent selection (null means all agents) - agent_ids: Optional[List[UUID]] = None - - # Notification settings - notify_frequency: AlertNotifyFrequency = AlertNotifyFrequency.IMMEDIATE - notify_emails: Optional[List[str]] = Field(default=None, description="List of email addresses to notify") - notify_webhooks: Optional[List[str]] = Field(default=None, description="List of webhook URLs (Slack, etc.)") - - model_config = ConfigDict(json_schema_extra={ - "example": { - "name": "High Call Volume Alert", - "description": "Alert when call volume exceeds threshold", - "metric_type": "number_of_calls", - "aggregation": "sum", - "operator": ">", - "threshold_value": 100, - "time_window_minutes": 60, - "agent_ids": None, - "notify_frequency": "immediate", - "notify_emails": ["admin@example.com"], - "notify_webhooks": ["https://hooks.slack.com/services/xxx"] - } - }) - - -class AlertUpdate(BaseModel): - """Schema for updating an alert.""" - name: Optional[str] = Field(None, min_length=1, max_length=255) - description: Optional[str] = None - - # Metric condition - metric_type: Optional[AlertMetricType] = None - aggregation: Optional[AlertAggregation] = None - operator: Optional[AlertOperator] = None - threshold_value: Optional[float] = None - time_window_minutes: Optional[int] = Field(default=None, ge=1) - - # Agent selection - agent_ids: Optional[List[UUID]] = None - - # Notification settings - notify_frequency: Optional[AlertNotifyFrequency] = None - notify_emails: Optional[List[str]] = None - notify_webhooks: Optional[List[str]] = None - - # Status - status: Optional[AlertStatus] = None - - -class AlertResponse(BaseModel): - """Schema for alert response.""" - id: UUID - organization_id: UUID - name: str - description: Optional[str] - - # Metric condition - metric_type: AlertMetricType - aggregation: AlertAggregation - operator: AlertOperator - threshold_value: float - time_window_minutes: int - - # Agent selection - agent_ids: Optional[List[UUID]] - - # Notification settings - notify_frequency: AlertNotifyFrequency - notify_emails: Optional[List[str]] - notify_webhooks: Optional[List[str]] - - # Status - status: AlertStatus - - # Metadata - created_at: datetime - updated_at: datetime - created_by: Optional[str] - - @field_validator('metric_type', mode='before') - @classmethod - def convert_metric_type(cls, v): - """Convert string to AlertMetricType.""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return AlertMetricType(v_lower) - except ValueError: - for enum_member in AlertMetricType: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid AlertMetricType value: {v}") - return v - - @field_validator('aggregation', mode='before') - @classmethod - def convert_aggregation(cls, v): - """Convert string to AlertAggregation.""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return AlertAggregation(v_lower) - except ValueError: - for enum_member in AlertAggregation: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid AlertAggregation value: {v}") - return v - - @field_validator('operator', mode='before') - @classmethod - def convert_operator(cls, v): - """Convert string to AlertOperator.""" - if v is None: - return None - if isinstance(v, str): - try: - return AlertOperator(v) - except ValueError: - for enum_member in AlertOperator: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid AlertOperator value: {v}") - return v - - @field_validator('notify_frequency', mode='before') - @classmethod - def convert_notify_frequency(cls, v): - """Convert string to AlertNotifyFrequency.""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return AlertNotifyFrequency(v_lower) - except ValueError: - for enum_member in AlertNotifyFrequency: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid AlertNotifyFrequency value: {v}") - return v - - @field_validator('status', mode='before') - @classmethod - def convert_status(cls, v): - """Convert string to AlertStatus.""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return AlertStatus(v_lower) - except ValueError: - for enum_member in AlertStatus: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid AlertStatus value: {v}") - return v - - model_config = ConfigDict(from_attributes=True) - - -class AlertHistoryResponse(BaseModel): - """Schema for alert history response.""" - id: UUID - organization_id: UUID - alert_id: UUID - - # Trigger information - triggered_at: datetime - triggered_value: float - threshold_value: float - - # Status - status: AlertHistoryStatus - - # Notification tracking - notified_at: Optional[datetime] - notification_details: Optional[Dict[str, Any]] - - # Resolution - acknowledged_at: Optional[datetime] - acknowledged_by: Optional[str] - resolved_at: Optional[datetime] - resolved_by: Optional[str] - resolution_notes: Optional[str] - - # Additional context - context_data: Optional[Dict[str, Any]] - - # Metadata - created_at: datetime - updated_at: datetime - - # Related alert info (optional) - alert: Optional[AlertResponse] = None - - @field_validator('status', mode='before') - @classmethod - def convert_status(cls, v): - """Convert string to AlertHistoryStatus.""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return AlertHistoryStatus(v_lower) - except ValueError: - for enum_member in AlertHistoryStatus: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid AlertHistoryStatus value: {v}") - return v - - model_config = ConfigDict(from_attributes=True) - - -class AlertHistoryUpdate(BaseModel): - """Schema for updating alert history (acknowledge/resolve).""" - status: Optional[AlertHistoryStatus] = None - acknowledged_by: Optional[str] = None - resolved_by: Optional[str] = None - resolution_notes: Optional[str] = None - - -# ============================================ -# CRON JOB SCHEMAS -# ============================================ - -class CronJobCreate(BaseModel): - """Schema for creating a cron job.""" - name: str = Field(..., min_length=1, max_length=255) - cron_expression: str = Field(..., min_length=1, max_length=100, description="Cron expression (e.g., '0 9 * * 1-5')") - timezone: str = Field(default="UTC", max_length=100, description="Timezone for the cron schedule") - max_runs: int = Field(default=10, ge=1, le=1000, description="Maximum number of times to run") - evaluator_ids: Optional[List[UUID]] = Field( - None, - description="Evaluator IDs to trigger (expanded with evaluator_suite_ids when both are set).", - ) - evaluator_suite_ids: Optional[List[UUID]] = Field( - None, - description="Evaluator suite IDs whose combinations are expanded into evaluator_ids.", - ) - - model_config = ConfigDict(json_schema_extra={ - "example": { - "name": "Daily Evaluation Run", - "cron_expression": "0 9 * * 1-5", - "timezone": "America/New_York", - "max_runs": 100, - "evaluator_ids": ["123e4567-e89b-12d3-a456-426614174000"] - } - }) - - -class CronJobUpdate(BaseModel): - """Schema for updating a cron job.""" - name: Optional[str] = Field(None, min_length=1, max_length=255) - cron_expression: Optional[str] = Field(None, min_length=1, max_length=100) - timezone: Optional[str] = Field(None, max_length=100) - max_runs: Optional[int] = Field(None, ge=1, le=1000) - evaluator_ids: Optional[List[UUID]] = None - evaluator_suite_ids: Optional[List[UUID]] = None - status: Optional[CronJobStatus] = None - - -class CronJobResponse(BaseModel): - """Schema for cron job response.""" - id: UUID - organization_id: UUID - name: str - cron_expression: str - timezone: str - max_runs: int - current_runs: int - evaluator_ids: List[UUID] - status: CronJobStatus - next_run_at: Optional[datetime] - last_run_at: Optional[datetime] - created_at: datetime - updated_at: datetime - created_by: Optional[str] - - @field_validator('status', mode='before') - @classmethod - def convert_status(cls, v): - """Convert string to CronJobStatus.""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return CronJobStatus(v_lower) - except ValueError: - for enum_member in CronJobStatus: - if enum_member.value.lower() == v_lower: - return enum_member - raise ValueError(f"Invalid status: {v}") - return v - - @field_validator('evaluator_ids', mode='before') - @classmethod - def convert_evaluator_ids(cls, v): - """Convert evaluator_ids from JSON to list of UUIDs.""" - if v is None: - return [] - if isinstance(v, list): - return [UUID(str(id)) if not isinstance(id, UUID) else id for id in v] - return v - - model_config = ConfigDict(from_attributes=True) - - -# ============================================ -# PROMPT PARTIAL SCHEMAS -# ============================================ - -class MetricPartialChild(BaseModel): - """One categorization label inside a metric partial.""" - - name: str = Field(..., min_length=1) - description: str = "" - example: str = "" - - -class MetricPartialContent(BaseModel): - """Structured JSON payload stored in metric partial ``content``.""" - - schema_version: int = 1 - metric_kind: Literal["single", "category"] - description: str = "" - children: Optional[List[MetricPartialChild]] = None - - -class PromptPartialCreate(BaseModel): - """Schema for creating a prompt partial.""" - name: str = Field(..., min_length=1, max_length=255) - description: Optional[str] = None - content: str = Field(..., min_length=1) - tags: Optional[List[str]] = None - - -class PromptPartialUpdate(BaseModel): - """Schema for updating a prompt partial.""" - name: Optional[str] = Field(None, min_length=1, max_length=255) - description: Optional[str] = None - content: Optional[str] = Field(None, min_length=1) - tags: Optional[List[str]] = None - change_summary: Optional[str] = None - - -class PromptPartialVersionResponse(BaseModel): - """Schema for prompt partial version response.""" - id: UUID - prompt_partial_id: UUID - version: int - content: str - change_summary: Optional[str] - created_at: datetime - created_by: Optional[str] - - model_config = ConfigDict(from_attributes=True) - - -class AgentFlowNode(BaseModel): - """One step in an LLM-inferred agent logic flowchart.""" - - id: str - label: str - node_type: Literal["start", "decision", "action", "terminal"] = "action" - position_x: Optional[float] = None - position_y: Optional[float] = None - prompt_excerpt: Optional[str] = None - start_offset: Optional[int] = None - end_offset: Optional[int] = None - - -class AgentFlowEdge(BaseModel): - """Directed transition between two agent flow nodes.""" - - source: str - target: str - condition: Optional[str] = None - - -class AgentFlowNodeLayout(BaseModel): - id: str - position_x: float - position_y: float - - -class AgentFlowLayoutSaveRequest(BaseModel): - nodes: List[AgentFlowNodeLayout] = Field(default_factory=list) - - -class AgentFlowGraph(BaseModel): - """Aggregate flow diagram for an imported production agent prompt.""" - - nodes: List[AgentFlowNode] = Field(default_factory=list) - edges: List[AgentFlowEdge] = Field(default_factory=list) - generated_at: Optional[datetime] = None - provider: Optional[str] = None - model: Optional[str] = None - layout_saved_at: Optional[datetime] = None - prompt_content_hash: Optional[str] = None - mapping_error: Optional[str] = None - generation_error: Optional[str] = None - - -class PromptPartialResponse(BaseModel): - """Schema for prompt partial response.""" - id: UUID - organization_id: UUID - name: str - description: Optional[str] - content: str - tags: Optional[List[str]] - current_version: int - agent_flowchart: Optional[AgentFlowGraph] = None - agent_flowchart_status: Optional[str] = None - created_at: datetime - updated_at: datetime - created_by: Optional[str] - - model_config = ConfigDict(from_attributes=True) - - -class PromptPartialDetailResponse(PromptPartialResponse): - """Schema for prompt partial detail with versions.""" - versions: List[PromptPartialVersionResponse] = [] - - model_config = ConfigDict(from_attributes=True) - - -# ============================================ -# TELEPHONY SCHEMAS (provider-agnostic) -# ============================================ - - -class TelephonyIntegrationCreate(BaseModel): - """Schema for creating a telephony provider integration.""" - - provider: str = "plivo" - name: Optional[str] = None - auth_id: str - auth_token: str - verify_app_uuid: Optional[str] = None - voice_app_id: Optional[str] = None - sip_domain: Optional[str] = None - masking_config: Optional[Dict[str, Any]] = None - is_default: Optional[bool] = Field( - None, - description=( - "Mark this credential as the default for the (org, provider). " - "If omitted and no default exists yet, this row becomes the default." - ), - ) - - -class TelephonyIntegrationUpdate(BaseModel): - """Schema for partial updates to a telephony provider integration.""" - - id: Optional[UUID] = None - provider: Optional[str] = None - name: Optional[str] = None - auth_id: Optional[str] = None - auth_token: Optional[str] = None - verify_app_uuid: Optional[str] = None - voice_app_id: Optional[str] = None - sip_domain: Optional[str] = None - masking_config: Optional[Dict[str, Any]] = None - is_active: Optional[bool] = None - - -class TelephonyIntegrationResponse(BaseModel): - """Safe response model for telephony integration without secrets.""" - - id: UUID - organization_id: UUID - provider: str - name: Optional[str] = None - verify_app_uuid: Optional[str] - voice_app_id: Optional[str] - sip_domain: Optional[str] - masking_config: Optional[Dict[str, Any]] - is_active: bool - is_default: bool = False - last_tested_at: Optional[datetime] - created_at: datetime - updated_at: datetime - - class Config: - from_attributes = True - - -class TelephonyPhoneNumberResponse(BaseModel): - """Telephony phone number inventory response schema.""" - - id: UUID - phone_number: str - country_iso2: Optional[str] - region: Optional[str] - number_type: Optional[str] - capabilities: Optional[Dict[str, Any]] - is_masking_pool: bool - inbound_enabled: Optional[bool] = None - outbound_enabled: Optional[bool] = None - source: Optional[str] = None - agent_id: Optional[UUID] - linked_agent_name: Optional[str] = None - provider: Optional[str] = None - is_active: bool - created_at: datetime - - class Config: - from_attributes = True - - -class TelephonyDialTargetCreate(BaseModel): - """Schema for creating a saved outbound dial target.""" - phone_number: str - label: Optional[str] = None - - -class TelephonyDialTargetUpdate(BaseModel): - """Schema for updating a saved outbound dial target.""" - phone_number: Optional[str] = None - label: Optional[str] = None - - -class TelephonyDialTargetResponse(BaseModel): - """Schema for dial target response.""" - id: UUID - phone_number: str - label: Optional[str] = None - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class TelephonyVerifyStartRequest(BaseModel): - """Request schema for starting voice OTP verification.""" - - phone_number: str - provider: str = "plivo" - - -class TelephonyVerifyStartResponse(BaseModel): - """Response schema for started voice OTP verification.""" - - session_id: UUID - provider_session_uuid: str - status: str - message: str - - -class TelephonyVerifyCheckRequest(BaseModel): - """Request schema for checking a submitted OTP code.""" - - session_id: UUID - otp_code: str - provider: str = "plivo" - - -class TelephonyVerifyCheckResponse(BaseModel): - """Response schema for OTP check status.""" - - verified: bool - status: str - message: str - - -class TelephonyMaskingSessionCreate(BaseModel): - """Request schema for creating a number masking session.""" - - party_a_number: str - party_b_number: str - provider: str = "plivo" - expires_in_minutes: Optional[int] = 60 - metadata: Optional[Dict[str, Any]] = None - provider: str = "plivo" - - -class TelephonyMaskingSessionResponse(BaseModel): - """Response schema for masking sessions.""" - - id: UUID - masked_number: str - party_a_number: str - party_b_number: str - status: str - expires_at: Optional[datetime] - created_at: datetime - - class Config: - from_attributes = True - - -class TelephonyOutboundCallRequest(BaseModel): - """Request schema for outbound call initiation.""" - - from_number: str - to_number: str - answer_url: Optional[str] = None - agent_id: Optional[UUID] = None - - -class TelephonyOutboundCallResponse(BaseModel): - """Response schema for outbound call initiation.""" - - provider_request_uuid: str - call_status: str - from_number: str - to_number: str - message: str - - -# --- Call Import Schemas --- - -class CallImportRowResponse(BaseModel): - """Single row within a call-import batch.""" - - id: UUID - row_index: int - # Renamed from ``external_call_id`` (DB column renamed in migration - # ``034_call_import_schemas``). Same data, same uniqueness rules. - conversation_id: str - recording_url: Optional[str] = None - recording_date: Optional[date] = None - # Production transcript: the value supplied via the CSV upload. - transcript: Optional[str] = None - transcript_source: Optional[str] = None - transcript_provider: Optional[str] = None - transcript_model: Optional[str] = None - transcript_status: Optional[str] = None - transcript_error: Optional[str] = None - transcribed_at: Optional[datetime] = None - # Diarised transcript: produced by the post-hoc diarisation - # worker. Independent of ``transcript`` so manual diarisation - # never overwrites the CSV-supplied production value. - diarised_transcript: Optional[str] = None - diarised_transcript_provider: Optional[str] = None - diarised_transcript_model: Optional[str] = None - diarised_transcript_status: Optional[str] = None - diarised_transcript_error: Optional[str] = None - diarised_at: Optional[datetime] = None - # LLM that turned the STT plain-text output into structured - # ``diarised_segments``. Surfaced in the row detail panel so - # reviewers can see "Diarised by openai/gpt-4o-mini" next to - # the swap toggle. NULL on rows diarised by the legacy pyannote - # worker (which has been removed). - diarised_llm_provider: Optional[str] = None - diarised_llm_model: Optional[str] = None - # The exact prompt the LLM diariser ran with. Persisted so a - # reviewer can copy it back into the modal and reproduce the - # turn layout against a different STT pass. - diarised_prompt: Optional[str] = None - # Structured speaker turns produced by the diarisation worker. Each - # entry is `{ "speaker": "agent"|"user"|"speaker_N", "text": str, - # "start": float, "end": float, "raw_speaker": "Speaker 1" }`. The - # plain ``diarised_transcript`` field above is a `: ` - # rendering of this list with ``diarised_speaker_swap`` applied. - diarised_segments: Optional[List[Dict[str, Any]]] = None - # When True the agent <-> user mapping in ``diarised_segments`` is - # inverted at render / export time. The worker writes the canonical - # mapping using the "first speaker is the agent" heuristic; the swap - # toggle lets reviewers correct that without re-running diarisation. - diarised_speaker_swap: bool = False - status: CallImportRowStatus - recording_s3_key: Optional[str] = None - recording_content_type: Optional[str] = None - recording_size_bytes: Optional[int] = None - error_message: Optional[str] = None - attempts: int - raw_columns: Optional[Dict[str, Any]] = None - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -# --- Call Import Schema (Input Parameter definitions) --- - - -class CallImportSchemaParameterBase(BaseModel): - """A single typed parameter inside a Call Import schema. - - Used both in request bodies (create / update) and as the building - block of :class:`CallImportSchemaParameterResponse`. Names are - case-insensitive unique within their parent schema. - """ - - name: str = Field( - ..., - min_length=1, - max_length=255, - description=( - "Parameter name as it appears in the schema editor and the " - "upload mapping table. Must be unique within the schema " - "(case-insensitive)." - ), - ) - type: CallImportParameterType = Field( - ..., - description=( - "Parameter type. One of conversation_id / recording_url / " - "recording_date / transcript / text / number / boolean / " - "datetime / url. Exactly one parameter of type " - "'conversation_id' and exactly one of type 'recording_url' " - "must be present; at most one each of 'recording_date' and " - "'transcript'. Both conversation_id and recording_url are " - "forced required." - ), - ) - description: Optional[str] = Field( - default=None, - max_length=2048, - description="Free-text help shown next to the parameter in the mapping UI.", - ) - is_required: bool = Field( - default=False, - description=( - "When True, the parameter must be mapped to a CSV column on " - "every upload. The ``conversation_id`` and ``recording_url`` " - "parameters are always required and are force-set to True by " - "the server." - ), - ) - - -class CallImportSchemaParameterCreate(CallImportSchemaParameterBase): - """Create payload for a single parameter (inside a schema CRUD body).""" - - -class CallImportSchemaParameterResponse(CallImportSchemaParameterBase): - """Response shape including the persisted id + ordering.""" - - id: UUID - ordering: int - - model_config = ConfigDict(from_attributes=True) - - -def _validate_schema_parameters( - parameters: List[CallImportSchemaParameterBase], -) -> List[CallImportSchemaParameterBase]: - """Apply the cross-parameter invariants shared by create + update.""" - - if not parameters: - raise ValueError("Schema must define at least one parameter.") - - seen_names: set[str] = set() - conv_count = 0 - recording_date_count = 0 - rec_url_count = 0 - transcript_count = 0 - for param in parameters: - norm = param.name.strip().lower() - if not norm: - raise ValueError("Parameter name must be non-empty.") - if norm in seen_names: - raise ValueError( - f"Duplicate parameter name '{param.name}' " - "(names must be unique within a schema)." - ) - seen_names.add(norm) - if param.type == CallImportParameterType.CONVERSATION_ID: - conv_count += 1 - elif param.type == CallImportParameterType.RECORDING_DATE: - recording_date_count += 1 - elif param.type == CallImportParameterType.RECORDING_URL: - rec_url_count += 1 - elif param.type == CallImportParameterType.TRANSCRIPT: - transcript_count += 1 - - if conv_count != 1: - raise ValueError( - "Schema must contain exactly one parameter of type " - "'conversation_id'." - ) - if rec_url_count != 1: - raise ValueError( - "Schema must contain exactly one parameter of type " - "'recording_url'." - ) - if recording_date_count > 1: - raise ValueError( - "Schema may contain at most one parameter of type " - "'recording_date'." - ) - if rec_url_count > 1: - raise ValueError( - "Schema may contain at most one parameter of type 'recording_url'." - ) - if transcript_count > 1: - raise ValueError( - "Schema may contain at most one parameter of type 'transcript'." - ) - return parameters - - -class CallImportSchemaCreate(BaseModel): - """Create body for a new call-import schema.""" - - name: str = Field(..., min_length=1, max_length=255) - description: Optional[str] = Field(default=None, max_length=2048) - parameters: List[CallImportSchemaParameterCreate] = Field( - ..., - description=( - "Ordered list of parameters. Order is preserved; the server " - "stamps ``ordering`` from the list index." - ), - ) - - @model_validator(mode="after") - def _check_parameters(self): - _validate_schema_parameters(list(self.parameters)) - return self - - -class CallImportSchemaUpdate(BaseModel): - """Patch body for an existing schema (full parameter replacement).""" - - name: Optional[str] = Field(default=None, min_length=1, max_length=255) - description: Optional[str] = Field(default=None, max_length=2048) - parameters: Optional[List[CallImportSchemaParameterCreate]] = Field( - default=None, - description=( - "If provided, REPLACES the full set of parameters on the " - "schema. Omit to leave parameters untouched." - ), - ) - - @model_validator(mode="after") - def _check_parameters(self): - if self.parameters is not None: - _validate_schema_parameters(list(self.parameters)) - return self - - -class CallImportSchemaResponse(BaseModel): - """Read response for a single schema.""" - - id: UUID - organization_id: UUID - workspace_id: UUID - name: str - description: Optional[str] = None - parameters: List[CallImportSchemaParameterResponse] = Field(default_factory=list) - # How many CallImport batches reference this schema. Populated by the - # router when listing; defaults to 0 on detail responses where the - # caller doesn't need it. - usage_count: int = 0 - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class CallImportSchemaListResponse(BaseModel): - """Paginated list of schemas.""" - - items: List[CallImportSchemaResponse] = Field(default_factory=list) - total: int - - -class CallImportTagResponse(BaseModel): - """Tag attached to call import batches.""" - - id: UUID - name: str - color: Optional[str] = None - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class CallImportTagCreate(BaseModel): - """Create a new call-import tag for the organization.""" - - name: str = Field(..., min_length=1, max_length=255) - color: Optional[str] = Field(None, max_length=32) - - -class CallImportTagUpdate(BaseModel): - """Partial update for a call-import tag.""" - - name: Optional[str] = Field(None, min_length=1, max_length=255) - color: Optional[str] = Field(None, max_length=32) - - -class CallImportPreviewSheet(BaseModel): - """One worksheet (or one CSV file synthesized as a single sheet).""" - - name: str = Field(..., description="Sheet name for xlsx; filename for csv.") - headers: List[str] = Field( - default_factory=list, - description="Column headers from the first non-empty row.", - ) - row_count: int = Field( - ..., - description="Approximate count of data rows (excluding the header row).", - ) - - -class CallImportSourceRowSkip(BaseModel): - """One source spreadsheet row skipped during parse (identity / recording URL).""" - - source_row: int = Field( - ..., - description="1-based row index in the source file (same semantics as parse errors).", - ) - reason: str = Field( - ..., - description=( - "Machine-readable skip reason, e.g. missing_conversation_id, " - "missing_recording_url, invalid_recording_url." - ), - ) - message: str = Field( - ..., - description="Human-readable explanation shown in the UI.", - ) - - -class CallImportResponse(BaseModel): - """Summary of a call-import batch.""" - - id: UUID - organization_id: UUID - workspace_id: UUID - # Provider is optional in the new staged flow (only resolved at the - # IMPORT stage). Stays populated for all post-import batches. - provider: Optional[str] = None - telephony_integration_id: Optional[UUID] = None - original_filename: Optional[str] = None - sheet_name: Optional[str] = None - dataset: Optional[str] = None - tags: List[CallImportTagResponse] = Field(default_factory=list) - # New schema-driven mapping. Empty on legacy batches; pre-schema - # batches keep their values in ``column_mapping`` / ``extra_columns`` - # / ``custom_column_mapping`` below for backwards-compatibility. - schema_id: Optional[UUID] = None - parameter_mapping: Dict[str, str] = Field(default_factory=dict) - column_mapping: Dict[str, Optional[str]] = Field(default_factory=dict) - extra_columns: List[str] = Field(default_factory=list) - custom_column_mapping: Dict[str, str] = Field(default_factory=dict) - # Persisted "drop these columns" decision captured at MAP time. - # Empty for legacy one-shot uploads where the value was ephemeral. - skipped_columns: List[str] = Field(default_factory=list) - source_row_skips: List[CallImportSourceRowSkip] = Field( - default_factory=list, - description=( - "Source rows skipped at parse time because of missing/invalid " - "conversation ID or recording URL." - ), - ) - # Source-file staging fields populated at UPLOAD time. ``None`` on - # legacy batches imported via the one-shot ``POST /upload`` endpoint. - source_s3_key: Optional[str] = None - source_format: Optional[str] = None - source_size_bytes: Optional[int] = None - source_content_type: Optional[str] = None - # Snapshot of the file's sheets + headers captured at UPLOAD time - # so the MAP UI can render without re-fetching the file from S3. - available_sheets: Optional[List[CallImportPreviewSheet]] = None - total_rows: int - completed_rows: int - failed_rows: int - status: CallImportStatus - error_message: Optional[str] = None - created_at: datetime - updated_at: datetime - created_by_email: Optional[str] = None - last_updated_by_email: Optional[str] = None - - model_config = ConfigDict(from_attributes=True) - - -class CallImportDetailResponse(CallImportResponse): - """A call-import batch with its rows expanded. - - ``filtered_total_rows`` is only set when the caller passed a ``q`` - search term G�� it lets the UI paginate against the filtered subset - while still showing the unfiltered ``total_rows`` in the header. - - The ``diarised_*_rows`` counters aggregate - ``CallImportRow.diarised_transcript_status`` across the batch so the - UI can render a transcribe-and-diarise progress bar without paging - through every row. Rows that have never been touched by the - transcribe/diarise worker (``status='idle'``) are NOT counted here G�� - callers compute the idle bucket as - ``total_rows - (pending + running + completed + failed)``. - """ - - rows: List[CallImportRowResponse] = Field(default_factory=list) - filtered_total_rows: Optional[int] = None - diarised_pending_rows: int = 0 - diarised_running_rows: int = 0 - diarised_completed_rows: int = 0 - diarised_failed_rows: int = 0 - - -class CallImportListResponse(BaseModel): - """Paginated list of call-import batches.""" - - items: List[CallImportResponse] - total: int - page: int - page_size: int - - -class CallImportDispatchLimitSnapshot(BaseModel): - """Configured and live Redis in-flight caps for eval work.""" - - global_limit: int - global_inflight: int - global_at_capacity: bool - org_limit: int - org_inflight: int - org_at_capacity: bool - workspace_limit: int - job_limit: int - fair_dispatch_batch_size: int - - -class CallImportDispatchFairDispatchSnapshot(BaseModel): - """Fair-dispatch scheduler metadata from Redis.""" - - global_rr_cursor: int - dispatch_dedupe_active: bool - dispatch_queue: str - at_capacity_backoff_seconds: int - - -class CallImportDispatchEvaluationSnapshot(BaseModel): - """One in-flight evaluation run with row counters.""" - - evaluation_id: UUID - call_import_id: UUID - status: str - total_rows: int - pending_rows: int - running_rows: int - job_inflight: int - job_at_capacity: bool - - -class CallImportDispatchWorkspaceSnapshot(BaseModel): - """Per-workspace pending dispatch + slot usage.""" - - workspace_id: UUID - workspace_name: Optional[str] = None - workspace_slug: Optional[str] = None - inflight: int - inflight_at_capacity: bool - pending_dispatch_rows: int - pending_import_rows: int - eval_rr_cursor: int - active_evaluations: int - evaluations: List[CallImportDispatchEvaluationSnapshot] = Field( - default_factory=list - ) - - -class CallImportDispatchDiagnosticsResponse(BaseModel): - """Live operator snapshot for call-import eval fair dispatch.""" - - limits: CallImportDispatchLimitSnapshot - fair_dispatch: CallImportDispatchFairDispatchSnapshot - workspaces: List[CallImportDispatchWorkspaceSnapshot] - generated_at: datetime - - -class CallImportUploadResponse(BaseModel): - """Response returned right after a CSV is accepted.""" - - id: UUID - total_rows: int - status: CallImportStatus - dataset: Optional[str] = None - tags: List[CallImportTagResponse] = Field(default_factory=list) - message: str - - -class CallImportDeleteResponse(BaseModel): - """Response after a whole-batch call-import delete is accepted.""" - - id: UUID - status: Literal["accepted", "completed"] = Field( - ..., - description=( - "``accepted`` when teardown was queued to run asynchronously; " - "``completed`` when the batch was already removed." - ), - ) - - -class CallImportPreviewResponse(BaseModel): - """Sheets/headers extracted from an uploaded CSV or Excel workbook. - - The frontend uses this to drive the column-mapping UI without doing - its own parsing G�� keeps client and server in lockstep on quoted - fields, encodings, and Excel cell coercion. - """ - - format: str = Field(..., description="One of 'csv' or 'xlsx'.") - sheets: List[CallImportPreviewSheet] = Field(default_factory=list) - - -class CallImportUpdate(BaseModel): - """Partial update of a call-import batch.""" - - dataset: Optional[str] = Field( - None, - description=( - "Free-text dataset label. Pass an empty string to clear the dataset." - ), - ) - tag_ids: Optional[List[UUID]] = Field( - None, - description=( - "Replace the full set of tag assignments. Pass an empty list to clear all tags." - ), - ) - schema_id: Optional[UUID] = Field( - None, - description=( - "Reassign the Input Parameter schema. Only honoured while the " - "batch is in ``uploaded`` or ``mapped`` state; once the batch " - "has rows it's locked to its original schema." - ), - ) - - -class CallImportMappingUpdate(BaseModel): - """Mapping payload for the MAP stage (``PATCH /call-imports/{id}/mapping``). - - Idempotent: callers can submit this multiple times against an - ``uploaded`` or ``mapped`` batch. Validation re-runs against the - persisted ``available_sheets`` snapshot every time so the user can - correct mistakes without re-uploading the file. - """ - - schema_id: UUID = Field( - ..., - description=( - "Reusable Input Parameter schema this batch is mapped against. " - "Must belong to the active workspace." - ), - ) - sheet_name: Optional[str] = Field( - None, - description=( - "Worksheet to use when the staged source file is an Excel " - "workbook. REQUIRED for xlsx; ignored / rejected for CSV." - ), - ) - parameter_mapping: Dict[str, str] = Field( - default_factory=dict, - description=( - "``{schema_parameter_name: source_header}`` map covering every " - "required schema parameter." - ), - ) - skipped_columns: List[str] = Field( - default_factory=list, - description=( - "Source headers the uploader has explicitly skipped. Every " - "source header must be either mapped or appear here." - ), - ) - - -class CallImportStartRequest(BaseModel): - """Provider + credential picker for the IMPORT stage.""" - - provider: Optional[str] = Field( - default=None, - description=( - "Telephony provider key. Must match the " - "``telephony_integration_id``'s provider. Omit together with " - "``telephony_integration_id`` to download recordings directly " - "from CSV-supplied URLs without credentials." - ), - ) - telephony_integration_id: Optional[UUID] = Field( - default=None, - description=( - "Specific TelephonyIntegration credential row to use when " - "downloading recordings for this batch. Omit together with " - "``provider`` for direct-URL import." - ), - ) - - @model_validator(mode="after") - def validate_credential_mode(self) -> "CallImportStartRequest": - has_provider = bool((self.provider or "").strip()) - has_integration = self.telephony_integration_id is not None - if has_provider != has_integration: - raise ValueError( - "provider and telephony_integration_id must both be provided " - "or both omitted for direct-URL import." - ) - return self - - -# --- Call Import Evaluation Schemas --- - - -class CallImportEvaluationLLMOverride(BaseModel): - """Per-metric LLM override used on top of the run-level default. - - Any field left ``None`` falls back to the run-level value (which - itself falls back to the historical OpenAI/gpt-4o default). This - lets users pick a specific provider/model for a single metric (e.g. - a stronger Anthropic model for a tricky qualitative metric) without - re-typing the rest of the metrics in the run. - """ - - provider: Optional[str] = Field( - default=None, - max_length=50, - description="Override LLM provider key, e.g. 'openai' or 'anthropic'.", - ) - model: Optional[str] = Field( - default=None, - max_length=100, - description="Override LLM model name, e.g. 'gpt-4o' or 'claude-3-opus'.", - ) - credential_id: Optional[UUID] = Field( - default=None, - description="Optional AIProvider id when the org has multiple credentials.", - ) - llm_config: Optional[Dict[str, Any]] = Field( - default=None, - description="Optional per-metric generation parameters (temperature, top_p, etc.).", - ) - - -CallImportEvaluationTranscriptSource = Literal["production", "diarised"] - - -class CallImportEvaluationCreate(BaseModel): - """Request body for triggering an evaluation over a call-import batch.""" - - metric_ids: List[UUID] = Field( - ..., - min_length=1, - description="Org Metric ids to score every completed row against.", - ) - name: Optional[str] = Field( - default=None, - max_length=255, - description=( - "Optional human-readable label for the run. Shown in the UI " - "instead of the UUID prefix." - ), - ) - transcript_sources: List[CallImportEvaluationTranscriptSource] = Field( - default_factory=lambda: ["diarised"], - min_length=1, - max_length=1, - description=( - "Which transcript to score against. ``'diarised'`` (default) " - "auto-diarises rows missing a diarised transcript then scores " - "``diarised_transcript``. ``'production'`` scores the CSV " - "``transcript`` column directly and skips diarisation." - ), - ) - - @field_validator("transcript_sources") - @classmethod - def _validate_transcript_sources( - cls, value: List[str] - ) -> List["CallImportEvaluationTranscriptSource"]: - allowed = {"production", "diarised"} - invalid = [src for src in value if src not in allowed] - if invalid: - raise ValueError( - "transcript_sources must be ['production'] or ['diarised'] " - "(received: " - + ", ".join(repr(src) for src in invalid) - + ")." - ) - return value # type: ignore[return-value] - # --- Run-level LLM config --- - llm_provider: Optional[str] = Field( - default=None, - max_length=50, - description=( - "Run-level LLM provider key (e.g. 'openai', 'anthropic'). NULL " - "preserves the historical OpenAI/gpt-4o default." - ), - ) - llm_model: Optional[str] = Field( - default=None, - max_length=100, - description="Run-level LLM model name. Required when llm_provider is set.", - ) - llm_credential_id: Optional[UUID] = Field( - default=None, - description="Optional AIProvider row to pin for the run-level LLM.", - ) - llm_config: Optional[Dict[str, Any]] = Field( - default=None, - description="Run-level LLM generation parameters (temperature, top_p, etc.).", - ) - metric_llm_overrides: Optional[ - Dict[str, CallImportEvaluationLLMOverride] - ] = Field( - default=None, - description=( - "Optional per-metric LLM overrides keyed by metric UUID. Each " - "entry overrides the run-level default for that metric only." - ), - ) - # --- Auto-transcribe / diarization hook --- - # Every diarised run auto-diarises rows that don't already have a - # diarised transcript. The flag stays on the schema so legacy API - # callers don't 400 immediately, but the route now requires - # ``stt_provider`` + ``stt_model`` on every run regardless of this - # value. - auto_transcribe: bool = Field( - default=True, - description=( - "Auto-diarise rows missing a diarised transcript before " - "evaluation. Defaults to true and is effectively required: " - "``stt_provider`` + ``stt_model`` are mandatory on every " - "evaluation run." - ), - ) - transcribe_overwrite: bool = Field( - default=False, - description=( - "When auto_transcribe is on, overwrite existing transcripts " - "instead of skipping rows that already have one." - ), - ) - transcribe_mode: Literal["stt_llm", "llm_only"] = Field( - default="stt_llm", - description=( - "Diarisation pipeline shape for the auto-transcribe step. " - "'stt_llm' (default) runs STT then an LLM diariser over the " - "resulting text G�� ``stt_provider`` + ``stt_model`` must be " - "provided. 'llm_only' skips STT and feeds the audio " - "directly to the multimodal ``diarization_llm_*`` model " - "along with ``diarization_prompt``; STT fields must be " - "omitted in that case." - ), - ) - stt_provider: Optional[str] = Field( - default=None, - max_length=50, - description=( - "STT provider key, e.g. 'deepgram', 'openai'. Required when " - "``transcribe_mode='stt_llm'`` (the default); must be omitted " - "when ``transcribe_mode='llm_only'``." - ), - ) - stt_model: Optional[str] = Field( - default=None, - max_length=100, - description=( - "STT model name, e.g. 'nova-2', 'whisper-1'. Same presence " - "rules as ``stt_provider``." - ), - ) - stt_credential_id: Optional[UUID] = Field( - default=None, - description="Optional AIProvider/Integration row to pin for STT.", - ) - stt_language: Optional[str] = Field( - default=None, - max_length=20, - description="ISO language hint for the STT provider, e.g. 'en'.", - ) - # --- LLM diariser config (mirror of CallImportTranscribeRequest) --- - # Auto-diarised eval rows go through the same LLM-based diariser as - # the standalone Transcribe modal G�� the run remembers the provider / - # model / prompt so a follow-up retry can reproduce them without - # having to re-prompt the user. - diarization_llm_provider: Optional[str] = Field( - default=None, - max_length=50, - description=( - "LLM provider for diarising STT output into agent/user " - "turns. Required when ``auto_transcribe`` is set (the worker " - "no longer falls back to pyannote)." - ), - ) - diarization_llm_model: Optional[str] = Field( - default=None, - max_length=100, - description="LLM model for the diariser.", - ) - diarization_llm_credential_id: Optional[UUID] = Field( - default=None, - description="Optional AIProvider row to pin for the diariser LLM.", - ) - diarization_prompt: Optional[str] = Field( - default=None, - max_length=10_000, - description=( - "Custom system prompt for the diariser LLM; falls back to " - "the canonical default when blank." - ), - ) - discover_new_metrics: bool = Field( - default=False, - description=( - "When true, the LLM is invited to propose net-new top-level " - "metrics (boolean / rating / category) observed in the " - "transcripts in addition to scoring the selected metrics. " - "Candidates surface in the Discovered metrics panel on the " - "evaluation detail Flow tab and can be promoted into real " - "standalone Metric rows. Defaults to false so existing " - "callers retain previous behaviour." - ), - ) - # Telephony credentials for unified pipeline (required when batch is mapped). - provider: Optional[str] = Field( - default=None, - description=( - "Telephony provider key. Required together with " - "``telephony_integration_id`` when starting evaluation " - "from a mapped batch. Omit both for direct-URL import." - ), - ) - telephony_integration_id: Optional[UUID] = Field( - default=None, - description=( - "TelephonyIntegration credential for recording fetch. " - "Required together with ``provider`` for credentialed import." - ), - ) - - @model_validator(mode="after") - def validate_telephony_credential_mode(self) -> "CallImportEvaluationCreate": - has_provider = bool((self.provider or "").strip()) - has_integration = self.telephony_integration_id is not None - if has_provider != has_integration: - raise ValueError( - "provider and telephony_integration_id must both be provided " - "or both omitted for direct-URL evaluation." - ) - return self - - -class CallImportEvaluationUpdate(BaseModel): - """Patch body for editing a previously-created evaluation run.""" - - name: Optional[str] = Field( - default=None, - max_length=255, - description="New name for the evaluation. Empty string clears it.", - ) - - -class CallImportEvaluationBulkDelete(BaseModel): - """Request body for deleting multiple evaluation runs in one call.""" - - evaluation_ids: List[UUID] = Field( - ..., - min_length=1, - description="Evaluation ids to delete.", - ) - - -class CallImportEvaluationRetryRequest(BaseModel): - """Body for retrying a subset (or all failed rows) of an evaluation run. - - ``eval_row_ids`` is optional: when ``None`` the retry applies to - every row in the run that is currently in the ``failed`` state. The - selection always intersects with the run's actual rows, so unknown - ids are silently skipped (and surfaced in the response's - ``skipped`` list with reason ``unknown``). - - The optional ``llm_*`` / ``metric_llm_overrides`` / ``stt_*`` fields - let the caller swap out the LLM or STT configuration that the - failed rows were originally evaluated with. When a field is left - ``None`` the run's existing value is preserved. When a field is - set, it is persisted onto the run (so a follow-up retry sees the - new value as the default) and used by the worker on the next - pass. Providing only one half of provider+model is rejected so - the worker never ends up with a half-configured run. - """ - - eval_row_ids: Optional[List[UUID]] = Field( - default=None, - description=( - "Restrict the retry to a specific subset of evaluation rows. " - "When omitted, every row with status='failed' in this run is " - "re-enqueued." - ), - ) - - # --- Metric-subset re-run --- - # When ``metric_ids`` is set, the retry recomputes ONLY those - # metrics instead of the whole row, and the new scores are merged - # into the existing ``metric_scores`` JSON (other metrics' - # previously-computed values are preserved). This is the path - # taken by the "Re-run metrics" UI in CallImportEvaluationDetail. - metric_ids: Optional[List[UUID]] = Field( - default=None, - description=( - "Restrict the retry to a specific subset of metrics. When " - "set, the worker recomputes only these metrics and merges " - "the new scores into the row's existing metric_scores " - "(other metrics' previous values are preserved). When " - "omitted, the row is fully re-scored as before. Every id " - "must already be present in the run's selected_metric_ids." - ), - ) - include_completed: bool = Field( - default=False, - description=( - "When True, rows whose status is currently 'completed' " - "become eligible for retry (otherwise only 'failed' rows " - "are picked up). Required when ``metric_ids`` is set on a " - "successful row, since otherwise the whole metric-subset " - "retry would be skipped as 'completed'." - ), - ) - - # --- LLM overrides --- - llm_provider: Optional[str] = Field( - default=None, - max_length=50, - description=( - "Override the run-level LLM provider for this retry (and " - "future retries). Must be paired with ``llm_model``." - ), - ) - llm_model: Optional[str] = Field( - default=None, - max_length=100, - description=( - "Override the run-level LLM model. Must be paired with " - "``llm_provider``." - ), - ) - llm_credential_id: Optional[UUID] = Field( - default=None, - description=( - "Pin a specific AIProvider credential row for the LLM. " - "When omitted, the resolver falls back to the org default." - ), - ) - llm_config: Optional[Dict[str, Any]] = Field( - default=None, - description="Override run-level LLM generation parameters for this retry.", - ) - metric_llm_overrides: Optional[ - Dict[str, CallImportEvaluationLLMOverride] - ] = Field( - default=None, - description=( - "Replace the run's per-metric LLM overrides. When omitted, " - "the existing overrides are kept; when set, this dict " - "fully replaces them (pass an empty object to clear)." - ), - ) - - # --- STT overrides --- - stt_provider: Optional[str] = Field( - default=None, - max_length=50, - description=( - "Override the run-level STT provider for this retry. Must " - "be paired with ``stt_model``. Only meaningful when the " - "run is configured for the diarised transcript source." - ), - ) - stt_model: Optional[str] = Field( - default=None, - max_length=100, - description="Override the run-level STT model.", - ) - stt_credential_id: Optional[UUID] = Field( - default=None, - description="Pin a specific credential row for the STT call.", - ) - # --- LLM diariser overrides --- - # When set, replace the run-stored diariser configuration for any - # rows that have to be re-diarised as part of the retry (i.e. - # ``transcribe_overwrite=True`` or the row never had a diarised - # transcript). Same provider+model pairing rule as STT. - diarization_llm_provider: Optional[str] = Field( - default=None, - max_length=50, - description=( - "Override the run's diariser LLM provider. Must be paired " - "with ``diarization_llm_model``." - ), - ) - diarization_llm_model: Optional[str] = Field( - default=None, - max_length=100, - description="Override the run's diariser LLM model.", - ) - diarization_llm_credential_id: Optional[UUID] = Field( - default=None, - description="Pin a specific credential row for the diariser LLM.", - ) - diarization_prompt: Optional[str] = Field( - default=None, - max_length=10_000, - description=( - "Override the run's diariser prompt. Pass an empty string " - "to clear the override and fall back to the canonical " - "default; pass None to leave the existing value untouched." - ), - ) - transcribe_overwrite: bool = Field( - default=False, - description=( - "When True, wipe the diarised transcript on every retried " - "row's source CallImportRow so the (possibly new) STT runs " - "from scratch. When False, rows that already have a " - "diarised transcript skip diarisation and only re-evaluate." - ), - ) - transcribe_mode: Optional[Literal["stt_llm", "llm_only"]] = Field( - default=None, - description=( - "Override the run's diarisation pipeline mode for this retry. " - "``stt_llm`` runs STT then an LLM diariser; ``llm_only`` feeds " - "audio directly to a multimodal diariser LLM." - ), - ) - - # Telephony credentials for rows that must re-fetch recordings. - provider: Optional[str] = Field( - default=None, - description=( - "Override the batch's telephony provider for this retry pass. " - "Must be paired with ``telephony_integration_id``. Omit both " - "fields to keep the batch's existing pinned credentials." - ), - ) - telephony_integration_id: Optional[UUID] = Field( - default=None, - description=( - "Override the telephony credential used when re-fetching " - "recordings during this retry. Must be paired with " - "``provider``. Omit both to keep existing credentials; send " - "both as null for direct-URL retry." - ), - ) - - @model_validator(mode="after") - def validate_telephony_credential_mode(self) -> "CallImportEvaluationRetryRequest": - fields_set = self.model_fields_set - if ( - "provider" not in fields_set - and "telephony_integration_id" not in fields_set - ): - return self - has_provider = bool((self.provider or "").strip()) - has_integration = self.telephony_integration_id is not None - if has_provider != has_integration: - raise ValueError( - "provider and telephony_integration_id must both be provided " - "or both omitted for direct-URL retry." - ) - return self - - -class CallImportEvaluationRetrySkippedItem(BaseModel): - """One entry in the retry response's ``skipped`` list.""" - - eval_row_id: UUID - reason: str = Field( - ..., - description=( - "Why this row was not re-enqueued. Known values: " - "'unknown' (id not in this run), 'in_progress' " - "(status is pending/running), 'completed' (already " - "successful), 'source_row_missing'." - ), - ) - - -class CallImportEvaluationRetryResponse(BaseModel): - """Summary of a retry fan-out request.""" - - requeued: int = Field( - ..., - description="How many evaluation rows were reset and re-enqueued.", - ) - transcribe_requeued: int = Field( - default=0, - description=( - "Of those, how many were chained through a diarisation " - "task first because the diarised transcript was missing " - "(matches the auto-transcribe behavior of the create-run " - "endpoint)." - ), - ) - skipped: List[CallImportEvaluationRetrySkippedItem] = Field( - default_factory=list, - description="Rows the caller asked for that we did not re-enqueue.", - ) - - -class CallImportEvaluationBulkActionResponse(BaseModel): - """Acknowledgement for bulk cancel / force-fail requests accepted off-thread.""" - - accepted: bool = True - target_count: int = Field( - ..., - description="How many rows the background worker will process.", - ) - evaluation_id: UUID - - -class CallImportMetricSummary(BaseModel): - """Lightweight metric descriptor returned alongside an evaluation.""" - - id: UUID - name: str - metric_type: Optional[str] = None - description: Optional[str] = None - parent_metric_id: Optional[UUID] = None - selection_mode: Optional[SelectionMode] = None - # Surfaced so the Flow tab can decide whether to render the - # Discovered Labels panel next to a multi_label parent. Defaults to - # False to keep legacy clients (and standalone metrics) unaffected. - allow_discovery: bool = False - - model_config = ConfigDict(from_attributes=True) - - -class CallImportEvaluationResponse(BaseModel): - """Parent record describing one evaluation run over a batch.""" - - id: UUID - call_import_id: UUID - organization_id: UUID - name: Optional[str] = None - selected_metric_ids: List[UUID] = Field(default_factory=list) - # Parent UUID string -> [child UUID string]. Captured at run creation - # so the UI can rebuild the parent/child tree even after metrics are - # renamed or deleted. Empty / NULL = no hierarchy was used. - selected_metric_groups: Optional[Dict[str, List[str]]] = None - metrics: List[CallImportMetricSummary] = Field(default_factory=list) - status: str - total_rows: int - completed_rows: int - failed_rows: int - error_message: Optional[str] = None - llm_provider: Optional[str] = None - llm_model: Optional[str] = None - llm_credential_id: Optional[UUID] = None - llm_config: Optional[Dict[str, Any]] = None - metric_llm_overrides: Optional[Dict[str, Any]] = None - stt_provider: Optional[str] = None - stt_model: Optional[str] = None - stt_credential_id: Optional[UUID] = None - # Run-level LLM diariser config. Surfaced so the UI can show - # "Diarised via openai/gpt-4o-mini" on the evaluation header and - # pre-fill the retry modal with the previously-used prompt. - diarisation_llm_provider: Optional[str] = None - diarisation_llm_model: Optional[str] = None - diarisation_llm_credential_id: Optional[UUID] = None - diarisation_prompt: Optional[str] = None - # Diarisation pipeline shape this run was created with. ``stt_llm`` - # (default) is the legacy STT-then-LLM-diariser flow; ``llm_only`` - # means the audio was fed directly to a multimodal diariser LLM. - # Surfaced so the retry modal can preselect the right mode and the - # eval header can render "Diarised via LLM only (Gemini)" instead of - # an empty STT label. - transcribe_mode: Literal["stt_llm", "llm_only"] = "stt_llm" - # Which transcript column this run scored against. All current runs - # use diarised; legacy rows may still carry ``production``. - transcript_source: CallImportEvaluationTranscriptSource = "diarised" - # Sibling evaluation ids created in the same Run Evaluation request. - # Populated only on the POST response (and only when the user ticked - # both Production and Diarised in the modal G�� the backend creates - # one ``CallImportEvaluation`` per source and links them via this - # field so the frontend can deep-link to either run). Empty for all - # other reads. - sibling_evaluation_ids: List[UUID] = Field(default_factory=list) - started_at: Optional[datetime] = None - finished_at: Optional[datetime] = None - created_at: datetime - updated_at: datetime - created_by_email: Optional[str] = None - last_updated_by_email: Optional[str] = None - # Cached LLM-generated TLDR for the Visualizations tab. Lazily - # populated by ``POST /evaluations/{eval_id}/insights``; ``None`` - # for runs the user has not summarised yet. ``is_stale`` on the - # nested object is set by the route, not the model. - tldr_summary: Optional["EvaluationTldrSummary"] = None - # Cached LLM-generated user insights for External Audit PDF section 03. - user_insights: Optional["EvaluationUserInsightsState"] = None - # Cached per-metric failure clustering for internal diagnostics. - metric_clusters: Optional["EvaluationMetricClustersState"] = None - # True when the user opted into top-level metric discovery on the - # Run Evaluation modal. The frontend uses this to gate the - # "Discovered metrics" panel on the Flow tab. - discover_new_metrics: bool = False - bulk_operation: Optional[ - Literal["abort", "force_fail_pending", "retry"] - ] = Field( - default=None, - description=( - "When set, a bulk background operation (abort, force-fail pending, " - "or retry) is still running for this evaluation. Other mutating " - "actions are rejected until it completes." - ), - ) - - model_config = ConfigDict(from_attributes=True) - - -class CallImportEvaluationListResponse(BaseModel): - """Wrapper for listing evaluations on a single batch.""" - - items: List[CallImportEvaluationResponse] - total: int - - -class CallImportEvaluationRowResponse(BaseModel): - """Per-source-row evaluation output (one Metric set applied to one row). - - ``raw_columns``, ``recording_url`` and ``recording_s3_key`` come from - the parent ``CallImportRow`` so the row-detail panel can show the - full CSV row metadata + audio without a second round-trip. The UI - prefers ``recording_s3_key`` (resolved via a presigned URL) over - ``recording_url`` so playback uses our downloaded copy instead of - the raw provider URL, which is often expired/auth-gated. - """ - - id: UUID - evaluation_id: UUID - call_import_row_id: UUID - row_index: Optional[int] = None - # Renamed from ``external_call_id``; same value, mirrors the renamed - # ``call_import_rows.conversation_id`` column. - conversation_id: Optional[str] = None - transcript: Optional[str] = None - raw_columns: Optional[Dict[str, Any]] = None - recording_url: Optional[str] = None - recording_date: Optional[date] = None - recording_s3_key: Optional[str] = None - diarised_transcript_status: Optional[str] = None - diarised_transcript_error: Optional[str] = None - status: str - metric_scores: Dict[str, Any] = Field(default_factory=dict) - error_message: Optional[str] = None - started_at: Optional[datetime] = None - finished_at: Optional[datetime] = None - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class CallImportEvaluationRowListResponse(BaseModel): - """Paginated per-row evaluation results.""" - - items: List[CallImportEvaluationRowResponse] - total: int - page: int - page_size: int - - -class CallImportRowBulkDelete(BaseModel): - """Request body for deleting multiple rows from a call-import batch.""" - - row_ids: List[UUID] = Field( - ..., - min_length=1, - description="Row ids to delete (must belong to the same call import).", - ) - - -class CallImportRowBulkDeleteResponse(BaseModel): - """Response after a bulk-delete pass over ``CallImportRow`` rows.""" - - deleted: int = Field( - ..., - description="How many rows were actually removed (unknown ids are skipped).", - ) - status: Literal["completed", "accepted"] = Field( - default="completed", - description=( - "``accepted`` when deletion was queued to run asynchronously; " - "``completed`` when rows were removed before the response." - ), - ) - - -class CallImportRetryFailedRowsRequest(BaseModel): - """Optional credential override when re-enqueueing failed import rows.""" - - provider: Optional[str] = Field( - default=None, - description=( - "Telephony provider key for this retry pass. Omit together with " - "``telephony_integration_id`` to download from CSV recording URLs." - ), - ) - telephony_integration_id: Optional[UUID] = Field( - default=None, - description=( - "Telephony credential to use for this retry pass. Omit together " - "with ``provider`` for direct-URL retry." - ), - ) - - @model_validator(mode="after") - def validate_credential_mode(self) -> "CallImportRetryFailedRowsRequest": - has_provider = bool((self.provider or "").strip()) - has_integration = self.telephony_integration_id is not None - if has_provider != has_integration: - raise ValueError( - "provider and telephony_integration_id must both be provided " - "or both omitted for direct-URL retry." - ) - return self - - -class CallImportRetryFailedRowsResponse(BaseModel): - """Summary of a retry pass over failed call-import rows.""" - - requeued: int = Field( - ..., - description=( - "Rows reset to pending and successfully re-enqueued on the " - "``imports`` worker queue." - ), - ) - enqueue_failed: int = Field( - default=0, - description=( - "Rows that were eligible for retry but failed to enqueue again. " - "These rows are left in ``failed`` with an enqueue error." - ), - ) - skipped: int = Field( - default=0, - description=( - "Rows skipped because they were no longer in ``failed`` at retry " - "time (for example, already retried from another tab)." - ), - ) - - -# --- Diarization / Transcription request/response shapes --- - - -class CallImportTranscribeRequest(BaseModel): - """Body for kicking off diarization for one or many call-import rows. - - The same shape powers both the per-row endpoint (where ``row_ids`` - is ignored) and the batch-level endpoint. ``only_missing`` is the - safe default G�� rows with an existing transcript are skipped unless - ``overwrite_existing`` is set. - - Two modes are supported: - - * ``mode="stt_llm"`` (default) G�� the legacy two-stage pipeline: STT - produces plain text, an LLM splits it into agent/user turns using - ``diarization_prompt``. ``stt_provider`` and ``stt_model`` are - required in this mode. - * ``mode="llm_only"`` G�� skip STT entirely and hand the recording's - audio bytes to a multimodal chat model along with - ``diarization_prompt``. The model both transcribes and diarises in - a single pass. The STT fields are ignored (and must be omitted / - null). Only providers whose chat API accepts audio input (OpenAI - ``gpt-4o-audio-*``, Google Gemini ``1.5/2.0``) are usable; other - providers will surface a typed error on the row. - """ - - mode: Literal["stt_llm", "llm_only"] = Field( - default="stt_llm", - description=( - "Pipeline shape. 'stt_llm' (default) runs STT then an LLM " - "diariser over the resulting text. 'llm_only' skips STT and " - "feeds the raw audio to a multimodal LLM together with " - "``diarization_prompt`` for a single-pass transcribe + " - "diarise." - ), - ) - stt_provider: Optional[str] = Field( - default=None, - max_length=50, - description=( - "STT provider key, e.g. 'deepgram' or 'openai'. Required when " - "``mode='stt_llm'``; must be omitted when ``mode='llm_only'``." - ), - ) - stt_model: Optional[str] = Field( - default=None, - max_length=100, - description=( - "STT model name, e.g. 'nova-2' or 'whisper-1'. Required when " - "``mode='stt_llm'``; must be omitted when ``mode='llm_only'``." - ), - ) - credential_id: Optional[UUID] = Field( - default=None, - description="Optional AIProvider/Integration row to pin for this run.", - ) - language: Optional[str] = Field( - default=None, - max_length=20, - description="Optional ISO language hint, e.g. 'en'.", - ) - only_missing: bool = Field( - default=True, - description=( - "When true, rows with an existing transcript are skipped (the " - "default safe behavior)." - ), - ) - overwrite_existing: bool = Field( - default=False, - description=( - "When true, existing transcripts are replaced. Mutually " - "exclusive with only_missing." - ), - ) - row_ids: Optional[List[UUID]] = Field( - default=None, - description=( - "Restrict the run to a specific subset of rows. NULL = every " - "row in the import (subject to only_missing)." - ), - ) - # --- LLM diariser config --- - # In ``stt_llm`` mode diarisation runs as a *second* step: STT - # produces plain text, then this LLM splits it into agent/user - # turns. In ``llm_only`` mode this same LLM directly receives the - # audio and the prompt. Both fields are always mandatory because - # there is no longer a pyannote fallback and ``llm_only`` cannot - # function without an LLM either. - diarization_llm_provider: str = Field( - ..., - max_length=50, - description=( - "LLM provider that diarises the call. In ``stt_llm`` it sees " - "the STT text; in ``llm_only`` it sees the raw audio." - ), - ) - diarization_llm_model: str = Field( - ..., - max_length=100, - description=( - "LLM model name. In ``llm_only`` mode this must be a model " - "that accepts audio input (e.g. 'gpt-4o-audio-preview', " - "'gemini-1.5-pro')." - ), - ) - diarization_llm_credential_id: Optional[UUID] = Field( - default=None, - description=( - "Optional AIProvider row to pin for the diarisation LLM." - ), - ) - diarization_prompt: Optional[str] = Field( - default=None, - max_length=10_000, - description=( - "Operator-supplied system prompt for the diariser LLM. " - "When NULL/empty the worker uses the canonical default " - "(see ``GET /api/v1/call-imports/diarisation-prompt-default``)." - ), - ) - - @model_validator(mode="after") - def _validate_mode_fields(self) -> "CallImportTranscribeRequest": - """Enforce STT-field presence rules based on ``mode``. - - ``stt_llm`` (default) requires both STT fields G�� the worker - cannot diarise without a transcript. ``llm_only`` forbids them - so the API contract makes it clear that the audio is going - straight to the LLM; passing both would be ambiguous about - which path the worker should take. - """ - stt_provider = (self.stt_provider or "").strip() if self.stt_provider else None - stt_model = (self.stt_model or "").strip() if self.stt_model else None - if self.mode == "stt_llm": - if not stt_provider or not stt_model: - raise ValueError( - "stt_provider and stt_model are required when " - "mode='stt_llm'." - ) - else: # llm_only - if stt_provider or stt_model: - raise ValueError( - "stt_provider/stt_model must be omitted when " - "mode='llm_only'; the LLM consumes the audio " - "directly." - ) - return self - - -class CallImportDiarisationPromptDefaultResponse(BaseModel): - """Wrapper for the canonical diariser-prompt fetched by the modal.""" - - prompt: str = Field( - ..., - description=( - "The exact prompt the worker falls back to when the caller " - "leaves ``diarization_prompt`` blank. The frontend pre-fills " - "the textarea with this value so the operator can edit it." - ), - ) - - -class CallImportRowIdsResponse(BaseModel): - """Flat row-id list for cross-page bulk selection. - - Powers the "Select all M rows in this import" affordance on the - detail page G�� returning only ids keeps the payload tiny so the UI - can hold the full set in memory even for batches with thousands - of rows. The frontend then passes those ids straight to the - existing bulk-delete / bulk-transcribe endpoints. - """ - - ids: List[UUID] = Field( - default_factory=list, - description=( - "Every ``CallImportRow.id`` that matches the ``q`` and " - "``diarised_status`` filters (or every row when neither is " - "supplied), sorted by ``row_index``." - ), - ) - total: int = Field( - ..., - description=( - "Length of ``ids``. Sent explicitly so callers can show a " - "count without re-measuring the array." - ), - ) - - -class CallImportTranscribeResponse(BaseModel): - """Summary of a transcribe fan-out request.""" - - queued: int = Field( - ..., - description=( - "How many rows were enqueued for diarization. Skipped rows " - "(missing recording, transcript already present, etc.) are " - "not counted." - ), - ) - skipped_rows: int = Field( - default=0, - description="Rows excluded by only_missing or because they had no recording.", - ) - skipped_reason_counts: Dict[str, int] = Field( - default_factory=dict, - description="Per-reason breakdown of skipped rows for the UI to surface.", - ) - accepted: bool = Field( - default=False, - description=( - "When true, diarization setup was queued to a background worker " - "and ``queued`` reflects zero until the worker finishes enqueue." - ), - ) - - -class CallImportCancelDiarisationRequest(BaseModel): - """Body for the batch cancel-diarisation endpoint. - - Omit ``row_ids`` (or pass ``null``) to cancel every row in the - import whose ``diarised_transcript_status`` is currently - ``pending`` or ``running``. Pass an explicit list to scope the - cancel to a subset (e.g. the rows the operator selected in the - UI). - """ - - row_ids: Optional[List[UUID]] = Field( - default=None, - description=( - "Optional subset of CallImportRow UUIDs. ``None`` cancels " - "every pending / running diarisation in the import." - ), - ) - - -class CallImportCancelDiarisationResponse(BaseModel): - """Summary of a cancel-diarisation request. - - ``cancelled`` counts rows that were actively pending / running - when the cancel landed and got flipped to ``failed`` with a - "Cancelled by user" error. ``skipped`` counts rows that were - requested (or matched the implicit "all rows" filter) but were - not in a cancellable state G�� typically because they had already - finished or were never queued for diarisation in the first place. - """ - - cancelled: int = Field( - ..., - description=( - "Rows whose in-flight Celery task was revoked and whose " - "``diarised_transcript_status`` was flipped to ``failed`` " - "with a 'Cancelled by user' error message." - ), - ) - skipped: int = Field( - default=0, - description=( - "Rows that were requested but not in a cancellable state " - "(idle / completed / already failed)." - ), - ) - - -# --- Per-run aggregation / visualization payloads --- - - -class CallImportMetricHistogramBucket(BaseModel): - """One bin of a numeric metric histogram.""" - - x0: float - x1: float - count: int - - -class CallImportMetricValueCount(BaseModel): - """One row of a categorical metric's value frequency table.""" - - label: str - count: int - - -class CallImportMetricLabelPair(BaseModel): - """One unordered pair-count cell of a multi-label parent's - co-occurrence matrix. - - ``a`` and ``b`` are child label names; ``count`` is the number of - rows on which both labels fired together (intersection size). - Pairs are emitted with ``a < b`` lexicographically so the matrix - can be reconstructed without duplicates on the frontend. - """ - - a: str - b: str - count: int - - -class CallImportMetricAggregate(BaseModel): - """Per-metric aggregate computed from an evaluation run's rows. - - Numeric metrics return summary statistics + histogram buckets; - categorical / pass-fail / text metrics return the top value counts. - Both shapes can coexist if a metric mixes types G�� the UI prefers - histogram when present, falls back to value_counts otherwise. - """ - - metric_id: str - metric_name: str - metric_type: Optional[str] = None - metric_category: str = "quality" - # True when this aggregate represents a multi-label parent metric - # (selection_mode == "multi_label" with no parent_metric_id). For - # those, ``value_counts`` lists per-child label tallies and the - # rows scored != sum(value_counts.count). The UI uses this flag to - # force a horizontal bar layout (slices wouldn't sum to 100%) and - # to label the n-badge as rows scored, not label occurrences. - is_multi_label_parent: bool = False - count: int = 0 - skipped_count: int = 0 - error_count: int = 0 - # Numeric stats (None when no numeric values were observed) - mean: Optional[float] = None - median: Optional[float] = None - p25: Optional[float] = None - p75: Optional[float] = None - p95: Optional[float] = None - min: Optional[float] = None - max: Optional[float] = None - stddev: Optional[float] = None - histogram_buckets: List[CallImportMetricHistogramBucket] = Field( - default_factory=list - ) - value_counts: List[CallImportMetricValueCount] = Field(default_factory=list) - # Pairwise label intersections for multi-label parent metrics. - # Empty for everything else. The frontend reconstructs a square - # symmetric matrix from these unordered pairs and renders the - # co-occurrence heatmap chart type. - co_occurrence: List[CallImportMetricLabelPair] = Field(default_factory=list) - - -class MetricPeriodDelta(BaseModel): - """Week-over-week (or baseline-run) delta for one metric.""" - - label: str - detail: str - why: Optional[str] = None - - -class CallImportEvaluationAggregateResponse(BaseModel): - """Aggregated metric distributions for a single evaluation run.""" - - evaluation_id: UUID - total_rows: int - completed_rows: int - failed_rows: int - metrics: List[CallImportMetricAggregate] = Field(default_factory=list) - period_deltas: Dict[str, MetricPeriodDelta] = Field(default_factory=dict) - baseline_evaluation_id: Optional[UUID] = None - failure_policies_source: Optional[Literal["inferred", "user"]] = Field( - default=None, - description=( - "Whether flagged-rate semantics use user-confirmed failure policies " - "or inferred defaults from the Failure diagnostics flow." - ), - ) - - -# --- LLM-generated TLDR for the Visualizations tab --- - - - -class EvaluatorResultsAggregateResponse(BaseModel): - """Chart-friendly metric rollups for evaluator results in a suite or scenario scope.""" - - scope: str - suite_id: Optional[UUID] = None - agent_id: Optional[UUID] = None - scenario_id: Optional[UUID] = None - total_rows: int = 0 - completed_rows: int = 0 - failed_rows: int = 0 - metrics: List[CallImportMetricAggregate] = Field(default_factory=list) - - -# --- LLM-generated TLDR for the Visualizations tab --- - - - -class EvaluationTldrSummary(BaseModel): - """Cached LLM-generated narrative + bullet patterns for an eval run. - - Persisted on ``CallImportEvaluation.tldr_summary`` (JSONB) and - rendered above the per-metric charts. ``generated_at_completed_rows`` - is the snapshot of ``completed_rows`` at the time the summary was - written; the API compares it against the current count to flag - ``is_stale`` so the UI can prompt for a regenerate. - """ - - narrative: str - patterns: List[str] = Field(default_factory=list) - metric_insights: Dict[str, str] = Field(default_factory=dict) - generated_at: datetime - generated_at_completed_rows: int = 0 - provider: Optional[str] = None - model: Optional[str] = None - is_stale: bool = False - - -class EvaluationInsightsRequest(BaseModel): - """Body for ``POST /evaluations/{eval_id}/insights``. - - All fields are optional. When ``provider``/``model`` are unset the - backend resolves the org's first active OpenAI/Anthropic/Google - provider (mirroring the Prompt Partials AI-generate flow) so - callers that don't care can simply post ``{}``. - """ - - regenerate: bool = False - provider: Optional[str] = None - model: Optional[str] = Field(default=None, min_length=1) - credential_id: Optional[UUID] = None - max_llm_calls: Optional[int] = Field( - default=None, - ge=20, - le=500, - description=( - "Max LLM calls for user-insights sampling (extraction + synthesis). " - "Defaults to 200 when omitted." - ), - ) - - -class UserInsightCategory(BaseModel): - label: str - count: int - share_pct: float - - -class UserInsightEvidenceTurn(BaseModel): - speaker: str - text: str - - -class UserInsightEvidence(BaseModel): - conversation_id: Optional[str] = None - quote: str - turns: List[UserInsightEvidenceTurn] = Field(default_factory=list) - - -class EvaluationUserInsightItem(BaseModel): - id: str - title: str - categories: List[UserInsightCategory] = Field(default_factory=list) - observation: str - evidence: UserInsightEvidence - - -class EvaluationUserInsightsState(BaseModel): - """Cached map-reduce LLM user insights for an evaluation run.""" - - status: Literal["idle", "running", "completed", "failed"] = "idle" - insights: List[EvaluationUserInsightItem] = Field(default_factory=list) - overview: Optional[str] = None - generated_at: Optional[datetime] = None - generated_at_completed_rows: int = 0 - progress: Optional[Dict[str, int]] = None - provider: Optional[str] = None - model: Optional[str] = None - llm_calls_used: int = 0 - max_llm_calls: Optional[int] = None - error_message: Optional[str] = None - is_stale: bool = False - - -class EvaluationUserInsightsRequest(BaseModel): - """Body for ``POST /evaluations/{eval_id}/user-insights``.""" - - regenerate: bool = False - force: bool = False - provider: Optional[str] = None - model: Optional[str] = Field(default=None, min_length=1) - credential_id: Optional[UUID] = None - max_llm_calls: Optional[int] = Field(default=None, ge=20, le=500) - - -MetricClusterGapLabel = Literal[ - "LOGIC_GAP", - "UNDERSPEC", - "EXISTS_NO_TRIGGER", - "MISSING", -] - -FailurePolicyNumericOp = Literal["lt", "lte", "gt", "gte"] - - -class MetricFailurePolicy(BaseModel): - """Per-metric definition of which scores count as failures for this evaluation.""" - - metric_id: str - failure_values: List[str] = Field( - default_factory=list, - description="Normalized lowercase labels that count as failure (single-choice, enum, boolean-as-category).", - ) - failure_child_names: List[str] = Field( - default_factory=list, - description="Child label names that count as failure for multi_label parents.", - ) - numeric_rule: Optional[Dict[str, Any]] = Field( - default=None, - description='Numeric failure rule, e.g. {"op": "lt", "threshold": 0.5}.', - ) - - -class MetricFailurePolicyValueCount(BaseModel): - label: str - count: int = 0 - - -class MetricFailurePolicyMetricPreview(BaseModel): - metric_id: str - metric_name: str - metric_type: Optional[str] = None - selection_mode: Optional[str] = None - is_multi_label_parent: bool = False - value_counts: List[MetricFailurePolicyValueCount] = Field(default_factory=list) - child_names: List[str] = Field(default_factory=list) - row_count_by_value: Dict[str, int] = Field(default_factory=dict) - suggested_policy: MetricFailurePolicy - effective_policy: MetricFailurePolicy - - -class MetricFailurePoliciesResponse(BaseModel): - previews: List[MetricFailurePolicyMetricPreview] = Field(default_factory=list) - policies: Dict[str, MetricFailurePolicy] = Field(default_factory=dict) - source: Literal["inferred", "user"] = "inferred" - updated_at: Optional[datetime] = None - - -class MetricFailurePoliciesSaveRequest(BaseModel): - policies: Dict[str, MetricFailurePolicy] = Field(default_factory=dict) - source: Literal["user"] = "user" - - -class MetricClusterEvidenceTurn(BaseModel): - speaker: str - text: str - - -class MetricClusterEvidence(BaseModel): - conversation_id: Optional[str] = None - evaluation_row_id: Optional[UUID] = None - quote: str = "" - turns: List[MetricClusterEvidenceTurn] = Field(default_factory=list) - - -class MetricSubCluster(BaseModel): - label: str - count: int = 0 - share_pct: float = 0.0 - - -class MetricCluster(BaseModel): - id: str - label: str - gap_label: MetricClusterGapLabel - level: int = 1 - count: int = 0 - share_pct: float = 0.0 - sub_clusters: List[MetricSubCluster] = Field(default_factory=list) - observation: str = "" - failure_reason: str = "" - evidence: MetricClusterEvidence = Field(default_factory=MetricClusterEvidence) - is_discovered: bool = False - - -class MetricClusterGroup(BaseModel): - metric_id: str - metric_name: str - flagged_count: int = 0 - failure_reason: str = "" - clusters: List[MetricCluster] = Field(default_factory=list) - - -class DiscoveredProblemCluster(BaseModel): - id: str - label: str - gap_label: MetricClusterGapLabel - count: int = 0 - share_pct: float = 0.0 - observation: str = "" - failure_reason: str = "" - evidence: MetricClusterEvidence = Field(default_factory=MetricClusterEvidence) - - -class RcaRepeatedPatternRow(BaseModel): - metric_id: str - metric_name: str - top_rca_patterns: str = "" - evidence_share_pct: float = 0.0 - evidence_calls: int = 0 - evidence_cluster_count: int = 0 - failure_reason: str = "" - - -class RcaMetricHotspotRow(BaseModel): - metric_id: str - metric_name: str - description: str = "" - metric_rate_pct: float = 0.0 - flagged_calls: int = 0 - - -class RcaPromptAreaRow(BaseModel): - label: str - share_pct: float = 0.0 - gap_label: MetricClusterGapLabel - - -class MetricClustersRcaSummary(BaseModel): - total_clusters: int = 0 - total_clustered_instances: int = 0 - total_flagged_instances: int = 0 - analysed_calls: int = 0 - repeated_patterns: List[RcaRepeatedPatternRow] = Field(default_factory=list) - metric_hotspots: List[RcaMetricHotspotRow] = Field(default_factory=list) - prompt_areas: List[RcaPromptAreaRow] = Field(default_factory=list) - - -class EvaluationMetricClustersState(BaseModel): - """Cached per-metric failure clustering for internal diagnostics.""" - - status: Literal["idle", "running", "completed", "failed", "cancelled"] = "idle" - groups: List[MetricClusterGroup] = Field(default_factory=list) - discovered_problems: List[DiscoveredProblemCluster] = Field( - default_factory=list - ) - overview: Optional[str] = None - generated_at: Optional[datetime] = None - generated_at_completed_rows: int = 0 - progress: Optional[Dict[str, int]] = None - provider: Optional[str] = None - model: Optional[str] = None - llm_calls_used: int = 0 - max_llm_calls: Optional[int] = None - error_message: Optional[str] = None - is_stale: bool = False - selected_evaluation_row_ids: List[str] = Field( - default_factory=list, - description="Evaluation row IDs included in the last clustering run.", - ) - failure_policies: Dict[str, MetricFailurePolicy] = Field(default_factory=dict) - failure_policies_source: Literal["inferred", "user"] = "inferred" - failure_policies_updated_at: Optional[datetime] = None - rca_summary: Optional[MetricClustersRcaSummary] = None - - -class MetricClusterEligibleRow(BaseModel): - """Completed evaluation row with at least one flagged quality metric.""" - - evaluation_row_id: UUID - conversation_id: Optional[str] = None - row_index: Optional[int] = None - flagged_metric_names: List[str] = Field(default_factory=list) - - -class MetricClusterEligibleRowsResponse(BaseModel): - items: List[MetricClusterEligibleRow] = Field(default_factory=list) - total: int = 0 - - -class PromptImprovementSuggestion(BaseModel): - """One LLM-generated prompt edit to address a failure cluster.""" - - id: str - metric_id: str - metric_name: str - cluster_id: str - cluster_label: str - gap_label: MetricClusterGapLabel - share_pct: float = 0.0 - priority: Literal["high", "medium", "low"] = "medium" - change_type: Literal["edit", "add"] = "add" - target_section: str = "" - anchor_excerpt: str = "" - current_gap: str = "" - suggested_text: str = "" - rationale: str = "" - flow_node_id: str = "" - flow_node_label: str = "" - - -class EvaluationPromptImprovementsState(BaseModel): - """Cached prompt improvement suggestions for an evaluation run.""" - - status: Literal["idle", "running", "completed", "failed"] = "idle" - imported_agent_id: Optional[str] = None - imported_agent_name: Optional[str] = None - suggestions: List[PromptImprovementSuggestion] = Field(default_factory=list) - overview: Optional[str] = None - generated_at: Optional[datetime] = None - generated_at_completed_rows: int = 0 - provider: Optional[str] = None - model: Optional[str] = None - error_message: Optional[str] = None - is_stale: bool = False - - -class EvaluationPromptImprovementsRequest(BaseModel): - """Body for ``POST /evaluations/{eval_id}/prompt-improvements``.""" - - imported_agent_id: UUID - regenerate: bool = False - force: bool = False - provider: Optional[str] = None - model: Optional[str] = None - credential_id: Optional[UUID] = None - - -class EvaluationMetricClustersRequest(BaseModel): - """Body for ``POST /evaluations/{eval_id}/metric-clusters``.""" - - regenerate: bool = False - force: bool = False - provider: Optional[str] = None - model: Optional[str] = Field(default=None, min_length=1) - credential_id: Optional[UUID] = None - max_llm_calls: Optional[int] = Field(default=None, ge=20, le=500) - evaluation_row_ids: Optional[List[UUID]] = Field( - default=None, - description=( - "Subset of completed evaluation row IDs to cluster. When omitted, " - "all completed rows with at least one flagged quality metric are used." - ), - ) - row_limit: Optional[int] = Field( - default=None, - ge=1, - description=( - "Use the first N eligible rows (by row order). Mutually exclusive " - "with evaluation_row_ids." - ), - ) - failure_policies: Optional[Dict[str, MetricFailurePolicy]] = Field( - default=None, - description="Per-metric failure policies confirmed in the cluster modal.", - ) - - -# Resolve the forward reference on ``CallImportEvaluationResponse`` -# (defined further up the file) now that ``EvaluationTldrSummary`` -# exists. Without this Pydantic raises at first ``.model_validate`` -# because the string annotation can't be evaluated. -CallImportEvaluationResponse.model_rebuild() - - -# --- Cross-run insights for a CallImport batch --- - - -class CallImportInsightsRunPoint(BaseModel): - """One run's mean for a metric, used to render trend lines.""" - - evaluation_id: UUID - name: Optional[str] = None - created_at: datetime - mean: Optional[float] = None - completed_rows: int = 0 - - -class CallImportInsightsMetric(BaseModel): - """Per-metric history across every evaluation run on this import.""" - - metric_id: str - metric_name: str - metric_type: Optional[str] = None - latest: Optional[CallImportMetricAggregate] = None - trend: List[CallImportInsightsRunPoint] = Field(default_factory=list) - - -class CallImportInsightsResponse(BaseModel): - """Aggregated cross-run signals for a single call-import batch.""" - - call_import_id: UUID - total_rows: int - rows_with_transcript: int - rows_without_transcript: int - transcript_source_counts: Dict[str, int] = Field(default_factory=dict) - evaluation_count: int = 0 - metrics: List[CallImportInsightsMetric] = Field(default_factory=list) - - -# --- Flow chart visualization for hierarchical metrics --- - - -class MetricFlowNode(BaseModel): - """One step in the LLM-inferred temporal flow for a parent metric. - - Represents a child sub-metric label. ``count`` is the number of rows - in the evaluation where this child appears anywhere in its - ``sequence`` array. ``is_terminal`` is set when the child is the - last entry in a meaningful fraction of those sequences. - - ``is_discovered`` is set when the node represents an LLM-discovered - candidate (parent has ``allow_discovery=true``) rather than a - user-defined child. The id of a discovered node is prefixed with - ``disc:`` so it can't collide with real child UUIDs. - """ - - id: str - label: str - count: int = 0 - is_terminal: bool = False - is_discovered: bool = False - - -class MetricFlowEdge(BaseModel): - """One directed transition between two children across all rows. - - ``count`` is the number of rows where ``source`` immediately - precedes ``target`` in the sequence. The synthetic ``START`` node - is used as the ``source`` for the first child in every sequence. - """ - - source: str - target: str - count: int = 0 - - -class MetricFlowResponse(BaseModel): - """Aggregate flow diagram payload for a single parent metric.""" - - parent_metric_id: str - parent_metric_name: str - selection_mode: Optional[SelectionMode] = None - nodes: List[MetricFlowNode] = Field(default_factory=list) - edges: List[MetricFlowEdge] = Field(default_factory=list) - total_rows: int = 0 - rows_with_sequence: int = 0 - - -class DiscoveredLabelItem(BaseModel): - """One LLM-discovered candidate sub-label aggregated across rows. - - ``key`` is the slugified label identifier (matches what appears in - ``sequence`` entries). ``count`` is the number of rows in the - evaluation that emitted this slug. ``sample_rationale`` is the - first non-empty rationale captured from any row (back-compat - field, identical to ``examples[0]`` when present). ``examples`` - holds up to 3 distinct rationales G�� the UI surfaces 2 of them as - ``Examples:`` in the rubric on Promote, with the third kept as - headroom in case the first is unhelpful. - """ - - key: str - name: str - description: Optional[str] = None - sample_rationale: Optional[str] = None - examples: List[str] = Field(default_factory=list, max_length=3) - count: int = 0 - - -class DiscoveredLabelsResponse(BaseModel): - """List of discovered candidate sub-labels for a parent metric.""" - - parent_metric_id: str - items: List[DiscoveredLabelItem] = Field(default_factory=list) - - -class DiscoveredLabelMergeRequest(BaseModel): - """Body for POST /evaluations/{eval_id}/discovered-labels/merge. - - Rewrites every row's ``metric_scores[parent_id].discovered_labels`` - entries whose key is ``from_key`` to use ``to_key`` instead, so the - user can collapse near-duplicate candidates ("On Hold" / "Customer - Put On Hold") into a single promoted child. - """ - - parent_metric_id: UUID - from_key: str = Field(..., min_length=1, max_length=120) - to_key: str = Field(..., min_length=1, max_length=120) - - -class DiscoveredLabelDeleteRequest(BaseModel): - """Body for POST /evaluations/{eval_id}/discovered-labels/delete. - - Strips a candidate sub-label from every row's - ``discovered_labels`` list AND from each row's ``sequence`` array, - then tombstones the slug at the evaluation level so workers - finishing later can't re-introduce it. Use for gibberish or - irrelevant candidates the LLM proposed; for near-duplicates that - you want to keep but unify, use the merge endpoint instead. - """ - - parent_metric_id: UUID - key: str = Field(..., min_length=1, max_length=120) - - -class PromoteDiscoveredChildRequest(BaseModel): - """Body for POST /metrics/{parent_id}/children/from-discovered. - - ``key`` is the slug under which the candidate is currently stored - on per-row ``metric_scores``. The newly-created child Metric's - name is normalized so ``slugify(name) == key``, which keeps every - already-scored row's ``sequence`` array resolvable against the - promoted child without a backfill. - """ - - key: str = Field(..., min_length=1, max_length=120) - name: str = Field(..., min_length=1, max_length=120) - description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) - # Default True: when promoting a discovered label we want the new - # sub-metric to always capture rationales going forward, since the - # candidate was itself proposed *with* a rationale and the user - # almost always wants to see why future rows hit it. Explicit False - # keeps the original opt-in behavior available for callers that - # don't care about rationales. - capture_rationale: bool = True - - -# --- Discovered top-level metrics (per-evaluation discovery) --- -# -# Parallel to ``DiscoveredLabelItem`` / merge / delete / promote G�� but -# scoped to the evaluation as a whole, not to a parent category metric. -# Used by the "Discovered metrics" panel at the top of the evaluation -# detail Flow tab when ``CallImportEvaluation.discover_new_metrics`` -# is true. - - -# The promote endpoint accepts these three suggested types; "category" -# creates a parent (no children yet) that the user can later extend in -# the Metrics page. -DiscoveredMetricSuggestedType = Literal["boolean", "rating", "category"] - - -class DiscoveredMetricItem(BaseModel): - """One LLM-discovered candidate top-level metric aggregated across rows. - - Mirrors :class:`DiscoveredLabelItem` but at the evaluation level - (no ``parent_metric_id``). ``suggested_type`` is the LLM's guess at - the best representation; the promote flow lets the user override - it before creating the real :class:`Metric` row. - """ - - key: str - name: str - description: Optional[str] = None - suggested_type: DiscoveredMetricSuggestedType = "boolean" - sample_rationale: Optional[str] = None - examples: List[str] = Field(default_factory=list, max_length=3) - count: int = 0 - - -class DiscoveredMetricsResponse(BaseModel): - """List of discovered candidate top-level metrics for an evaluation.""" - - evaluation_id: UUID - items: List[DiscoveredMetricItem] = Field(default_factory=list) - - -class DiscoveredMetricMergeRequest(BaseModel): - """Body for POST /evaluations/{eval_id}/discovered-metrics/merge. - - Rewrites every row's ``metric_scores["__discovered_metrics__"]`` - entries whose key is ``from_key`` to use ``to_key`` instead, and - records the redirect in ``CallImportEvaluation.discovered_metric_aliases`` - so workers finishing later can't resurrect the merged-out slug. - """ - - from_key: str = Field(..., min_length=1, max_length=120) - to_key: str = Field(..., min_length=1, max_length=120) - - -class DiscoveredMetricDeleteRequest(BaseModel): - """Body for POST /evaluations/{eval_id}/discovered-metrics/delete. - - Strips a candidate from every row's - ``metric_scores["__discovered_metrics__"]`` list and tombstones - the slug at the evaluation level (empty-string alias) so workers - finishing later can't re-introduce it. - """ - - key: str = Field(..., min_length=1, max_length=120) - - -class PromoteDiscoveredMetricRequest(BaseModel): - """Body for POST /metrics/from-discovered. - - Creates a standalone :class:`Metric` (``parent_metric_id=None``) - from an LLM-discovered candidate. The new metric's name is - normalized so ``slugify(name) == key`` to keep already-scored row - payloads resolvable against the promoted metric. ``metric_type`` - selects how the new metric will be scored on future runs; - ``"category"`` creates a ``multi_label`` parent with no children - (the user adds children via the existing Metrics page). - """ - - key: str = Field(..., min_length=1, max_length=120) - name: str = Field(..., min_length=1, max_length=120) - description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) - metric_type: DiscoveredMetricSuggestedType = "boolean" - capture_rationale: bool = True - # Optional per-type config knobs passed through to ``Metric.custom_config``. - # For ``rating`` the frontend can supply {"min": 1, "max": 5}; for - # ``boolean`` / ``category`` the field is typically empty. - custom_config: Optional[Dict[str, Any]] = None - - -# --- Workspace Schemas --- - - -class WorkspaceBase(BaseModel): - """Shared fields for workspace create/update payloads.""" - - name: str = Field(..., min_length=1, max_length=255) - - -class WorkspaceCreate(WorkspaceBase): - """Body for POST /workspaces.""" - - # Optional: derived from name when omitted; uniqueness is per-org. - slug: Optional[str] = Field( - default=None, min_length=1, max_length=255 - ) - - -class WorkspaceUpdate(BaseModel): - """Body for PATCH /workspaces/{id} (rename only in v1).""" - - name: str = Field(..., min_length=1, max_length=255) - - -class WorkspaceResponse(BaseModel): - """Response schema for a single workspace.""" - - id: UUID - organization_id: UUID - name: str - slug: str - is_default: bool - created_at: datetime - updated_at: datetime - role_id: Optional[UUID] = None - role_name: Optional[str] = None - capabilities: List[str] = Field(default_factory=list) - - model_config = ConfigDict(from_attributes=True) - - -class WorkspaceRoleBase(BaseModel): - name: str = Field(..., min_length=1, max_length=255) - description: Optional[str] = None - capabilities: List[str] = Field(default_factory=list) - - -class WorkspaceRoleCreate(WorkspaceRoleBase): - pass - - -class WorkspaceRoleUpdate(BaseModel): - name: Optional[str] = Field(default=None, min_length=1, max_length=255) - description: Optional[str] = None - capabilities: Optional[List[str]] = None - - -class WorkspaceRoleResponse(BaseModel): - id: UUID - organization_id: UUID - name: str - description: Optional[str] = None - capabilities: List[str] - is_system: bool - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class WorkspaceMemberResponse(BaseModel): - id: UUID - workspace_id: UUID - user_id: UUID - role_id: UUID - role_name: str - user_email: str - user_name: Optional[str] = None - added_by_user_id: Optional[UUID] = None - created_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class WorkspaceMemberCreate(BaseModel): - user_id: UUID - role_id: UUID - - -class WorkspaceMemberUpdate(BaseModel): - role_id: UUID - - -class CapabilityInfoResponse(BaseModel): - key: str - label: str - - -class CapabilityDomainResponse(BaseModel): - key: str - label: str - capabilities: List[CapabilityInfoResponse] +"""Pydantic schemas for request/response validation.""" + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator, validator +import re +from typing import Optional, List, Dict, Any, Literal +from datetime import date, datetime +from uuid import UUID +from app.models.enums import ( + EvaluationType, EvaluationStatus, EvaluatorResultStatus, RoleEnum, InvitationStatus, + LanguageEnum, CallTypeEnum, CallMediumEnum, GenderEnum, AccentEnum, BackgroundNoiseEnum, + IntegrationPlatform, ModelProvider, CredentialRoutingMode, GatewayInterfaceMode, VoiceBundleType, TestAgentConversationStatus, + MetricType, MetricCategory, MetricTrigger, CallRecordingStatus, AlertMetricType, AlertAggregation, + AlertOperator, AlertNotifyFrequency, AlertStatus, AlertHistoryStatus, CronJobStatus, + CallImportStatus, CallImportRowStatus, CallImportParameterType, +) + + + +# Audio File Schemas +class AudioFileBase(BaseModel): + """Base audio file schema.""" + + filename: str + format: str + + +class AudioFileCreate(AudioFileBase): + """Schema for audio file creation.""" + + file_size: int + duration: Optional[float] = None + sample_rate: Optional[int] = None + channels: Optional[int] = None + + +class AudioFileResponse(AudioFileBase): + """Schema for audio file response.""" + + id: UUID + file_size: int + duration: Optional[float] = None + sample_rate: Optional[int] = None + channels: Optional[int] = None + uploaded_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +# Evaluation Schemas +class EvaluationCreate(BaseModel): + """Schema for creating an evaluation.""" + + audio_id: UUID + reference_text: Optional[str] = None + evaluation_type: EvaluationType + model_name: Optional[str] = Field(None, description="Model to use for evaluation") + metrics: Optional[List[str]] = Field( + default=["wer", "latency"], description="Metrics to calculate" + ) + + @field_validator("metrics") + @classmethod + def validate_metrics(cls, v): + """Validate metrics list.""" + allowed_metrics = ["wer", "cer", "latency", "quality_score", "rtf"] + if v: + invalid = [m for m in v if m not in allowed_metrics] + if invalid: + raise ValueError(f"Invalid metrics: {invalid}") + return v + + +class EvaluationResponse(BaseModel): + """Schema for evaluation response.""" + + id: UUID + audio_id: UUID + reference_text: Optional[str] = None + evaluation_type: EvaluationType + model_name: Optional[str] = None + status: EvaluationStatus + metrics_requested: Optional[List[str]] = None + created_at: datetime + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + error_message: Optional[str] = None + + model_config = ConfigDict(from_attributes=True) + + +class EvaluationStatusResponse(BaseModel): + """Schema for evaluation status response.""" + + id: UUID + status: EvaluationStatus + created_at: datetime + completed_at: Optional[datetime] = None + error_message: Optional[str] = None + + +# Evaluation Result Schemas +class EvaluationResultResponse(BaseModel): + """Schema for evaluation result response.""" + + evaluation_id: UUID + status: EvaluationStatus + transcript: Optional[str] = None + metrics: Dict[str, Any] + processing_time: Optional[float] = None + model_used: Optional[str] = None + created_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class MetricsResponse(BaseModel): + """Schema for metrics breakdown.""" + + evaluation_id: UUID + metrics: Dict[str, Any] + processing_time: Optional[float] = None + + +# Comparison Schema +class ComparisonRequest(BaseModel): + """Schema for comparing multiple evaluations.""" + + evaluation_ids: List[UUID] = Field(..., min_length=2, description="At least 2 evaluation IDs to compare") + + +class ComparisonResponse(BaseModel): + """Schema for comparison results.""" + + evaluations: List[EvaluationResultResponse] + comparison_metrics: Dict[str, Any] + + +# API Key Schemas +class APIKeyCreate(BaseModel): + """Schema for creating API key.""" + + name: Optional[str] = None + + +class APIKeyResponse(BaseModel): + """Schema for API key response.""" + + id: UUID + key: str + name: Optional[str] = None + is_active: bool + created_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +# Generic Response Schemas +class MessageResponse(BaseModel): + """Generic message response.""" + + message: str + + +class ErrorResponse(BaseModel): + """Error response schema.""" + + detail: str + +# ============================================ +# VAIOPS SCHEMAS - Voice AI Ops +# ============================================ + +# Enums moved to enums.py + + +# Agent Schemas +class AgentCreate(BaseModel): + """Schema for creating a new agent""" + name: str = Field(..., min_length=1, max_length=255) + phone_number: Optional[str] = None + language: LanguageEnum = LanguageEnum.ENGLISH + description: str = Field(..., min_length=1) + call_type: CallTypeEnum = CallTypeEnum.OUTBOUND + call_medium: CallMediumEnum = CallMediumEnum.PHONE_CALL + telephony_phone_number_id: Optional[UUID] = None + voice_bundle_id: UUID = Field(..., description="Required voice bundle for test agent execution") + ai_provider_id: Optional[UUID] = None + voice_ai_integration_id: Optional[UUID] = None + voice_ai_agent_id: Optional[str] = None + provider_prompt: Optional[str] = None + silence_hangup_secs: int = Field( + default=15, + ge=0, + le=600, + description="End live calls after this many seconds of silence (0 disables)", + ) + + @field_validator('description') + @classmethod + def description_min_words(cls, v: str) -> str: + if len(v.split()) < 10: + raise ValueError('Description must be at least 10 words.') + return v + + @field_validator('phone_number') + @classmethod + def phone_number_format(cls, v: Optional[str]) -> Optional[str]: + if v is not None and v != '': + import re + if not re.fullmatch(r'[\d+]+', v): + raise ValueError('Phone number must contain only digits and the + character.') + return v + + @model_validator(mode='after') + def validate_phone_number(self): + """Ensure phone_number is provided when call_medium is phone_call""" + if self.call_medium == CallMediumEnum.PHONE_CALL and not self.phone_number: + raise ValueError('phone_number is required when call_medium is phone_call') + return self + + model_config = ConfigDict(json_schema_extra={ + "example": { + "name": "Customer Support Bot", + "phone_number": "+1234567890", + "language": "en", + "description": "A customer support bot that handles inquiries about orders, returns, and general questions", + "call_type": "outbound", + "voice_bundle_id": "123e4567-e89b-12d3-a456-426614174000", + "voice_ai_integration_id": "123e4567-e89b-12d3-a456-426614174001", + "voice_ai_agent_id": "agent_abc123" + } + }) + + +class AgentUpdate(BaseModel): + """Schema for updating an agent""" + name: Optional[str] = None + phone_number: Optional[str] = None + language: Optional[LanguageEnum] = None + description: Optional[str] = None + call_type: Optional[CallTypeEnum] = None + call_medium: Optional[CallMediumEnum] = None + telephony_phone_number_id: Optional[UUID] = None + voice_bundle_id: Optional[UUID] = None + voice_ai_integration_id: Optional[UUID] = None + voice_ai_agent_id: Optional[str] = None + provider_prompt: Optional[str] = None + prompt_variables: Optional[Dict[str, str]] = None + silence_hangup_secs: Optional[int] = Field(default=None, ge=0, le=600) + + @model_validator(mode='after') + def validate_voice_config(self): + """Validate voice configuration - both voice_bundle_id and voice_ai_integration_id can be provided independently""" + voice_bundle = self.voice_bundle_id + voice_ai_integration = self.voice_ai_integration_id + + # If voice_ai_integration_id is provided, voice_ai_agent_id must also be provided + if voice_ai_integration and not self.voice_ai_agent_id: + raise ValueError('voice_ai_agent_id is required when voice_ai_integration_id is provided.') + + return self + + @model_validator(mode='after') + def validate_phone_number(self): + """Ensure phone_number is provided when call_medium is phone_call""" + # Only validate if call_medium is being set to phone_call + if self.call_medium == CallMediumEnum.PHONE_CALL and not self.phone_number: + # If phone_number is not being updated, we need to check existing value + # This will be handled in the route + pass + return self + + + +class PreviewIntegrationAgentPromptRequest(BaseModel): + """Fetch a provider agent prompt before an EfficientAI agent exists.""" + voice_ai_agent_id: str = Field(..., min_length=1) + + + + +class PreviewIntegrationAgentPromptResponse(BaseModel): + provider_prompt: str + + + + +class AgentPhoneAssignmentConflict(BaseModel): + """Another agent already owns this phone number.""" + agent_id: UUID + agent_name: str + phone_number: str + + + + +class AgentPhoneAssignmentCheckResponse(BaseModel): + """Result of checking whether a phone number is free to assign.""" + available: bool + phone_number: Optional[str] = None + conflict: Optional[AgentPhoneAssignmentConflict] = None + + + + +class TestPromptSectionResponse(BaseModel): + """One canonical section of a generated test agent prompt.""" + key: str + title: str + content: str + + + + +class GeneratedScenarioDraftResponse(BaseModel): + """LLM-generated scenario draft before persistence.""" + name: str + description: str + goal: Optional[str] = None + + + + +class GenerateTestPromptRequest(BaseModel): + """Stage 1: generate foundational test agent prompt from production prompt.""" + production_prompt: str = Field(..., min_length=1) + agent_name: str = Field(..., min_length=1, max_length=255) + language: Optional[str] = None + call_type: Optional[str] = None + provider: Optional[str] = None + model: Optional[str] = None + credential_id: Optional[UUID] = None + llm_config: Optional[Dict[str, Any]] = None + additional_context: Optional[str] = None + + + + +class GenerateTestPromptResponse(BaseModel): + sections: List[TestPromptSectionResponse] + test_agent_prompt: str + provider: str + model: str + + + + +class GenerateScenariosFromPromptRequest(BaseModel): + """Stage 2: generate scenario drafts from test agent prompt.""" + test_agent_prompt: str = Field(..., min_length=1) + agent_name: str = Field(..., min_length=1, max_length=255) + scenario_count: int = Field(default=5, ge=1, le=10) + language: Optional[str] = None + call_type: Optional[str] = None + provider: Optional[str] = None + model: Optional[str] = None + credential_id: Optional[UUID] = None + llm_config: Optional[Dict[str, Any]] = None + additional_context: Optional[str] = None + + + + +class GenerateScenariosFromPromptResponse(BaseModel): + scenarios: List[GeneratedScenarioDraftResponse] + provider: str + model: str + + + + +class GenerateTestSetupRequest(BaseModel): + """Convenience: run stage 1 then stage 2 sequentially.""" + production_prompt: str = Field(..., min_length=1) + agent_name: str = Field(..., min_length=1, max_length=255) + scenario_count: int = Field(default=5, ge=1, le=10) + language: Optional[str] = None + call_type: Optional[str] = None + provider: Optional[str] = None + model: Optional[str] = None + credential_id: Optional[UUID] = None + llm_config: Optional[Dict[str, Any]] = None + additional_context: Optional[str] = None + + + + +class GenerateTestSetupResponse(BaseModel): + sections: List[TestPromptSectionResponse] + test_agent_prompt: str + scenarios: List[GeneratedScenarioDraftResponse] + provider: str + model: str + + + +class AgentResponse(BaseModel): + """Schema for agent response""" + id: UUID + agent_id: Optional[str] = None + name: str + phone_number: Optional[str] = None + language: LanguageEnum + description: Optional[str] + call_type: CallTypeEnum + call_medium: CallMediumEnum + telephony_phone_number_id: Optional[UUID] = None + voice_bundle_id: Optional[UUID] + ai_provider_id: Optional[UUID] + voice_ai_integration_id: Optional[UUID] + voice_ai_agent_id: Optional[str] + provider_prompt: Optional[str] = None + provider_prompt_synced_at: Optional[datetime] = None + created_at: datetime + updated_at: datetime + + @field_validator('language', mode='before') + @classmethod + def convert_language(cls, v): + """Convert string to LanguageEnum (handles uppercase DB values like ENGLISH -> en).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + # Map old uppercase names to new values + language_map = {'english': 'en', 'spanish': 'es', 'french': 'fr', 'german': 'de', + 'chinese': 'zh', 'japanese': 'ja', 'hindi': 'hi', 'arabic': 'ar'} + if v_lower in language_map: + return LanguageEnum(language_map[v_lower]) + try: + return LanguageEnum(v_lower) + except ValueError: + for enum_member in LanguageEnum: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid LanguageEnum value: {v}") + return v + + @field_validator('call_type', mode='before') + @classmethod + def convert_call_type(cls, v): + """Convert string to CallTypeEnum (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return CallTypeEnum(v_lower) + except ValueError: + for enum_member in CallTypeEnum: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid CallTypeEnum value: {v}") + return v + + @field_validator('call_medium', mode='before') + @classmethod + def convert_call_medium(cls, v): + """Convert string to CallMediumEnum (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return CallMediumEnum(v_lower) + except ValueError: + for enum_member in CallMediumEnum: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid CallMediumEnum value: {v}") + return v + + model_config = ConfigDict(from_attributes=True) + + +# Persona Schemas +class PersonaCreate(BaseModel): + """Schema for creating a new persona (TTS provider-tied voice identity)""" + name: str = Field(..., min_length=1, max_length=255) + gender: GenderEnum = GenderEnum.NEUTRAL + tts_provider: Optional[str] = None + tts_voice_id: Optional[str] = None + tts_voice_name: Optional[str] = None + is_custom: bool = False + description: Optional[str] = None + tts_config: Optional[Dict[str, Any]] = None + llm_temperature: Optional[float] = Field(None, ge=0.0, le=2.0) + llm_max_tokens: Optional[int] = Field(None, gt=0, le=8192) + response_delay_ms: Optional[int] = Field(None, ge=0, le=10000) + max_turns: Optional[int] = Field(None, ge=1, le=100) + allow_interruptions: Optional[bool] = None + + @model_validator(mode="after") + def validate_tts_config(self): + from app.services.personas.persona_tts_config import validate_persona_tts_config + + validate_persona_tts_config(self.tts_provider, self.tts_config) + return self + + +class PersonaUpdate(BaseModel): + """Schema for updating a persona""" + name: Optional[str] = None + gender: Optional[GenderEnum] = None + tts_provider: Optional[str] = None + tts_voice_id: Optional[str] = None + tts_voice_name: Optional[str] = None + is_custom: Optional[bool] = None + description: Optional[str] = None + tts_config: Optional[Dict[str, Any]] = None + llm_temperature: Optional[float] = Field(None, ge=0.0, le=2.0) + llm_max_tokens: Optional[int] = Field(None, gt=0, le=8192) + response_delay_ms: Optional[int] = Field(None, ge=0, le=10000) + max_turns: Optional[int] = Field(None, ge=1, le=100) + allow_interruptions: Optional[bool] = None + + @model_validator(mode="after") + def validate_tts_config(self): + from app.services.personas.persona_tts_config import validate_persona_tts_config + + if self.tts_config is not None: + validate_persona_tts_config(self.tts_provider, self.tts_config) + return self + + +class PersonaResponse(BaseModel): + """Schema for persona response""" + id: UUID + name: str + gender: str + tts_provider: Optional[str] = None + tts_voice_id: Optional[str] = None + tts_voice_name: Optional[str] = None + is_custom: bool = False + description: Optional[str] = None + tts_config: Optional[Dict[str, Any]] = None + llm_temperature: Optional[float] = None + llm_max_tokens: Optional[int] = None + response_delay_ms: Optional[int] = None + max_turns: Optional[int] = None + allow_interruptions: Optional[bool] = None + created_at: datetime + updated_at: datetime + + @field_validator('gender', mode='before') + @classmethod + def convert_gender(cls, v): + if v is None: + return "neutral" + if isinstance(v, str): + return v.lower() + if hasattr(v, 'value'): + return v.value + return v + + model_config = ConfigDict(from_attributes=True) + + +class PersonaCloneRequest(BaseModel): + """Schema for cloning a persona""" + name: Optional[str] = None + + +# Scenario Schemas + +class AgentPromptSourcesResponse(BaseModel): + """Prompt texts from an agent that can seed a persona description.""" + agent_id: UUID + agent_name: str + test_agent_prompt: str + agent_prompt: str + + + + +class GeneratePersonaPromptRequest(BaseModel): + """Generate a persona caller prompt from an agent prompt via LLM.""" + agent_id: UUID + source: str = Field(default="auto", pattern="^(test_agent|agent|auto)$") + persona_name: Optional[str] = Field(None, max_length=255) + persona_gender: Optional[str] = None + additional_context: Optional[str] = None + provider: Optional[str] = None + model: Optional[str] = None + credential_id: Optional[UUID] = None + llm_config: Optional[Dict[str, Any]] = None + + + + +class GeneratePersonaPromptResponse(BaseModel): + persona_prompt: str + source_used: str + provider: str + model: str + + +# Scenario Schemas + +class ScenarioCreate(BaseModel): + """Schema for creating a new scenario""" + name: str = Field(..., min_length=1, max_length=255) + agent_id: Optional[UUID] = None + description: Optional[str] = None + required_info: Dict[str, str] = Field(default_factory=dict) + + +class ScenarioUpdate(BaseModel): + """Schema for updating a scenario""" + name: Optional[str] = None + agent_id: Optional[UUID] = None + description: Optional[str] = None + required_info: Optional[Dict[str, str]] = None + + +class ScenarioResponse(BaseModel): + """Schema for scenario response""" + id: UUID + name: str + agent_id: Optional[UUID] + description: Optional[str] + required_info: Dict[str, str] + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +# ============================================ +# IAM & USER SCHEMAS +# ============================================ + +# User Schemas +class UserCreate(BaseModel): + """Schema for creating a user.""" + email: str = Field(..., description="User email address") + name: Optional[str] = None + password: Optional[str] = None # Optional for invitation-based signup + + +class UserUpdate(BaseModel): + """Schema for updating user profile.""" + name: Optional[str] = None + first_name: Optional[str] = None + last_name: Optional[str] = None + email: Optional[str] = None + + +class UserResponse(BaseModel): + """Schema for user response.""" + id: UUID + email: str + name: Optional[str] + first_name: Optional[str] + last_name: Optional[str] + is_active: bool + created_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class OrganizationMemberResponse(BaseModel): + """Schema for organization member response.""" + id: UUID + user_id: UUID + organization_id: UUID + role: RoleEnum + joined_at: datetime + user: UserResponse # Include user details + + model_config = ConfigDict(from_attributes=True) + + +# Invitation Schemas +class InvitationCreate(BaseModel): + """Schema for creating an invitation.""" + email: str = Field(..., description="Email address of the user to invite") + role: RoleEnum = RoleEnum.READER + + +class InvitationResponse(BaseModel): + """Schema for invitation response.""" + id: UUID + organization_id: UUID + email: str + role: RoleEnum + status: InvitationStatus + expires_at: datetime + created_at: datetime + organization_name: Optional[str] = None # Include organization name + + @field_validator('role', mode='before') + @classmethod + def convert_role(cls, v): + """Convert string to RoleEnum (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return RoleEnum(v_lower) + except ValueError: + for enum_member in RoleEnum: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid RoleEnum value: {v}") + return v + + @field_validator('status', mode='before') + @classmethod + def convert_status(cls, v): + """Convert string to InvitationStatus (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return InvitationStatus(v_lower) + except ValueError: + for enum_member in InvitationStatus: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid InvitationStatus value: {v}") + return v + + model_config = ConfigDict(from_attributes=True) + + +class InvitationUpdate(BaseModel): + """Schema for updating invitation (accept/decline).""" + token: str + + +class RoleUpdate(BaseModel): + """Schema for updating user role in organization.""" + role: RoleEnum + + +# Profile Schemas +class ProfileResponse(BaseModel): + """Schema for user profile response.""" + id: UUID + email: str + name: Optional[str] + first_name: Optional[str] + last_name: Optional[str] + created_at: datetime + organizations: List[dict] = Field(default_factory=list) # List of org memberships + + model_config = ConfigDict(from_attributes=True) + + +# ============================================ +# INTEGRATION SCHEMAS +# ============================================ + +class IntegrationCreate(BaseModel): + """Schema for creating an integration.""" + platform: IntegrationPlatform + api_key: str = Field(..., description="Private API key for the platform") + public_key: Optional[str] = Field(None, description="Optional public API key (e.g. for Vapi)") + name: Optional[str] = Field(None, description="Optional friendly name for the integration") + routing_mode: CredentialRoutingMode = Field( + CredentialRoutingMode.INHERIT, + description="LLM routing preference: inherit org default, force gateway, or direct API key.", + ) + is_default: Optional[bool] = Field( + None, + description=( + "Mark this credential as the default for the (org, platform). " + "If omitted and no default exists yet, this row becomes the default." + ), + ) + + +class IntegrationUpdate(BaseModel): + """Schema for updating an integration.""" + name: Optional[str] = None + api_key: Optional[str] = None + public_key: Optional[str] = None + is_active: Optional[bool] = None + routing_mode: Optional[CredentialRoutingMode] = None + + +class IntegrationResponse(BaseModel): + """Schema for integration response.""" + id: UUID + organization_id: UUID + platform: IntegrationPlatform + name: Optional[str] + public_key: Optional[str] = None + is_active: bool + is_default: bool = False + routing_mode: CredentialRoutingMode = CredentialRoutingMode.INHERIT + effective_routing: Literal["inherit", "direct", "gateway", "bifrost", "litellm_proxy"] = "inherit" + created_at: datetime + updated_at: datetime + last_tested_at: Optional[datetime] = None + # Note: api_key is NOT included in response for security + + @field_validator('platform', mode='before') + @classmethod + def convert_platform(cls, v): + """Convert string to IntegrationPlatform enum if needed (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + # Try lowercase first (enum value) + v_lower = v.lower() + try: + return IntegrationPlatform(v_lower) + except ValueError: + # Try to find by enum name (uppercase) + for enum_member in IntegrationPlatform: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid IntegrationPlatform value: {v}") + return v + + @field_validator('routing_mode', mode='before') + @classmethod + def convert_routing_mode(cls, v): + if v is None: + return CredentialRoutingMode.INHERIT + if isinstance(v, str): + try: + return CredentialRoutingMode(v.lower()) + except ValueError: + return CredentialRoutingMode.INHERIT + return v + + model_config = ConfigDict(from_attributes=True) + + +# ============================================ +# DATA SOURCES SCHEMAS +# ============================================ + +class S3ConnectionTest(BaseModel): + """Schema for testing S3 connection.""" + bucket_name: str + region: str = "us-east-1" + access_key_id: str + secret_access_key: str + endpoint_url: Optional[str] = None + + +class S3ConnectionTestResponse(BaseModel): + """Schema for S3 connection test response.""" + success: bool + message: str + bucket_name: Optional[str] = None + + +class S3FileInfo(BaseModel): + """Schema for S3 file information.""" + key: str + filename: str + size: int + last_modified: str + + +class S3ListFilesResponse(BaseModel): + """Schema for listing S3 files response.""" + files: List[S3FileInfo] + total: int + prefix: Optional[str] = None + + +class S3FolderInfo(BaseModel): + """Schema for S3 folder information.""" + name: str + path: str + + +class S3BrowseResponse(BaseModel): + """Schema for browsing S3 folders within an organization.""" + folders: List[S3FolderInfo] + files: List[S3FileInfo] + current_path: str + organization_id: str + + +class S3UploadResponse(BaseModel): + """Schema for S3 upload response.""" + key: str + bucket: str + file_id: UUID + message: str + + +# AIProvider Schemas +_MAX_GATEWAY_EXTRA_HEADERS = 20 + + +def _validate_gateway_extra_headers( + value: Optional[Dict[str, Any]], +) -> Optional[Dict[str, str]]: + if value is None: + return None + if not isinstance(value, dict): + raise ValueError("gateway_extra_headers must be a JSON object of string keys and values.") + if len(value) > _MAX_GATEWAY_EXTRA_HEADERS: + raise ValueError( + f"gateway_extra_headers supports at most {_MAX_GATEWAY_EXTRA_HEADERS} headers." + ) + normalized: Dict[str, str] = {} + for raw_key, raw_val in value.items(): + key = str(raw_key).strip() + if not key: + raise ValueError("gateway_extra_headers keys must be non-empty strings.") + if len(key) > 64 or any(ch.isspace() for ch in key): + raise ValueError(f"Invalid gateway header name: {key!r}") + if raw_val is None: + raise ValueError(f"gateway_extra_headers[{key!r}] must be a string value.") + val = str(raw_val).strip() + if not val: + raise ValueError(f"gateway_extra_headers[{key!r}] must be a non-empty string.") + if len(val) > 1024 or "\n" in val or "\r" in val: + raise ValueError(f"gateway_extra_headers[{key!r}] value is invalid.") + normalized[key] = val + return normalized or None + + +class AIProviderCreate(BaseModel): + """Schema for creating an AI Provider.""" + provider: ModelProvider + api_key: Optional[str] = Field( + None, + description=( + "Provider API key. Optional when routing via gateway with " + "gateway-managed credentials (passthrough_provider_keys: false)." + ), + ) + name: Optional[str] = None + routing_mode: CredentialRoutingMode = Field( + CredentialRoutingMode.INHERIT, + description="LLM routing preference: inherit org default, force gateway, or direct API key.", + ) + gateway_model: Optional[str] = Field( + None, + min_length=1, + max_length=255, + description="Bifrost custom model ID sent when routing via gateway.", + ) + gateway_interface: GatewayInterfaceMode = Field( + GatewayInterfaceMode.INHERIT, + description="Bifrost API surface: inherit org default, LiteLLM shim, or native OpenAI-compatible.", + ) + gateway_base_url: Optional[str] = Field( + None, + max_length=512, + description="Optional per-credential Bifrost/gateway base URL override.", + ) + gateway_auth_header: Optional[str] = Field( + None, + max_length=64, + description="Auth header name for Bifrost (default x-bf-vk).", + ) + gateway_auth_secret_env: Optional[str] = Field( + None, + max_length=128, + description="Environment variable name holding the gateway auth secret.", + ) + gateway_auth_secret: Optional[str] = Field( + None, + description="Inline gateway auth secret (encrypted at rest). Alternative to env var.", + ) + gateway_extra_headers: Optional[Dict[str, str]] = Field( + None, + description="Arbitrary HTTP headers sent with gateway-routed LiteLLM calls.", + ) + is_default: Optional[bool] = Field( + None, + description=( + "Mark this credential as the default for the (org, provider). " + "If omitted and no default exists yet, this row becomes the default." + ), + ) + endpoint_url: Optional[str] = Field( + None, + description="Provider endpoint URL (required for Azure OpenAI).", + ) + + @field_validator("api_key") + @classmethod + def validate_api_key(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("endpoint_url") + @classmethod + def validate_endpoint_url(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_model") + @classmethod + def validate_gateway_model(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_base_url") + @classmethod + def validate_gateway_base_url(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_auth_header") + @classmethod + def validate_gateway_auth_header(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + if not trimmed: + return None + if len(trimmed) > 64 or any(ch.isspace() for ch in trimmed): + raise ValueError("gateway_auth_header must be a single non-empty header name.") + return trimmed + + @field_validator("gateway_auth_secret_env") + @classmethod + def validate_gateway_auth_secret_env(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + if not trimmed: + return None + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", trimmed): + raise ValueError( + "gateway_auth_secret_env must be a valid environment variable name." + ) + return trimmed + + @field_validator("gateway_auth_secret") + @classmethod + def validate_gateway_auth_secret(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_extra_headers") + @classmethod + def validate_gateway_extra_headers(cls, v: Optional[Dict[str, Any]]) -> Optional[Dict[str, str]]: + return _validate_gateway_extra_headers(v) + + +class AIProviderUpdate(BaseModel): + """Schema for updating an AI Provider.""" + api_key: Optional[str] = Field(None, min_length=1) + name: Optional[str] = None + endpoint_url: Optional[str] = None + is_active: Optional[bool] = None + routing_mode: Optional[CredentialRoutingMode] = None + gateway_model: Optional[str] = Field(None, min_length=1, max_length=255) + gateway_interface: Optional[GatewayInterfaceMode] = None + gateway_base_url: Optional[str] = Field(None, max_length=512) + gateway_auth_header: Optional[str] = Field(None, max_length=64) + gateway_auth_secret_env: Optional[str] = Field(None, max_length=128) + gateway_auth_secret: Optional[str] = None + clear_gateway_auth_secret: bool = False + gateway_extra_headers: Optional[Dict[str, str]] = None + + @field_validator("gateway_model") + @classmethod + def validate_gateway_model(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_base_url") + @classmethod + def validate_gateway_base_url_update(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_auth_header") + @classmethod + def validate_gateway_auth_header_update(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + if not trimmed: + return None + if len(trimmed) > 64 or any(ch.isspace() for ch in trimmed): + raise ValueError("gateway_auth_header must be a single non-empty header name.") + return trimmed + + @field_validator("gateway_auth_secret_env") + @classmethod + def validate_gateway_auth_secret_env_update(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + if not trimmed: + return None + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", trimmed): + raise ValueError( + "gateway_auth_secret_env must be a valid environment variable name." + ) + return trimmed + + @field_validator("gateway_auth_secret") + @classmethod + def validate_gateway_auth_secret_update(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_extra_headers") + @classmethod + def validate_gateway_extra_headers_update( + cls, v: Optional[Dict[str, Any]] + ) -> Optional[Dict[str, str]]: + return _validate_gateway_extra_headers(v) + + +class AIProviderResponse(BaseModel): + """Schema for AI Provider response.""" + id: UUID + provider: ModelProvider + api_key: Optional[str] = None # Will be None in response for security + name: Optional[str] + endpoint_url: Optional[str] = None + is_active: bool + is_default: bool = False + routing_mode: CredentialRoutingMode = CredentialRoutingMode.INHERIT + gateway_model: Optional[str] = None + gateway_interface: GatewayInterfaceMode = GatewayInterfaceMode.INHERIT + gateway_base_url: Optional[str] = None + gateway_auth_header: Optional[str] = None + gateway_auth_secret_env: Optional[str] = None + has_gateway_auth_secret: bool = False + gateway_extra_headers: Optional[Dict[str, str]] = None + gateway_managed: bool = False + effective_routing: Literal["inherit", "direct", "gateway", "bifrost", "litellm_proxy"] = "inherit" + effective_gateway_interface: Literal["litellm_shim", "native_openai"] = "litellm_shim" + created_at: datetime + updated_at: datetime + last_tested_at: Optional[datetime] + + @field_validator('provider', mode='before') + @classmethod + def convert_provider(cls, v): + """Convert string to ModelProvider (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return ModelProvider(v_lower) + except ValueError: + for enum_member in ModelProvider: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid ModelProvider value: {v}") + return v + + @field_validator('routing_mode', mode='before') + @classmethod + def convert_routing_mode(cls, v): + if v is None: + return CredentialRoutingMode.INHERIT + if isinstance(v, str): + try: + return CredentialRoutingMode(v.lower()) + except ValueError: + return CredentialRoutingMode.INHERIT + return v + + model_config = ConfigDict(from_attributes=True) + + +class LLMGenerationConfig(BaseModel): + """User-tunable LLM sampling / generation parameters.""" + + temperature: Optional[float] = Field(None, ge=0.0, le=2.0) + max_tokens: Optional[int] = Field(None, gt=0) + top_p: Optional[float] = Field(None, ge=0.0, le=1.0) + top_k: Optional[int] = Field(None, ge=0) + frequency_penalty: Optional[float] = Field(None, ge=-2.0, le=2.0) + presence_penalty: Optional[float] = Field(None, ge=-2.0, le=2.0) + seed: Optional[int] = Field(None, ge=0) + + def to_dict(self) -> Dict[str, Any]: + """Return only explicitly set fields.""" + return self.model_dump(exclude_none=True) + + +# VoiceBundle Schemas +class VoiceBundleCreate(BaseModel): + """Schema for creating a VoiceBundle.""" + name: str = Field(..., min_length=1, max_length=255) + description: Optional[str] = None + + # Bundle type: either STT+LLM+TTS or S2S + bundle_type: VoiceBundleType = Field(default=VoiceBundleType.STT_LLM_TTS) + + # STT Configuration - required for STT_LLM_TTS, optional for S2S + stt_provider: Optional[ModelProvider] = None + stt_model: Optional[str] = Field(None, min_length=1) + stt_credential_id: Optional[UUID] = Field( + None, + description=( + "Optional explicit AIProvider/Integration row id to use for STT. " + "When omitted the resolver picks the default credential for stt_provider." + ), + ) + + # LLM Configuration - required for STT_LLM_TTS, optional for S2S + llm_provider: Optional[ModelProvider] = None + llm_model: Optional[str] = Field(None, min_length=1) + llm_temperature: Optional[float] = Field(None, ge=0.0, le=2.0) + llm_max_tokens: Optional[int] = Field(None, gt=0) + llm_config: Optional[Dict[str, Any]] = None + llm_credential_id: Optional[UUID] = None + + # TTS Configuration - required for STT_LLM_TTS, optional for S2S + tts_provider: Optional[ModelProvider] = None + tts_model: Optional[str] = Field(None, min_length=1) + tts_voice: Optional[str] = None + tts_config: Optional[Dict[str, Any]] = None + tts_credential_id: Optional[UUID] = None + + # S2S Configuration - required for S2S, optional for STT_LLM_TTS + s2s_provider: Optional[ModelProvider] = None + s2s_model: Optional[str] = Field(None, min_length=1) + s2s_config: Optional[Dict[str, Any]] = None + s2s_credential_id: Optional[UUID] = None + + # Additional metadata + extra_metadata: Optional[Dict[str, Any]] = None + + @model_validator(mode='after') + def validate_bundle_configuration(self): + """Validate that required fields are provided based on bundle_type.""" + if self.bundle_type == VoiceBundleType.STT_LLM_TTS: + if not self.stt_provider or not self.stt_model: + raise ValueError('STT provider and model are required for STT_LLM_TTS bundle type') + if not self.llm_provider or not self.llm_model: + raise ValueError('LLM provider and model are required for STT_LLM_TTS bundle type') + if not self.tts_provider or not self.tts_model: + raise ValueError('TTS provider and model are required for STT_LLM_TTS bundle type') + elif self.bundle_type == VoiceBundleType.S2S: + if not self.s2s_provider or not self.s2s_model: + raise ValueError('S2S provider and model are required for S2S bundle type') + return self + + +class VoiceBundleUpdate(BaseModel): + """Schema for updating a VoiceBundle.""" + name: Optional[str] = Field(None, min_length=1, max_length=255) + description: Optional[str] = None + + # Bundle type + bundle_type: Optional[VoiceBundleType] = None + + # STT Configuration + stt_provider: Optional[ModelProvider] = None + stt_model: Optional[str] = Field(None, min_length=1) + stt_credential_id: Optional[UUID] = None + + # LLM Configuration + llm_provider: Optional[ModelProvider] = None + llm_model: Optional[str] = Field(None, min_length=1) + llm_temperature: Optional[float] = Field(None, ge=0.0, le=2.0) + llm_max_tokens: Optional[int] = Field(None, gt=0) + llm_config: Optional[Dict[str, Any]] = None + llm_credential_id: Optional[UUID] = None + + # TTS Configuration + tts_provider: Optional[ModelProvider] = None + tts_model: Optional[str] = Field(None, min_length=1) + tts_voice: Optional[str] = None + tts_config: Optional[Dict[str, Any]] = None + tts_credential_id: Optional[UUID] = None + + # S2S Configuration + s2s_provider: Optional[ModelProvider] = None + s2s_model: Optional[str] = Field(None, min_length=1) + s2s_config: Optional[Dict[str, Any]] = None + s2s_credential_id: Optional[UUID] = None + + # Additional metadata + extra_metadata: Optional[Dict[str, Any]] = None + is_active: Optional[bool] = None + + +class VoiceBundleResponse(BaseModel): + """Schema for VoiceBundle response.""" + id: UUID + name: str + description: Optional[str] + + # Bundle type - can be string from DB or enum, validator handles conversion + bundle_type: VoiceBundleType + + @field_validator('bundle_type', mode='before') + @classmethod + def convert_bundle_type(cls, v): + """Convert string to VoiceBundleType enum if needed.""" + if isinstance(v, str): + try: + return VoiceBundleType(v) + except ValueError: + # Try to find by value + for enum_member in VoiceBundleType: + if enum_member.value == v: + return enum_member + raise ValueError(f"Invalid bundle_type value: {v}") + return v + + @field_validator('stt_provider', 'llm_provider', 'tts_provider', 's2s_provider', mode='before') + @classmethod + def convert_model_provider(cls, v): + """Convert string to ModelProvider enum if needed (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + # Try lowercase first (enum value) + v_lower = v.lower() + try: + return ModelProvider(v_lower) + except ValueError: + # Try to find by enum name (uppercase) + for enum_member in ModelProvider: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid ModelProvider value: {v}") + return v + + # STT Configuration + stt_provider: Optional[ModelProvider] + stt_model: Optional[str] + stt_credential_id: Optional[UUID] = None + + # LLM Configuration + llm_provider: Optional[ModelProvider] + llm_model: Optional[str] + llm_temperature: Optional[float] + llm_max_tokens: Optional[int] + llm_config: Optional[Dict[str, Any]] + llm_credential_id: Optional[UUID] = None + + # TTS Configuration + tts_provider: Optional[ModelProvider] + tts_model: Optional[str] + tts_voice: Optional[str] + tts_config: Optional[Dict[str, Any]] + tts_credential_id: Optional[UUID] = None + + # S2S Configuration + s2s_provider: Optional[ModelProvider] + s2s_model: Optional[str] + s2s_config: Optional[Dict[str, Any]] + s2s_credential_id: Optional[UUID] = None + + # Additional metadata + extra_metadata: Optional[Dict[str, Any]] + is_active: bool + created_at: datetime + updated_at: datetime + created_by: Optional[str] + + model_config = ConfigDict(from_attributes=True) + + +# Test Agent Conversation Schemas +class TestAgentConversationCreate(BaseModel): + """Schema for creating a new test agent conversation.""" + agent_id: UUID + persona_id: UUID + scenario_id: UUID + voice_bundle_id: UUID + conversation_metadata: Optional[Dict[str, Any]] = None + + model_config = ConfigDict(json_schema_extra={ + "example": { + "agent_id": "123e4567-e89b-12d3-a456-426614174000", + "persona_id": "123e4567-e89b-12d3-a456-426614174001", + "scenario_id": "123e4567-e89b-12d3-a456-426614174002", + "voice_bundle_id": "123e4567-e89b-12d3-a456-426614174003" + } + }) + + +class TestAgentConversationUpdate(BaseModel): + """Schema for updating a test agent conversation.""" + status: Optional[str] = None + live_transcription: Optional[List[Dict[str, Any]]] = None + full_transcript: Optional[str] = None + conversation_metadata: Optional[Dict[str, Any]] = None + + +class TestAgentConversationResponse(BaseModel): + """Schema for test agent conversation response.""" + id: UUID + organization_id: UUID + agent_id: UUID + persona_id: UUID + scenario_id: UUID + voice_bundle_id: UUID + status: str + live_transcription: Optional[List[Dict[str, Any]]] + conversation_audio_key: Optional[str] + full_transcript: Optional[str] + started_at: datetime + ended_at: Optional[datetime] + duration_seconds: Optional[float] + conversation_metadata: Optional[Dict[str, Any]] + created_at: datetime + updated_at: datetime + created_by: Optional[str] + + model_config = ConfigDict(from_attributes=True) + + +class ConversationTurn(BaseModel): + """Schema for a single conversation turn.""" + speaker: str # "test_agent" or "voice_agent" + text: str + timestamp: float # Time in seconds from start + audio_segment_key: Optional[str] = None # S3 key for this segment's audio + + +# Conversation Evaluation Schemas +class ConversationEvaluationCreate(BaseModel): + """Schema for creating a conversation evaluation.""" + transcription_id: UUID + agent_id: UUID + llm_provider: Optional[ModelProvider] = ModelProvider.OPENAI + llm_model: Optional[str] = "gpt-4o" + + model_config = ConfigDict(json_schema_extra={ + "example": { + "transcription_id": "123e4567-e89b-12d3-a456-426614174000", + "agent_id": "123e4567-e89b-12d3-a456-426614174001", + "llm_provider": "openai", + "llm_model": "gpt-4o" + } + }) + + +class ConversationEvaluationResponse(BaseModel): + """Schema for conversation evaluation response.""" + id: UUID + organization_id: UUID + transcription_id: UUID + agent_id: UUID + objective_achieved: bool + objective_achieved_reason: Optional[str] + additional_metrics: Optional[Dict[str, Any]] + overall_score: Optional[float] + llm_provider: Optional[ModelProvider] + llm_model: Optional[str] + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +# Evaluator Schemas +class EvaluatorCreate(BaseModel): + """Schema for creating an evaluator. Either provide agent_id+persona_id+scenario_id (standard) or metric_ids/custom_prompt (custom).""" + name: Optional[str] = None + agent_id: Optional[UUID] = None + persona_id: Optional[UUID] = None + scenario_id: Optional[UUID] = None + custom_prompt: Optional[str] = None + metric_ids: Optional[List[UUID]] = None + llm_provider: Optional[ModelProvider] = None + llm_model: Optional[str] = None + llm_config: Optional[Dict[str, Any]] = None + tags: Optional[List[str]] = None + + +class EvaluatorUpdate(BaseModel): + """Schema for updating an evaluator.""" + name: Optional[str] = None + agent_id: Optional[UUID] = None + persona_id: Optional[UUID] = None + scenario_id: Optional[UUID] = None + custom_prompt: Optional[str] = None + metric_ids: Optional[List[UUID]] = None + llm_provider: Optional[ModelProvider] = None + llm_model: Optional[str] = None + llm_config: Optional[Dict[str, Any]] = None + tags: Optional[List[str]] = None + + +class EvaluatorResponse(BaseModel): + """Schema for evaluator response.""" + id: UUID + evaluator_id: str + organization_id: UUID + name: Optional[str] = None + agent_id: Optional[UUID] = None + persona_id: Optional[UUID] = None + scenario_id: Optional[UUID] = None + custom_prompt: Optional[str] = None + metric_ids: Optional[List[UUID]] = None + llm_provider: Optional[ModelProvider] = None + llm_model: Optional[str] = None + llm_config: Optional[Dict[str, Any]] = None + tags: Optional[List[str]] + created_at: datetime + updated_at: datetime + created_by: Optional[str] + + @field_validator('llm_provider', mode='before') + @classmethod + def convert_llm_provider(cls, v): + """Convert string to ModelProvider (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return ModelProvider(v_lower) + except ValueError: + for enum_member in ModelProvider: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid ModelProvider value: {v}") + return v + + model_config = ConfigDict(from_attributes=True) + + +class EvaluatorBulkCreate(BaseModel): + """Schema for creating multiple evaluators at once.""" + name: Optional[str] = None + agent_id: UUID + scenario_id: UUID + persona_ids: List[UUID] + tags: Optional[List[str]] = None + + +class RunEvaluatorsRequest(BaseModel): + """Schema for running evaluators.""" + evaluator_ids: List[UUID] = Field(..., description="List of evaluator IDs to run") + + +class RunEvaluatorsResponse(BaseModel): + """Schema for run evaluators response.""" + task_ids: List[str] = Field(..., description="List of Celery task IDs for tracking") + evaluator_results: List["EvaluatorResultResponse"] = Field(default_factory=list, description="List of created evaluator results") + + model_config = ConfigDict( + from_attributes=True, + json_schema_extra={ + "example": { + "agent_id": "123e4567-e89b-12d3-a456-426614174000", + "scenario_id": "123e4567-e89b-12d3-a456-426614174002", + "persona_ids": [ + "123e4567-e89b-12d3-a456-426614174001", + "123e4567-e89b-12d3-a456-426614174003" + ], + "tags": ["test", "production"] + } + }, + ) + + +# Metric Schemas +SelectionMode = Literal["single_choice", "multi_label"] + + +MetricScope = Literal["workspace", "organization"] + +# Max length for metric rubric text (description / example) accepted by +# the API. DB columns are TEXT or unbounded VARCHAR; this cap is validation-only. +METRIC_RUBRIC_TEXT_MAX_LENGTH = 32_000 + + + +class EvaluatorSuiteCombinationResponse(BaseModel): + """One agent+persona+scenario combination inside a suite.""" + id: UUID + evaluator_id: str + scenario_id: Optional[UUID] = None + scenario_name: Optional[str] = None + scenario_description: Optional[str] = None + scenario_required_info: Optional[Any] = None + + + + +class EvaluatorSuiteCreate(BaseModel): + """Schema for creating an evaluator suite.""" + name: Optional[str] = None + agent_id: UUID + persona_id: UUID + scenario_ids: List[UUID] + metric_ids: Optional[List[UUID]] = None + llm_provider: Optional[ModelProvider] = None + llm_model: Optional[str] = None + llm_config: Optional[Dict[str, Any]] = None + tags: Optional[List[str]] = None + default_runs_per_combination: int = 1 + + + + +class EvaluatorSuiteUpdate(BaseModel): + """Schema for updating an evaluator suite.""" + name: Optional[str] = None + tags: Optional[List[str]] = None + default_runs_per_combination: Optional[int] = None + llm_provider: Optional[ModelProvider] = None + llm_model: Optional[str] = None + llm_config: Optional[Dict[str, Any]] = None + metric_ids: Optional[List[UUID]] = None + + + + +class EvaluatorSuiteResponse(BaseModel): + """Schema for evaluator suite response.""" + id: UUID + organization_id: UUID + name: Optional[str] = None + agent_id: UUID + persona_id: UUID + agent_name: Optional[str] = None + persona_name: Optional[str] = None + agent_call_type: Optional[str] = None + agent_call_medium: Optional[str] = None + metric_ids: Optional[List[str]] = None + llm_provider: Optional[str] = None + llm_model: Optional[str] = None + llm_config: Optional[Dict[str, Any]] = None + tags: Optional[List[str]] = None + default_runs_per_combination: int = 1 + round_robin_index: int = 0 + is_active: bool = False + agent_suite_count: int = 1 + combination_count: int = 0 + combinations: List[EvaluatorSuiteCombinationResponse] = Field(default_factory=list) + created_at: datetime + updated_at: datetime + created_by: Optional[str] = None + + + + +class EvaluatorSuiteAddScenariosRequest(BaseModel): + """Schema for adding scenarios to an existing suite.""" + scenario_ids: List[UUID] + + + + +class RunEvaluatorSuiteRequest(BaseModel): + """Schema for running all combinations in a suite.""" + runs_per_combination: Optional[int] = None + to_number: Optional[str] = None + from_number: Optional[str] = None + + + + +class RunEvaluatorSuiteResponse(BaseModel): + """Schema for suite run response.""" + total_runs: int + task_ids: List[str] = Field(default_factory=list) + evaluator_results: List["EvaluatorResultResponse"] = Field(default_factory=list) + phone_call_refs: List[str] = Field(default_factory=list) + + + + +class RunNextCombinationRequest(BaseModel): + """Schema for running the next round-robin combination.""" + from_number: Optional[str] = None + + + + +class RunNextCombinationResponse(BaseModel): + """Schema for round-robin run response.""" + evaluator_id: UUID + scenario_id: Optional[UUID] = None + scenario_name: str + combination_index: int + next_index: int + evaluator_result_id: Optional[UUID] = None + result_id: Optional[str] = None + task_id: Optional[str] = None + phone_call_ref: Optional[str] = None + call_short_id: Optional[str] = None + + + + +class ChooseNextCombinationResponse(BaseModel): + """Advance inbound round-robin without initiating a call or evaluation run.""" + evaluator_id: UUID + scenario_id: Optional[UUID] = None + scenario_name: str + combination_index: int + next_index: int + + +# Metric Schemas +SelectionMode = Literal["single_choice", "multi_label"] + + +MetricScope = Literal["workspace", "organization"] + +# Max length for metric rubric text (description / example) accepted by +# the API. DB columns are TEXT or unbounded VARCHAR; this cap is validation-only. +METRIC_RUBRIC_TEXT_MAX_LENGTH = 32_000 + + + +class MetricCreate(BaseModel): + """Schema for creating a metric. + + Hierarchy: + - ``parent_metric_id`` set => this is a child sub-metric. ``metric_type`` + is forced to ``boolean`` server-side; ``selection_mode`` must be None. + - ``selection_mode`` set => this is a parent category metric. + ``parent_metric_id`` must be None (max depth = 2). + + Scope: + - ``scope="workspace"`` (default) stamps the metric with the active + ``X-Workspace-Id`` so it only shows up inside that workspace. + - ``scope="organization"`` stamps ``workspace_id=NULL`` so the metric + is visible in every workspace of the org. Children always inherit + their parent's scope; setting ``scope`` on a child request body is + ignored server-side. + """ + name: str + description: Optional[str] = None + # Optional illustrative example surfaced alongside ``description`` + # in the LLM judge's rubric. Today this is mainly populated on + # child sub-labels (one example per categorization label) but + # standalone metrics may carry it too without a schema change. + example: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) + metric_type: MetricType = MetricType.RATING + metric_category: MetricCategory = MetricCategory.QUALITY + trigger: MetricTrigger = MetricTrigger.ALWAYS + enabled: bool = True + metric_origin: str = "custom" + supported_surfaces: List[str] = ["agent"] + enabled_surfaces: Optional[List[str]] = None + custom_data_type: Optional[str] = None + custom_config: Optional[Dict[str, Any]] = None + tags: Optional[List[str]] = None + capture_rationale: Optional[bool] = False + parent_metric_id: Optional[UUID] = None + selection_mode: Optional[SelectionMode] = None + # Only meaningful on multi_label parents; ignored everywhere else. + # When true, the LLM is invited during call-import evaluation to emit + # additional candidate sub-labels beyond the user-defined children. + allow_discovery: bool = False + # When true, this metric is a "transcript-compare judge": at + # call-import evaluation time the worker feeds BOTH the production + # transcript (``call_import_rows.transcript``, CSV-supplied) and + # the diarised transcript (``call_import_rows.diarised_transcript``, + # worker-produced) to the LLM as a labeled pair. The parent + # evaluation's ``transcript_source`` is ignored for these metrics. + # Mutually exclusive with ``parent_metric_id`` / ``selection_mode`` + # G�� comparison metrics stay standalone so the LLM grouping logic + # doesn't have to second-guess which prompt template to use within + # a hierarchy. (Parent-level keyword auto-detection in the worker + # still routes a categorisation parent through the comparison + # prompt without setting this flag.) + compare_transcripts: bool = False + # When ``"organization"``, the metric is stored with + # ``workspace_id=NULL`` so it surfaces in every workspace of the + # caller's org. Default ``"workspace"`` preserves the historical + # behavior of stamping the metric with the active ``X-Workspace-Id``. + # Ignored when ``parent_metric_id`` is set (children inherit the + # parent's scope unconditionally). + scope: MetricScope = "workspace" + + @model_validator(mode='after') + def validate_compare_transcripts_exclusions(self): + """Reject body combinations that don't make sense for a + transcript-compare judge. + + The Metric ORM column accepts the value; the validator just + prevents the user from accidentally requesting an incoherent + metric shape (e.g. "compare two transcripts but also live + inside a categorisation hierarchy" � different prompt + templates). + """ + if not self.compare_transcripts: + return self + if self.parent_metric_id is not None: + raise ValueError( + "Transcript-compare metrics must be standalone: " + "they cannot be a child sub-metric in this version." + ) + if self.selection_mode is not None: + raise ValueError( + "Transcript-compare metrics must be standalone: " + "they cannot own children (selection_mode must be " + "unset) in this version." + ) + return self + + model_config = ConfigDict(json_schema_extra={ + "example": { + "name": "Professionalism", + "description": "Measures the professional tone and behavior", + "metric_type": "rating", + "trigger": "always", + "enabled": True + } + }) + + +class MetricChildDraft(BaseModel): + """One child sub-metric in a parent + children atomic create body.""" + + name: str = Field(..., max_length=120) + description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) + # Optional illustrative example for this label. Surfaced alongside + # ``description`` in the LLM judge's rubric so each label can carry + # both its definition AND a "what does this look like?" example. + example: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) + enabled: bool = True + capture_rationale: Optional[bool] = True + tags: Optional[List[str]] = None + + +class MetricCreateWithChildren(BaseModel): + """One-shot create body: a parent metric + N children, atomically. + + Children are persisted as full ``Metric`` rows with + ``parent_metric_id`` set to the new parent. ``metric_type`` on every + child is forced to ``boolean`` server-side regardless of what's + passed in the parent body. + """ + + name: str = Field(..., max_length=120) + description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) + selection_mode: SelectionMode + metric_category: MetricCategory = MetricCategory.QUALITY + enabled: bool = True + supported_surfaces: List[str] = Field(default_factory=lambda: ["agent"]) + enabled_surfaces: Optional[List[str]] = None + tags: Optional[List[str]] = None + # When true on a multi_label parent, allow the LLM to emit candidate + # labels beyond the listed children at evaluation time. Validator + # rejects allow_discovery=True on single_choice parents. + allow_discovery: bool = False + # Parent-level "Enable LLM Rationale" toggle. When true the LLM + # judge emits a single rationale string at the parent level + # (children never carry rationales in hierarchical mode), which the + # table renders as the " - LLM Rationale" column. + capture_rationale: bool = False + children: List[MetricChildDraft] = Field( + default_factory=list, + description="Child sub-metric labels under this parent.", + ) + # See ``MetricCreate.scope``. Same semantics: ``"organization"`` + # creates the parent + all children with ``workspace_id=NULL`` so + # the whole category subtree is shared across every workspace in + # the org. + scope: MetricScope = "workspace" + + +class MetricUpdate(BaseModel): + """Schema for updating a metric.""" + name: Optional[str] = None + description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) + # ``None`` here means "leave unchanged"; pass an empty string to + # clear a previously stored example. + example: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) + metric_type: Optional[MetricType] = None + trigger: Optional[MetricTrigger] = None + enabled: Optional[bool] = None + metric_origin: Optional[str] = None + supported_surfaces: Optional[List[str]] = None + enabled_surfaces: Optional[List[str]] = None + custom_data_type: Optional[str] = None + custom_config: Optional[Dict[str, Any]] = None + tags: Optional[List[str]] = None + metric_category: Optional[MetricCategory] = None + capture_rationale: Optional[bool] = None + selection_mode: Optional[SelectionMode] = None + allow_discovery: Optional[bool] = None + # See ``MetricCreate.compare_transcripts``. ``None`` here means + # "leave unchanged". The route layer enforces mutual exclusion + # against the row's existing ``parent_metric_id`` / + # ``selection_mode`` when this is set to True, because the patch + # body alone doesn't have enough context to validate cross-state. + compare_transcripts: Optional[bool] = None + + @model_validator(mode='after') + def validate_compare_transcripts_exclusions(self): + """Reject patch bodies that flip compare_transcripts on while + ALSO trying to set a conflicting field in the same request. + + Cross-state validation against the persisted row (e.g. "the + existing metric already has a parent") is done in the + update route since the schema doesn't have the row in hand. + """ + if self.compare_transcripts is not True: + return self + if self.selection_mode is not None: + raise ValueError( + "Transcript-compare metrics must be standalone: " + "selection_mode must be cleared before enabling " + "compare_transcripts." + ) + return self + + +class MetricResponse(BaseModel): + """Schema for metric response. + + ``children`` is populated for parent metrics (those with + ``selection_mode`` set) and is otherwise an empty list. The list is + built once at serialization time so callers get a single tree + structure without follow-up requests. + """ + id: UUID + organization_id: UUID + # ``None`` when the metric is org-shared (``scope == "organization"``). + # See the ORM ``Metric.workspace_id`` docstring. + workspace_id: Optional[UUID] = None + # Computed convenience field so the UI doesn't have to do + # ``workspace_id == null`` checks everywhere. Always one of + # ``"workspace"`` or ``"organization"``. + scope: MetricScope = "workspace" + name: str + description: Optional[str] + # Optional illustrative example. Populated mainly on categorization + # child labels but surfaced for every metric so the UI can render + # it uniformly without branching on parent/child shape. + example: Optional[str] = None + metric_type: MetricType + metric_category: MetricCategory = MetricCategory.QUALITY + trigger: MetricTrigger + enabled: bool + is_default: bool + metric_origin: str + supported_surfaces: List[str] + enabled_surfaces: List[str] + custom_data_type: Optional[str] + custom_config: Optional[Dict[str, Any]] + tags: Optional[List[str]] + capture_rationale: bool = False + parent_metric_id: Optional[UUID] = None + selection_mode: Optional[SelectionMode] = None + allow_discovery: bool = False + # See ``MetricCreate.compare_transcripts``. Surfaced so the UI can + # render a "Compare transcripts" badge in the metric picker and + # know to skip the run's transcript_source toggle for this metric. + compare_transcripts: bool = False + lifecycle: str = "active" + promoted_from_draft_at: Optional[datetime] = None + studio_notes: Optional[str] = None + children: List["MetricResponse"] = Field(default_factory=list) + created_at: datetime + updated_at: datetime + created_by: Optional[str] + + @field_validator('metric_type', mode='before') + @classmethod + def convert_metric_type(cls, v): + """Convert string to MetricType (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return MetricType(v_lower) + except ValueError: + for enum_member in MetricType: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid MetricType value: {v}") + return v + + @field_validator('trigger', mode='before') + @classmethod + def convert_trigger(cls, v): + """Convert string to MetricTrigger (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return MetricTrigger(v_lower) + except ValueError: + for enum_member in MetricTrigger: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid MetricTrigger value: {v}") + return v + + @validator('supported_surfaces', 'enabled_surfaces', pre=True) + def normalize_surfaces(cls, v): + if v is None: + return [] + if isinstance(v, str): + return [v] + if isinstance(v, (list, tuple)): + return [str(item).lower() for item in v if item] + return [] + + @validator('metric_origin', pre=True) + def normalize_metric_origin(cls, v): + if v is None: + return "custom" + return str(v).lower() + + model_config = ConfigDict(from_attributes=True) + + +MetricResponse.model_rebuild() + + +class MetricDraftCreate(MetricCreate): + """Create a draft metric for Metrics Studio experimentation.""" + + studio_notes: Optional[str] = Field( + default=None, + description="Optional notes about what this draft is testing.", + ) + + +class MetricDraftCreateWithChildren(MetricCreateWithChildren): + """Atomically create a draft parent category metric plus its children.""" + + studio_notes: Optional[str] = Field( + default=None, + description="Optional notes about what this draft category is testing.", + ) + + +class MetricPromoteResponse(BaseModel): + """Response after promoting a draft metric to active.""" + + metric: MetricResponse + promoted_at: datetime + + +MetricStudioSourceKind = Literal[ + "call_import_row", "call_recording", "evaluator_result" +] + + +class MetricStudioSourceItem(BaseModel): + """One call source selected for a Studio run.""" + + source_kind: MetricStudioSourceKind + source_ref: str = Field( + ..., + min_length=1, + description="UUID for import rows / evaluator results; call_short_id for recordings.", + ) + display_label: Optional[str] = Field( + default=None, + max_length=512, + description="Optional UI label; resolved server-side when omitted.", + ) + + +class MetricStudioRunCreate(BaseModel): + """Request body for triggering a Metrics Studio evaluation run.""" + + metric_ids: List[UUID] = Field(..., min_length=1) + sources: List[MetricStudioSourceItem] = Field(..., min_length=1) + name: Optional[str] = Field(default=None, max_length=255) + transcript_source: Literal["production", "diarised"] = "diarised" + llm_provider: Optional[str] = Field(default=None, max_length=50) + llm_model: Optional[str] = Field(default=None, max_length=100) + llm_credential_id: Optional[UUID] = None + llm_config: Optional[Dict[str, Any]] = None + metric_llm_overrides: Optional[Dict[str, Any]] = None + + +class MetricStudioRunRetryRequest(BaseModel): + """Retry failed or selected Studio run results.""" + + result_ids: Optional[List[UUID]] = Field( + default=None, + description="When omitted, retry all failed results in the run.", + ) + + +class MetricStudioRunResultResponse(BaseModel): + """Per-source result row for a Studio run.""" + + id: UUID + run_id: UUID + source_kind: str + source_ref: str + display_label: Optional[str] = None + source_metadata: Optional[Dict[str, Any]] = None + status: str + metric_scores: Dict[str, Any] = Field(default_factory=dict) + error_message: Optional[str] = None + started_at: Optional[datetime] = None + finished_at: Optional[datetime] = None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class MetricStudioRunResponse(BaseModel): + """Metrics Studio run summary.""" + + id: UUID + organization_id: UUID + workspace_id: UUID + name: Optional[str] = None + selected_metric_ids: List[str] = Field(default_factory=list) + selected_metric_groups: Optional[Dict[str, List[str]]] = None + transcript_source: str + llm_provider: Optional[str] = None + llm_model: Optional[str] = None + status: str + total_items: int + completed_items: int + failed_items: int + error_message: Optional[str] = None + started_at: Optional[datetime] = None + finished_at: Optional[datetime] = None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class MetricStudioRunListResponse(BaseModel): + items: List[MetricStudioRunResponse] + total: int + + +class MetricStudioRunResultListResponse(BaseModel): + items: List[MetricStudioRunResultResponse] + total: int + + +# Evaluator Result Schemas +class EvaluatorResultCreate(BaseModel): + """Schema for creating an evaluator result.""" + evaluator_id: UUID + agent_id: Optional[UUID] = None + persona_id: Optional[UUID] = None + scenario_id: Optional[UUID] = None + name: Optional[str] = None + duration_seconds: Optional[float] = None + audio_s3_key: Optional[str] = None + + +class EvaluatorResultCreateManual(BaseModel): + """Schema for manually creating an evaluator result from existing audio file.""" + evaluator_id: UUID + audio_s3_key: str + duration_seconds: Optional[float] = None + + +class EvaluatorResultUpdate(BaseModel): + """Schema for updating an evaluator result.""" + status: Optional[EvaluatorResultStatus] = None + transcription: Optional[str] = None + metric_scores: Optional[Dict[str, Any]] = None + error_message: Optional[str] = None + duration_seconds: Optional[float] = None + + + +class EvaluatorResultCounts(BaseModel): + """Rollup counts for evaluator result navigation.""" + + total: int = 0 + completed: int = 0 + failed: int = 0 + in_progress: int = 0 + last_run_at: Optional[datetime] = None + + + + +class EvaluatorResultsScenarioSummary(BaseModel): + scenario_id: UUID + scenario_name: str + counts: EvaluatorResultCounts + + + + +class EvaluatorResultsSuiteSummary(BaseModel): + suite_id: UUID + suite_name: Optional[str] = None + agent_id: UUID + persona_id: Optional[UUID] = None + counts: EvaluatorResultCounts + scenarios: Optional[List["EvaluatorResultsScenarioSummary"]] = None + + + + +class EvaluatorResultsAgentSummary(BaseModel): + agent_id: UUID + agent_name: str + counts: EvaluatorResultCounts + suites: Optional[List[EvaluatorResultsSuiteSummary]] = None + + + + +class EvaluatorResultsUnassignedSummary(BaseModel): + counts: EvaluatorResultCounts + recent_result_ids: List[str] = Field(default_factory=list) + + + + +class EvaluatorResultsOverviewResponse(BaseModel): + workspace_counts: EvaluatorResultCounts + agents: List[EvaluatorResultsAgentSummary] = Field(default_factory=list) + unassigned: EvaluatorResultsUnassignedSummary + + + + +class EvaluatorResultListResponse(BaseModel): + items: List["EvaluatorResultResponse"] + total: int + + + +class EvaluatorResultResponse(BaseModel): + """Schema for evaluator result response.""" + id: UUID + result_id: str + organization_id: UUID + evaluator_id: Optional[UUID] = None # Optional for playground test results + agent_id: Optional[UUID] = None # Nullable for custom evaluators + persona_id: Optional[UUID] = None # Optional for playground test results + scenario_id: Optional[UUID] = None # Optional for playground test results + name: Optional[str] = None # Optional for playground test results + timestamp: datetime + duration_seconds: Optional[float] + status: EvaluatorResultStatus + audio_s3_key: Optional[str] + transcription: Optional[str] + speaker_segments: Optional[List[Dict[str, Any]]] = None # [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] + metric_scores: Optional[Dict[str, Any]] + celery_task_id: Optional[str] + error_message: Optional[str] + + # Call tracking fields (for voice AI integrations) + call_event: Optional[str] = None + provider_call_id: Optional[str] = None + provider_platform: Optional[str] = None + call_data: Optional[Dict[str, Any]] = None # Full call details from provider + + created_at: datetime + updated_at: datetime + created_by: Optional[str] + + # Related entities (optional, populated when requested) + agent: Optional[AgentResponse] = None + persona: Optional[PersonaResponse] = None + scenario: Optional[ScenarioResponse] = None + evaluator: Optional[EvaluatorResponse] = None + + @field_validator('status', mode='before') + @classmethod + def convert_status(cls, v): + """Convert string to EvaluatorResultStatus (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return EvaluatorResultStatus(v_lower) + except ValueError: + for enum_member in EvaluatorResultStatus: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid EvaluatorResultStatus value: {v}") + return v + + model_config = ConfigDict(from_attributes=True) + + +# ============================================ +# ALERTING SCHEMAS +# ============================================ + +class AlertCreate(BaseModel): + """Schema for creating an alert.""" + name: str = Field(..., min_length=1, max_length=255) + description: Optional[str] = None + + # Metric condition + metric_type: AlertMetricType = AlertMetricType.NUMBER_OF_CALLS + aggregation: AlertAggregation = AlertAggregation.SUM + operator: AlertOperator = AlertOperator.GREATER_THAN + threshold_value: float = Field(..., description="Threshold value for the alert") + time_window_minutes: int = Field(default=60, ge=1, description="Time window in minutes for aggregation") + + # Agent selection (null means all agents) + agent_ids: Optional[List[UUID]] = None + + # Notification settings + notify_frequency: AlertNotifyFrequency = AlertNotifyFrequency.IMMEDIATE + notify_emails: Optional[List[str]] = Field(default=None, description="List of email addresses to notify") + notify_webhooks: Optional[List[str]] = Field(default=None, description="List of webhook URLs (Slack, etc.)") + + model_config = ConfigDict(json_schema_extra={ + "example": { + "name": "High Call Volume Alert", + "description": "Alert when call volume exceeds threshold", + "metric_type": "number_of_calls", + "aggregation": "sum", + "operator": ">", + "threshold_value": 100, + "time_window_minutes": 60, + "agent_ids": None, + "notify_frequency": "immediate", + "notify_emails": ["admin@example.com"], + "notify_webhooks": ["https://hooks.slack.com/services/xxx"] + } + }) + + +class AlertUpdate(BaseModel): + """Schema for updating an alert.""" + name: Optional[str] = Field(None, min_length=1, max_length=255) + description: Optional[str] = None + + # Metric condition + metric_type: Optional[AlertMetricType] = None + aggregation: Optional[AlertAggregation] = None + operator: Optional[AlertOperator] = None + threshold_value: Optional[float] = None + time_window_minutes: Optional[int] = Field(default=None, ge=1) + + # Agent selection + agent_ids: Optional[List[UUID]] = None + + # Notification settings + notify_frequency: Optional[AlertNotifyFrequency] = None + notify_emails: Optional[List[str]] = None + notify_webhooks: Optional[List[str]] = None + + # Status + status: Optional[AlertStatus] = None + + +class AlertResponse(BaseModel): + """Schema for alert response.""" + id: UUID + organization_id: UUID + name: str + description: Optional[str] + + # Metric condition + metric_type: AlertMetricType + aggregation: AlertAggregation + operator: AlertOperator + threshold_value: float + time_window_minutes: int + + # Agent selection + agent_ids: Optional[List[UUID]] + + # Notification settings + notify_frequency: AlertNotifyFrequency + notify_emails: Optional[List[str]] + notify_webhooks: Optional[List[str]] + + # Status + status: AlertStatus + + # Metadata + created_at: datetime + updated_at: datetime + created_by: Optional[str] + + @field_validator('metric_type', mode='before') + @classmethod + def convert_metric_type(cls, v): + """Convert string to AlertMetricType.""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return AlertMetricType(v_lower) + except ValueError: + for enum_member in AlertMetricType: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid AlertMetricType value: {v}") + return v + + @field_validator('aggregation', mode='before') + @classmethod + def convert_aggregation(cls, v): + """Convert string to AlertAggregation.""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return AlertAggregation(v_lower) + except ValueError: + for enum_member in AlertAggregation: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid AlertAggregation value: {v}") + return v + + @field_validator('operator', mode='before') + @classmethod + def convert_operator(cls, v): + """Convert string to AlertOperator.""" + if v is None: + return None + if isinstance(v, str): + try: + return AlertOperator(v) + except ValueError: + for enum_member in AlertOperator: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid AlertOperator value: {v}") + return v + + @field_validator('notify_frequency', mode='before') + @classmethod + def convert_notify_frequency(cls, v): + """Convert string to AlertNotifyFrequency.""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return AlertNotifyFrequency(v_lower) + except ValueError: + for enum_member in AlertNotifyFrequency: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid AlertNotifyFrequency value: {v}") + return v + + @field_validator('status', mode='before') + @classmethod + def convert_status(cls, v): + """Convert string to AlertStatus.""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return AlertStatus(v_lower) + except ValueError: + for enum_member in AlertStatus: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid AlertStatus value: {v}") + return v + + model_config = ConfigDict(from_attributes=True) + + +class AlertHistoryResponse(BaseModel): + """Schema for alert history response.""" + id: UUID + organization_id: UUID + alert_id: UUID + + # Trigger information + triggered_at: datetime + triggered_value: float + threshold_value: float + + # Status + status: AlertHistoryStatus + + # Notification tracking + notified_at: Optional[datetime] + notification_details: Optional[Dict[str, Any]] + + # Resolution + acknowledged_at: Optional[datetime] + acknowledged_by: Optional[str] + resolved_at: Optional[datetime] + resolved_by: Optional[str] + resolution_notes: Optional[str] + + # Additional context + context_data: Optional[Dict[str, Any]] + + # Metadata + created_at: datetime + updated_at: datetime + + # Related alert info (optional) + alert: Optional[AlertResponse] = None + + @field_validator('status', mode='before') + @classmethod + def convert_status(cls, v): + """Convert string to AlertHistoryStatus.""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return AlertHistoryStatus(v_lower) + except ValueError: + for enum_member in AlertHistoryStatus: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid AlertHistoryStatus value: {v}") + return v + + model_config = ConfigDict(from_attributes=True) + + +class AlertHistoryUpdate(BaseModel): + """Schema for updating alert history (acknowledge/resolve).""" + status: Optional[AlertHistoryStatus] = None + acknowledged_by: Optional[str] = None + resolved_by: Optional[str] = None + resolution_notes: Optional[str] = None + + +# ============================================ +# CRON JOB SCHEMAS +# ============================================ + +class CronJobCreate(BaseModel): + """Schema for creating a cron job.""" + name: str = Field(..., min_length=1, max_length=255) + cron_expression: str = Field(..., min_length=1, max_length=100, description="Cron expression (e.g., '0 9 * * 1-5')") + timezone: str = Field(default="UTC", max_length=100, description="Timezone for the cron schedule") + max_runs: int = Field(default=10, ge=1, le=1000, description="Maximum number of times to run") + evaluator_ids: Optional[List[UUID]] = Field( + None, + description="Evaluator IDs to trigger (expanded with evaluator_suite_ids when both are set).", + ) + evaluator_suite_ids: Optional[List[UUID]] = Field( + None, + description="Evaluator suite IDs whose combinations are expanded into evaluator_ids.", + ) + + model_config = ConfigDict(json_schema_extra={ + "example": { + "name": "Daily Evaluation Run", + "cron_expression": "0 9 * * 1-5", + "timezone": "America/New_York", + "max_runs": 100, + "evaluator_ids": ["123e4567-e89b-12d3-a456-426614174000"] + } + }) + + +class CronJobUpdate(BaseModel): + """Schema for updating a cron job.""" + name: Optional[str] = Field(None, min_length=1, max_length=255) + cron_expression: Optional[str] = Field(None, min_length=1, max_length=100) + timezone: Optional[str] = Field(None, max_length=100) + max_runs: Optional[int] = Field(None, ge=1, le=1000) + evaluator_ids: Optional[List[UUID]] = None + evaluator_suite_ids: Optional[List[UUID]] = None + status: Optional[CronJobStatus] = None + + +class CronJobResponse(BaseModel): + """Schema for cron job response.""" + id: UUID + organization_id: UUID + name: str + cron_expression: str + timezone: str + max_runs: int + current_runs: int + evaluator_ids: List[UUID] + status: CronJobStatus + next_run_at: Optional[datetime] + last_run_at: Optional[datetime] + created_at: datetime + updated_at: datetime + created_by: Optional[str] + + @field_validator('status', mode='before') + @classmethod + def convert_status(cls, v): + """Convert string to CronJobStatus.""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return CronJobStatus(v_lower) + except ValueError: + for enum_member in CronJobStatus: + if enum_member.value.lower() == v_lower: + return enum_member + raise ValueError(f"Invalid status: {v}") + return v + + @field_validator('evaluator_ids', mode='before') + @classmethod + def convert_evaluator_ids(cls, v): + """Convert evaluator_ids from JSON to list of UUIDs.""" + if v is None: + return [] + if isinstance(v, list): + return [UUID(str(id)) if not isinstance(id, UUID) else id for id in v] + return v + + model_config = ConfigDict(from_attributes=True) + + +# ============================================ +# PROMPT PARTIAL SCHEMAS +# ============================================ + +class MetricPartialChild(BaseModel): + """One categorization label inside a metric partial.""" + + name: str = Field(..., min_length=1) + description: str = "" + example: str = "" + + +class MetricPartialContent(BaseModel): + """Structured JSON payload stored in metric partial ``content``.""" + + schema_version: int = 1 + metric_kind: Literal["single", "category"] + description: str = "" + children: Optional[List[MetricPartialChild]] = None + + +class PromptPartialCreate(BaseModel): + """Schema for creating a prompt partial.""" + name: str = Field(..., min_length=1, max_length=255) + description: Optional[str] = None + content: str = Field(..., min_length=1) + tags: Optional[List[str]] = None + + +class PromptPartialUpdate(BaseModel): + """Schema for updating a prompt partial.""" + name: Optional[str] = Field(None, min_length=1, max_length=255) + description: Optional[str] = None + content: Optional[str] = Field(None, min_length=1) + tags: Optional[List[str]] = None + change_summary: Optional[str] = None + + +class PromptPartialVersionResponse(BaseModel): + """Schema for prompt partial version response.""" + id: UUID + prompt_partial_id: UUID + version: int + content: str + change_summary: Optional[str] + created_at: datetime + created_by: Optional[str] + + model_config = ConfigDict(from_attributes=True) + + +class AgentFlowNode(BaseModel): + """One step in an LLM-inferred agent logic flowchart.""" + + id: str + label: str + node_type: Literal["start", "decision", "action", "terminal"] = "action" + position_x: Optional[float] = None + position_y: Optional[float] = None + prompt_excerpt: Optional[str] = None + start_offset: Optional[int] = None + end_offset: Optional[int] = None + + +class AgentFlowEdge(BaseModel): + """Directed transition between two agent flow nodes.""" + + source: str + target: str + condition: Optional[str] = None + + +class AgentFlowNodeLayout(BaseModel): + id: str + position_x: float + position_y: float + + +class AgentFlowLayoutSaveRequest(BaseModel): + nodes: List[AgentFlowNodeLayout] = Field(default_factory=list) + + +class AgentFlowGraph(BaseModel): + """Aggregate flow diagram for an imported production agent prompt.""" + + nodes: List[AgentFlowNode] = Field(default_factory=list) + edges: List[AgentFlowEdge] = Field(default_factory=list) + generated_at: Optional[datetime] = None + provider: Optional[str] = None + model: Optional[str] = None + layout_saved_at: Optional[datetime] = None + prompt_content_hash: Optional[str] = None + mapping_error: Optional[str] = None + generation_error: Optional[str] = None + + +class PromptPartialResponse(BaseModel): + """Schema for prompt partial response.""" + id: UUID + organization_id: UUID + name: str + description: Optional[str] + content: str + tags: Optional[List[str]] + current_version: int + agent_flowchart: Optional[AgentFlowGraph] = None + agent_flowchart_status: Optional[str] = None + created_at: datetime + updated_at: datetime + created_by: Optional[str] + + model_config = ConfigDict(from_attributes=True) + + +class PromptPartialDetailResponse(PromptPartialResponse): + """Schema for prompt partial detail with versions.""" + versions: List[PromptPartialVersionResponse] = [] + + model_config = ConfigDict(from_attributes=True) + + +# ============================================ +# TELEPHONY SCHEMAS (provider-agnostic) +# ============================================ + + +class TelephonyIntegrationCreate(BaseModel): + """Schema for creating a telephony provider integration.""" + + provider: str = "plivo" + name: Optional[str] = None + auth_id: str + auth_token: str + verify_app_uuid: Optional[str] = None + voice_app_id: Optional[str] = None + sip_domain: Optional[str] = None + masking_config: Optional[Dict[str, Any]] = None + is_default: Optional[bool] = Field( + None, + description=( + "Mark this credential as the default for the (org, provider). " + "If omitted and no default exists yet, this row becomes the default." + ), + ) + + +class TelephonyIntegrationUpdate(BaseModel): + """Schema for partial updates to a telephony provider integration.""" + + id: Optional[UUID] = None + provider: Optional[str] = None + name: Optional[str] = None + auth_id: Optional[str] = None + auth_token: Optional[str] = None + verify_app_uuid: Optional[str] = None + voice_app_id: Optional[str] = None + sip_domain: Optional[str] = None + masking_config: Optional[Dict[str, Any]] = None + is_active: Optional[bool] = None + + +class TelephonyIntegrationResponse(BaseModel): + """Safe response model for telephony integration without secrets.""" + + id: UUID + organization_id: UUID + provider: str + name: Optional[str] = None + verify_app_uuid: Optional[str] + voice_app_id: Optional[str] + sip_domain: Optional[str] + masking_config: Optional[Dict[str, Any]] + is_active: bool + is_default: bool = False + last_tested_at: Optional[datetime] + created_at: datetime + updated_at: datetime + + class Config: + from_attributes = True + + +class TelephonyPhoneNumberResponse(BaseModel): + """Telephony phone number inventory response schema.""" + + id: UUID + phone_number: str + country_iso2: Optional[str] + region: Optional[str] + number_type: Optional[str] + capabilities: Optional[Dict[str, Any]] + is_masking_pool: bool + inbound_enabled: Optional[bool] = None + outbound_enabled: Optional[bool] = None + source: Optional[str] = None + agent_id: Optional[UUID] + linked_agent_name: Optional[str] = None + provider: Optional[str] = None + is_active: bool + created_at: datetime + + class Config: + from_attributes = True + + +class TelephonyDialTargetCreate(BaseModel): + """Schema for creating a saved outbound dial target.""" + phone_number: str + label: Optional[str] = None + + +class TelephonyDialTargetUpdate(BaseModel): + """Schema for updating a saved outbound dial target.""" + phone_number: Optional[str] = None + label: Optional[str] = None + + +class TelephonyDialTargetResponse(BaseModel): + """Schema for dial target response.""" + id: UUID + phone_number: str + label: Optional[str] = None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class TelephonyVerifyStartRequest(BaseModel): + """Request schema for starting voice OTP verification.""" + + phone_number: str + provider: str = "plivo" + + +class TelephonyVerifyStartResponse(BaseModel): + """Response schema for started voice OTP verification.""" + + session_id: UUID + provider_session_uuid: str + status: str + message: str + + +class TelephonyVerifyCheckRequest(BaseModel): + """Request schema for checking a submitted OTP code.""" + + session_id: UUID + otp_code: str + provider: str = "plivo" + + +class TelephonyVerifyCheckResponse(BaseModel): + """Response schema for OTP check status.""" + + verified: bool + status: str + message: str + + +class TelephonyMaskingSessionCreate(BaseModel): + """Request schema for creating a number masking session.""" + + party_a_number: str + party_b_number: str + provider: str = "plivo" + expires_in_minutes: Optional[int] = 60 + metadata: Optional[Dict[str, Any]] = None + provider: str = "plivo" + + +class TelephonyMaskingSessionResponse(BaseModel): + """Response schema for masking sessions.""" + + id: UUID + masked_number: str + party_a_number: str + party_b_number: str + status: str + expires_at: Optional[datetime] + created_at: datetime + + class Config: + from_attributes = True + + +class TelephonyOutboundCallRequest(BaseModel): + """Request schema for outbound call initiation.""" + + from_number: str + to_number: str + answer_url: Optional[str] = None + agent_id: Optional[UUID] = None + + +class TelephonyOutboundCallResponse(BaseModel): + """Response schema for outbound call initiation.""" + + provider_request_uuid: str + call_status: str + from_number: str + to_number: str + message: str + + +# --- Call Import Schemas --- + +class CallImportRowResponse(BaseModel): + """Single row within a call-import batch.""" + + id: UUID + row_index: int + # Renamed from ``external_call_id`` (DB column renamed in migration + # ``034_call_import_schemas``). Same data, same uniqueness rules. + conversation_id: str + recording_url: Optional[str] = None + recording_date: Optional[date] = None + # Production transcript: the value supplied via the CSV upload. + transcript: Optional[str] = None + transcript_source: Optional[str] = None + transcript_provider: Optional[str] = None + transcript_model: Optional[str] = None + transcript_status: Optional[str] = None + transcript_error: Optional[str] = None + transcribed_at: Optional[datetime] = None + # Diarised transcript: produced by the post-hoc diarisation + # worker. Independent of ``transcript`` so manual diarisation + # never overwrites the CSV-supplied production value. + diarised_transcript: Optional[str] = None + diarised_transcript_provider: Optional[str] = None + diarised_transcript_model: Optional[str] = None + diarised_transcript_status: Optional[str] = None + diarised_transcript_error: Optional[str] = None + diarised_at: Optional[datetime] = None + # LLM that turned the STT plain-text output into structured + # ``diarised_segments``. Surfaced in the row detail panel so + # reviewers can see "Diarised by openai/gpt-4o-mini" next to + # the swap toggle. NULL on rows diarised by the legacy pyannote + # worker (which has been removed). + diarised_llm_provider: Optional[str] = None + diarised_llm_model: Optional[str] = None + # The exact prompt the LLM diariser ran with. Persisted so a + # reviewer can copy it back into the modal and reproduce the + # turn layout against a different STT pass. + diarised_prompt: Optional[str] = None + # Structured speaker turns produced by the diarisation worker. Each + # entry is `{ "speaker": "agent"|"user"|"speaker_N", "text": str, + # "start": float, "end": float, "raw_speaker": "Speaker 1" }`. The + # plain ``diarised_transcript`` field above is a `: ` + # rendering of this list with ``diarised_speaker_swap`` applied. + diarised_segments: Optional[List[Dict[str, Any]]] = None + # When True the agent <-> user mapping in ``diarised_segments`` is + # inverted at render / export time. The worker writes the canonical + # mapping using the "first speaker is the agent" heuristic; the swap + # toggle lets reviewers correct that without re-running diarisation. + diarised_speaker_swap: bool = False + status: CallImportRowStatus + recording_s3_key: Optional[str] = None + recording_content_type: Optional[str] = None + recording_size_bytes: Optional[int] = None + error_message: Optional[str] = None + attempts: int + raw_columns: Optional[Dict[str, Any]] = None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +# --- Call Import Schema (Input Parameter definitions) --- + + +class CallImportSchemaParameterBase(BaseModel): + """A single typed parameter inside a Call Import schema. + + Used both in request bodies (create / update) and as the building + block of :class:`CallImportSchemaParameterResponse`. Names are + case-insensitive unique within their parent schema. + """ + + name: str = Field( + ..., + min_length=1, + max_length=255, + description=( + "Parameter name as it appears in the schema editor and the " + "upload mapping table. Must be unique within the schema " + "(case-insensitive)." + ), + ) + type: CallImportParameterType = Field( + ..., + description=( + "Parameter type. One of conversation_id / recording_url / " + "recording_date / transcript / text / number / boolean / " + "datetime / url. Exactly one parameter of type " + "'conversation_id' must be present; at most one each of " + "'recording_url', 'recording_date', and 'transcript'. " + "Only conversation_id is forced required." + ), + ) + description: Optional[str] = Field( + default=None, + max_length=2048, + description="Free-text help shown next to the parameter in the mapping UI.", + ) + is_required: bool = Field( + default=False, + description=( + "When True, the parameter must be mapped to a CSV column on " + "every upload. The ``conversation_id`` parameter is always " + "required and is force-set to True by the server." + ), + ) + + +class CallImportSchemaParameterCreate(CallImportSchemaParameterBase): + """Create payload for a single parameter (inside a schema CRUD body).""" + + +class CallImportSchemaParameterResponse(CallImportSchemaParameterBase): + """Response shape including the persisted id + ordering.""" + + id: UUID + ordering: int + + model_config = ConfigDict(from_attributes=True) + + +def _validate_schema_parameters( + parameters: List[CallImportSchemaParameterBase], +) -> List[CallImportSchemaParameterBase]: + """Apply the cross-parameter invariants shared by create + update.""" + + if not parameters: + raise ValueError("Schema must define at least one parameter.") + + seen_names: set[str] = set() + conv_count = 0 + recording_date_count = 0 + rec_url_count = 0 + transcript_count = 0 + for param in parameters: + norm = param.name.strip().lower() + if not norm: + raise ValueError("Parameter name must be non-empty.") + if norm in seen_names: + raise ValueError( + f"Duplicate parameter name '{param.name}' " + "(names must be unique within a schema)." + ) + seen_names.add(norm) + if param.type == CallImportParameterType.CONVERSATION_ID: + conv_count += 1 + elif param.type == CallImportParameterType.RECORDING_DATE: + recording_date_count += 1 + elif param.type == CallImportParameterType.RECORDING_URL: + rec_url_count += 1 + elif param.type == CallImportParameterType.TRANSCRIPT: + transcript_count += 1 + + if conv_count != 1: + raise ValueError( + "Schema must contain exactly one parameter of type " + "'conversation_id'." + ) + if rec_url_count > 1: + raise ValueError( + "Schema may contain at most one parameter of type " + "'recording_url'." + ) + if recording_date_count > 1: + raise ValueError( + "Schema may contain at most one parameter of type " + "'recording_date'." + ) + if transcript_count > 1: + raise ValueError( + "Schema may contain at most one parameter of type 'transcript'." + ) + return parameters + + +class CallImportSchemaCreate(BaseModel): + """Create body for a new call-import schema.""" + + name: str = Field(..., min_length=1, max_length=255) + description: Optional[str] = Field(default=None, max_length=2048) + parameters: List[CallImportSchemaParameterCreate] = Field( + ..., + description=( + "Ordered list of parameters. Order is preserved; the server " + "stamps ``ordering`` from the list index." + ), + ) + + @model_validator(mode="after") + def _check_parameters(self): + _validate_schema_parameters(list(self.parameters)) + return self + + +class CallImportSchemaUpdate(BaseModel): + """Patch body for an existing schema (full parameter replacement).""" + + name: Optional[str] = Field(default=None, min_length=1, max_length=255) + description: Optional[str] = Field(default=None, max_length=2048) + parameters: Optional[List[CallImportSchemaParameterCreate]] = Field( + default=None, + description=( + "If provided, REPLACES the full set of parameters on the " + "schema. Omit to leave parameters untouched." + ), + ) + + @model_validator(mode="after") + def _check_parameters(self): + if self.parameters is not None: + _validate_schema_parameters(list(self.parameters)) + return self + + +class CallImportSchemaResponse(BaseModel): + """Read response for a single schema.""" + + id: UUID + organization_id: UUID + workspace_id: UUID + name: str + description: Optional[str] = None + parameters: List[CallImportSchemaParameterResponse] = Field(default_factory=list) + # How many CallImport batches reference this schema. Populated by the + # router when listing; defaults to 0 on detail responses where the + # caller doesn't need it. + usage_count: int = 0 + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class CallImportSchemaListResponse(BaseModel): + """Paginated list of schemas.""" + + items: List[CallImportSchemaResponse] = Field(default_factory=list) + total: int + + +class CallImportTagResponse(BaseModel): + """Tag attached to call import batches.""" + + id: UUID + name: str + color: Optional[str] = None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class CallImportTagCreate(BaseModel): + """Create a new call-import tag for the organization.""" + + name: str = Field(..., min_length=1, max_length=255) + color: Optional[str] = Field(None, max_length=32) + + +class CallImportTagUpdate(BaseModel): + """Partial update for a call-import tag.""" + + name: Optional[str] = Field(None, min_length=1, max_length=255) + color: Optional[str] = Field(None, max_length=32) + + +class CallImportPreviewSheet(BaseModel): + """One worksheet (or one CSV file synthesized as a single sheet).""" + + name: str = Field(..., description="Sheet name for xlsx; filename for csv.") + headers: List[str] = Field( + default_factory=list, + description="Column headers from the first non-empty row.", + ) + row_count: int = Field( + ..., + description="Approximate count of data rows (excluding the header row).", + ) + + +class CallImportSourceRowSkip(BaseModel): + """One source spreadsheet row skipped during parse (identity / recording URL).""" + + source_row: int = Field( + ..., + description="1-based row index in the source file (same semantics as parse errors).", + ) + reason: str = Field( + ..., + description=( + "Machine-readable skip reason, e.g. missing_conversation_id, " + "missing_recording_url, invalid_recording_url." + ), + ) + message: str = Field( + ..., + description="Human-readable explanation shown in the UI.", + ) + + +class CallImportResponse(BaseModel): + """Summary of a call-import batch.""" + + id: UUID + organization_id: UUID + workspace_id: UUID + # Provider is optional in the new staged flow (only resolved at the + # IMPORT stage). Stays populated for all post-import batches. + provider: Optional[str] = None + telephony_integration_id: Optional[UUID] = None + original_filename: Optional[str] = None + sheet_name: Optional[str] = None + dataset: Optional[str] = None + tags: List[CallImportTagResponse] = Field(default_factory=list) + # New schema-driven mapping. Empty on legacy batches; pre-schema + # batches keep their values in ``column_mapping`` / ``extra_columns`` + # / ``custom_column_mapping`` below for backwards-compatibility. + schema_id: Optional[UUID] = None + parameter_mapping: Dict[str, str] = Field(default_factory=dict) + column_mapping: Dict[str, Optional[str]] = Field(default_factory=dict) + extra_columns: List[str] = Field(default_factory=list) + custom_column_mapping: Dict[str, str] = Field(default_factory=dict) + # Persisted "drop these columns" decision captured at MAP time. + # Empty for legacy one-shot uploads where the value was ephemeral. + skipped_columns: List[str] = Field(default_factory=list) + source_row_skips: List[CallImportSourceRowSkip] = Field( + default_factory=list, + description=( + "Source rows skipped at parse time because of missing/invalid " + "conversation ID or recording URL." + ), + ) + # Source-file staging fields populated at UPLOAD time. ``None`` on + # legacy batches imported via the one-shot ``POST /upload`` endpoint. + source_s3_key: Optional[str] = None + source_format: Optional[str] = None + source_size_bytes: Optional[int] = None + source_content_type: Optional[str] = None + # Snapshot of the file's sheets + headers captured at UPLOAD time + # so the MAP UI can render without re-fetching the file from S3. + available_sheets: Optional[List[CallImportPreviewSheet]] = None + total_rows: int + completed_rows: int + failed_rows: int + status: CallImportStatus + error_message: Optional[str] = None + created_at: datetime + updated_at: datetime + created_by_email: Optional[str] = None + last_updated_by_email: Optional[str] = None + + model_config = ConfigDict(from_attributes=True) + + +class CallImportDetailResponse(CallImportResponse): + """A call-import batch with its rows expanded. + + ``filtered_total_rows`` is only set when the caller passed a ``q`` + search term G�� it lets the UI paginate against the filtered subset + while still showing the unfiltered ``total_rows`` in the header. + + The ``diarised_*_rows`` counters aggregate + ``CallImportRow.diarised_transcript_status`` across the batch so the + UI can render a transcribe-and-diarise progress bar without paging + through every row. Rows that have never been touched by the + transcribe/diarise worker (``status='idle'``) are NOT counted here G�� + callers compute the idle bucket as + ``total_rows - (pending + running + completed + failed)``. + """ + + rows: List[CallImportRowResponse] = Field(default_factory=list) + filtered_total_rows: Optional[int] = None + diarised_pending_rows: int = 0 + diarised_running_rows: int = 0 + diarised_completed_rows: int = 0 + diarised_failed_rows: int = 0 + + +class CallImportListResponse(BaseModel): + """Paginated list of call-import batches.""" + + items: List[CallImportResponse] + total: int + page: int + page_size: int + + +class CallImportDispatchLimitSnapshot(BaseModel): + """Configured and live Redis in-flight caps for eval work.""" + + global_limit: int + global_inflight: int + global_at_capacity: bool + org_limit: int + org_inflight: int + org_at_capacity: bool + workspace_limit: int + job_limit: int + fair_dispatch_batch_size: int + + +class CallImportDispatchFairDispatchSnapshot(BaseModel): + """Fair-dispatch scheduler metadata from Redis.""" + + global_rr_cursor: int + dispatch_dedupe_active: bool + dispatch_queue: str + at_capacity_backoff_seconds: int + + +class CallImportDispatchEvaluationSnapshot(BaseModel): + """One in-flight evaluation run with row counters.""" + + evaluation_id: UUID + call_import_id: UUID + status: str + total_rows: int + pending_rows: int + running_rows: int + job_inflight: int + job_at_capacity: bool + + +class CallImportDispatchWorkspaceSnapshot(BaseModel): + """Per-workspace pending dispatch + slot usage.""" + + workspace_id: UUID + workspace_name: Optional[str] = None + workspace_slug: Optional[str] = None + inflight: int + inflight_at_capacity: bool + pending_dispatch_rows: int + pending_import_rows: int + eval_rr_cursor: int + active_evaluations: int + evaluations: List[CallImportDispatchEvaluationSnapshot] = Field( + default_factory=list + ) + + +class CallImportDispatchDiagnosticsResponse(BaseModel): + """Live operator snapshot for call-import eval fair dispatch.""" + + limits: CallImportDispatchLimitSnapshot + fair_dispatch: CallImportDispatchFairDispatchSnapshot + workspaces: List[CallImportDispatchWorkspaceSnapshot] + generated_at: datetime + + +class CallImportUploadResponse(BaseModel): + """Response returned right after a CSV is accepted.""" + + id: UUID + total_rows: int + status: CallImportStatus + dataset: Optional[str] = None + tags: List[CallImportTagResponse] = Field(default_factory=list) + message: str + + +class CallImportDeleteResponse(BaseModel): + """Response after a whole-batch call-import delete is accepted.""" + + id: UUID + status: Literal["accepted", "completed"] = Field( + ..., + description=( + "``accepted`` when teardown was queued to run asynchronously; " + "``completed`` when the batch was already removed." + ), + ) + + +class CallImportPreviewResponse(BaseModel): + """Sheets/headers extracted from an uploaded CSV or Excel workbook. + + The frontend uses this to drive the column-mapping UI without doing + its own parsing G�� keeps client and server in lockstep on quoted + fields, encodings, and Excel cell coercion. + """ + + format: str = Field(..., description="One of 'csv' or 'xlsx'.") + sheets: List[CallImportPreviewSheet] = Field(default_factory=list) + + +class CallImportUpdate(BaseModel): + """Partial update of a call-import batch.""" + + original_filename: Optional[str] = Field( + None, + description=( + "User-facing batch label shown in the UI. Pass an empty string to clear." + ), + ) + dataset: Optional[str] = Field( + None, + description=( + "Free-text dataset label. Pass an empty string to clear the dataset." + ), + ) + tag_ids: Optional[List[UUID]] = Field( + None, + description=( + "Replace the full set of tag assignments. Pass an empty list to clear all tags." + ), + ) + schema_id: Optional[UUID] = Field( + None, + description=( + "Reassign the Input Parameter schema. Only honoured while the " + "batch is in ``uploaded`` or ``mapped`` state; once the batch " + "has rows it's locked to its original schema." + ), + ) + + +class CallImportMappingUpdate(BaseModel): + """Mapping payload for the MAP stage (``PATCH /call-imports/{id}/mapping``). + + Idempotent: callers can submit this multiple times against an + ``uploaded`` or ``mapped`` batch. Validation re-runs against the + persisted ``available_sheets`` snapshot every time so the user can + correct mistakes without re-uploading the file. + """ + + schema_id: UUID = Field( + ..., + description=( + "Reusable Input Parameter schema this batch is mapped against. " + "Must belong to the active workspace." + ), + ) + sheet_name: Optional[str] = Field( + None, + description=( + "Worksheet to use when the staged source file is an Excel " + "workbook. REQUIRED for xlsx; ignored / rejected for CSV." + ), + ) + parameter_mapping: Dict[str, str] = Field( + default_factory=dict, + description=( + "``{schema_parameter_name: source_header}`` map covering every " + "required schema parameter." + ), + ) + skipped_columns: List[str] = Field( + default_factory=list, + description=( + "Source headers the uploader has explicitly skipped. Every " + "source header must be either mapped or appear here." + ), + ) + + +class CallImportStartRequest(BaseModel): + """Provider + credential picker for the IMPORT stage.""" + + provider: Optional[str] = Field( + default=None, + description=( + "Telephony provider key. Must match the " + "``telephony_integration_id``'s provider. Omit together with " + "``telephony_integration_id`` to download recordings directly " + "from CSV-supplied URLs without credentials." + ), + ) + telephony_integration_id: Optional[UUID] = Field( + default=None, + description=( + "Specific TelephonyIntegration credential row to use when " + "downloading recordings for this batch. Omit together with " + "``provider`` for direct-URL import." + ), + ) + + @model_validator(mode="after") + def validate_credential_mode(self) -> "CallImportStartRequest": + has_provider = bool((self.provider or "").strip()) + has_integration = self.telephony_integration_id is not None + if has_provider != has_integration: + raise ValueError( + "provider and telephony_integration_id must both be provided " + "or both omitted for direct-URL import." + ) + return self + + +# --- Call Import Evaluation Schemas --- + + +class CallImportEvaluationLLMOverride(BaseModel): + """Per-metric LLM override used on top of the run-level default. + + Any field left ``None`` falls back to the run-level value (which + itself falls back to the historical OpenAI/gpt-4o default). This + lets users pick a specific provider/model for a single metric (e.g. + a stronger Anthropic model for a tricky qualitative metric) without + re-typing the rest of the metrics in the run. + """ + + provider: Optional[str] = Field( + default=None, + max_length=50, + description="Override LLM provider key, e.g. 'openai' or 'anthropic'.", + ) + model: Optional[str] = Field( + default=None, + max_length=100, + description="Override LLM model name, e.g. 'gpt-4o' or 'claude-3-opus'.", + ) + credential_id: Optional[UUID] = Field( + default=None, + description="Optional AIProvider id when the org has multiple credentials.", + ) + llm_config: Optional[Dict[str, Any]] = Field( + default=None, + description="Optional per-metric generation parameters (temperature, top_p, etc.).", + ) + + +CallImportEvaluationTranscriptSource = Literal["production", "diarised"] + + +class CallImportEvaluationCreate(BaseModel): + """Request body for triggering an evaluation over a call-import batch.""" + + metric_ids: List[UUID] = Field( + ..., + min_length=1, + description="Org Metric ids to score every completed row against.", + ) + name: Optional[str] = Field( + default=None, + max_length=255, + description=( + "Optional human-readable label for the run. Shown in the UI " + "instead of the UUID prefix." + ), + ) + transcript_sources: List[CallImportEvaluationTranscriptSource] = Field( + default_factory=lambda: ["diarised"], + min_length=1, + max_length=1, + description=( + "Which transcript to score against. ``'diarised'`` (default) " + "auto-diarises rows missing a diarised transcript then scores " + "``diarised_transcript``. ``'production'`` scores the CSV " + "``transcript`` column directly and skips diarisation." + ), + ) + + @field_validator("transcript_sources") + @classmethod + def _validate_transcript_sources( + cls, value: List[str] + ) -> List["CallImportEvaluationTranscriptSource"]: + allowed = {"production", "diarised"} + invalid = [src for src in value if src not in allowed] + if invalid: + raise ValueError( + "transcript_sources must be ['production'] or ['diarised'] " + "(received: " + + ", ".join(repr(src) for src in invalid) + + ")." + ) + return value # type: ignore[return-value] + # --- Run-level LLM config --- + llm_provider: Optional[str] = Field( + default=None, + max_length=50, + description=( + "Run-level LLM provider key (e.g. 'openai', 'anthropic'). NULL " + "preserves the historical OpenAI/gpt-4o default." + ), + ) + llm_model: Optional[str] = Field( + default=None, + max_length=100, + description="Run-level LLM model name. Required when llm_provider is set.", + ) + llm_credential_id: Optional[UUID] = Field( + default=None, + description="Optional AIProvider row to pin for the run-level LLM.", + ) + llm_config: Optional[Dict[str, Any]] = Field( + default=None, + description="Run-level LLM generation parameters (temperature, top_p, etc.).", + ) + metric_llm_overrides: Optional[ + Dict[str, CallImportEvaluationLLMOverride] + ] = Field( + default=None, + description=( + "Optional per-metric LLM overrides keyed by metric UUID. Each " + "entry overrides the run-level default for that metric only." + ), + ) + # --- Auto-transcribe / diarization hook --- + # Every diarised run auto-diarises rows that don't already have a + # diarised transcript. The flag stays on the schema so legacy API + # callers don't 400 immediately, but the route now requires + # ``stt_provider`` + ``stt_model`` on every run regardless of this + # value. + auto_transcribe: bool = Field( + default=True, + description=( + "Auto-diarise rows missing a diarised transcript before " + "evaluation. Defaults to true and is effectively required: " + "``stt_provider`` + ``stt_model`` are mandatory on every " + "evaluation run." + ), + ) + transcribe_overwrite: bool = Field( + default=False, + description=( + "When auto_transcribe is on, overwrite existing transcripts " + "instead of skipping rows that already have one." + ), + ) + transcribe_mode: Literal["stt_llm", "llm_only"] = Field( + default="stt_llm", + description=( + "Diarisation pipeline shape for the auto-transcribe step. " + "'stt_llm' (default) runs STT then an LLM diariser over the " + "resulting text G�� ``stt_provider`` + ``stt_model`` must be " + "provided. 'llm_only' skips STT and feeds the audio " + "directly to the multimodal ``diarization_llm_*`` model " + "along with ``diarization_prompt``; STT fields must be " + "omitted in that case." + ), + ) + stt_provider: Optional[str] = Field( + default=None, + max_length=50, + description=( + "STT provider key, e.g. 'deepgram', 'openai'. Required when " + "``transcribe_mode='stt_llm'`` (the default); must be omitted " + "when ``transcribe_mode='llm_only'``." + ), + ) + stt_model: Optional[str] = Field( + default=None, + max_length=100, + description=( + "STT model name, e.g. 'nova-2', 'whisper-1'. Same presence " + "rules as ``stt_provider``." + ), + ) + stt_credential_id: Optional[UUID] = Field( + default=None, + description="Optional AIProvider/Integration row to pin for STT.", + ) + stt_language: Optional[str] = Field( + default=None, + max_length=20, + description="ISO language hint for the STT provider, e.g. 'en'.", + ) + # --- LLM diariser config (mirror of CallImportTranscribeRequest) --- + # Auto-diarised eval rows go through the same LLM-based diariser as + # the standalone Transcribe modal G�� the run remembers the provider / + # model / prompt so a follow-up retry can reproduce them without + # having to re-prompt the user. + diarization_llm_provider: Optional[str] = Field( + default=None, + max_length=50, + description=( + "LLM provider for diarising STT output into agent/user " + "turns. Required when ``auto_transcribe`` is set (the worker " + "no longer falls back to pyannote)." + ), + ) + diarization_llm_model: Optional[str] = Field( + default=None, + max_length=100, + description="LLM model for the diariser.", + ) + diarization_llm_credential_id: Optional[UUID] = Field( + default=None, + description="Optional AIProvider row to pin for the diariser LLM.", + ) + diarization_prompt: Optional[str] = Field( + default=None, + max_length=10_000, + description=( + "Custom system prompt for the diariser LLM; falls back to " + "the canonical default when blank." + ), + ) + discover_new_metrics: bool = Field( + default=False, + description=( + "When true, the LLM is invited to propose net-new top-level " + "metrics (boolean / rating / category) observed in the " + "transcripts in addition to scoring the selected metrics. " + "Candidates surface in the Discovered metrics panel on the " + "evaluation detail Flow tab and can be promoted into real " + "standalone Metric rows. Defaults to false so existing " + "callers retain previous behaviour." + ), + ) + # Telephony credentials for unified pipeline (required when batch is mapped). + provider: Optional[str] = Field( + default=None, + description=( + "Telephony provider key. Required together with " + "``telephony_integration_id`` when starting evaluation " + "from a mapped batch. Omit both for direct-URL import." + ), + ) + telephony_integration_id: Optional[UUID] = Field( + default=None, + description=( + "TelephonyIntegration credential for recording fetch. " + "Required together with ``provider`` for credentialed import." + ), + ) + + @model_validator(mode="after") + def validate_telephony_credential_mode(self) -> "CallImportEvaluationCreate": + has_provider = bool((self.provider or "").strip()) + has_integration = self.telephony_integration_id is not None + if has_provider != has_integration: + raise ValueError( + "provider and telephony_integration_id must both be provided " + "or both omitted for direct-URL evaluation." + ) + return self + + +class CallImportEvaluationUpdate(BaseModel): + """Patch body for editing a previously-created evaluation run.""" + + name: Optional[str] = Field( + default=None, + max_length=255, + description="New name for the evaluation. Empty string clears it.", + ) + + +class CallImportEvaluationBulkDelete(BaseModel): + """Request body for deleting multiple evaluation runs in one call.""" + + evaluation_ids: List[UUID] = Field( + ..., + min_length=1, + description="Evaluation ids to delete.", + ) + + +class CallImportEvaluationRetryRequest(BaseModel): + """Body for retrying a subset (or all failed rows) of an evaluation run. + + ``eval_row_ids`` is optional: when ``None`` the retry applies to + every row in the run that is currently in the ``failed`` state. The + selection always intersects with the run's actual rows, so unknown + ids are silently skipped (and surfaced in the response's + ``skipped`` list with reason ``unknown``). + + The optional ``llm_*`` / ``metric_llm_overrides`` / ``stt_*`` fields + let the caller swap out the LLM or STT configuration that the + failed rows were originally evaluated with. When a field is left + ``None`` the run's existing value is preserved. When a field is + set, it is persisted onto the run (so a follow-up retry sees the + new value as the default) and used by the worker on the next + pass. Providing only one half of provider+model is rejected so + the worker never ends up with a half-configured run. + """ + + eval_row_ids: Optional[List[UUID]] = Field( + default=None, + description=( + "Restrict the retry to a specific subset of evaluation rows. " + "When omitted, every row with status='failed' in this run is " + "re-enqueued." + ), + ) + + # --- Metric-subset re-run --- + # When ``metric_ids`` is set, the retry recomputes ONLY those + # metrics instead of the whole row, and the new scores are merged + # into the existing ``metric_scores`` JSON (other metrics' + # previously-computed values are preserved). This is the path + # taken by the "Re-run metrics" UI in CallImportEvaluationDetail. + metric_ids: Optional[List[UUID]] = Field( + default=None, + description=( + "Restrict the retry to a specific subset of metrics. When " + "set, the worker recomputes only these metrics and merges " + "the new scores into the row's existing metric_scores " + "(other metrics' previous values are preserved). When " + "omitted, the row is fully re-scored as before. Every id " + "must already be present in the run's selected_metric_ids." + ), + ) + include_completed: bool = Field( + default=False, + description=( + "When True, rows whose status is currently 'completed' " + "become eligible for retry (otherwise only 'failed' rows " + "are picked up). Required when ``metric_ids`` is set on a " + "successful row, since otherwise the whole metric-subset " + "retry would be skipped as 'completed'." + ), + ) + + # --- LLM overrides --- + llm_provider: Optional[str] = Field( + default=None, + max_length=50, + description=( + "Override the run-level LLM provider for this retry (and " + "future retries). Must be paired with ``llm_model``." + ), + ) + llm_model: Optional[str] = Field( + default=None, + max_length=100, + description=( + "Override the run-level LLM model. Must be paired with " + "``llm_provider``." + ), + ) + llm_credential_id: Optional[UUID] = Field( + default=None, + description=( + "Pin a specific AIProvider credential row for the LLM. " + "When omitted, the resolver falls back to the org default." + ), + ) + llm_config: Optional[Dict[str, Any]] = Field( + default=None, + description="Override run-level LLM generation parameters for this retry.", + ) + metric_llm_overrides: Optional[ + Dict[str, CallImportEvaluationLLMOverride] + ] = Field( + default=None, + description=( + "Replace the run's per-metric LLM overrides. When omitted, " + "the existing overrides are kept; when set, this dict " + "fully replaces them (pass an empty object to clear)." + ), + ) + + # --- STT overrides --- + stt_provider: Optional[str] = Field( + default=None, + max_length=50, + description=( + "Override the run-level STT provider for this retry. Must " + "be paired with ``stt_model``. Only meaningful when the " + "run is configured for the diarised transcript source." + ), + ) + stt_model: Optional[str] = Field( + default=None, + max_length=100, + description="Override the run-level STT model.", + ) + stt_credential_id: Optional[UUID] = Field( + default=None, + description="Pin a specific credential row for the STT call.", + ) + # --- LLM diariser overrides --- + # When set, replace the run-stored diariser configuration for any + # rows that have to be re-diarised as part of the retry (i.e. + # ``transcribe_overwrite=True`` or the row never had a diarised + # transcript). Same provider+model pairing rule as STT. + diarization_llm_provider: Optional[str] = Field( + default=None, + max_length=50, + description=( + "Override the run's diariser LLM provider. Must be paired " + "with ``diarization_llm_model``." + ), + ) + diarization_llm_model: Optional[str] = Field( + default=None, + max_length=100, + description="Override the run's diariser LLM model.", + ) + diarization_llm_credential_id: Optional[UUID] = Field( + default=None, + description="Pin a specific credential row for the diariser LLM.", + ) + diarization_prompt: Optional[str] = Field( + default=None, + max_length=10_000, + description=( + "Override the run's diariser prompt. Pass an empty string " + "to clear the override and fall back to the canonical " + "default; pass None to leave the existing value untouched." + ), + ) + transcribe_overwrite: bool = Field( + default=False, + description=( + "When True, wipe the diarised transcript on every retried " + "row's source CallImportRow so the (possibly new) STT runs " + "from scratch. When False, rows that already have a " + "diarised transcript skip diarisation and only re-evaluate." + ), + ) + transcribe_mode: Optional[Literal["stt_llm", "llm_only"]] = Field( + default=None, + description=( + "Override the run's diarisation pipeline mode for this retry. " + "``stt_llm`` runs STT then an LLM diariser; ``llm_only`` feeds " + "audio directly to a multimodal diariser LLM." + ), + ) + + # Telephony credentials for rows that must re-fetch recordings. + provider: Optional[str] = Field( + default=None, + description=( + "Override the batch's telephony provider for this retry pass. " + "Must be paired with ``telephony_integration_id``. Omit both " + "fields to keep the batch's existing pinned credentials." + ), + ) + telephony_integration_id: Optional[UUID] = Field( + default=None, + description=( + "Override the telephony credential used when re-fetching " + "recordings during this retry. Must be paired with " + "``provider``. Omit both to keep existing credentials; send " + "both as null for direct-URL retry." + ), + ) + + @model_validator(mode="after") + def validate_telephony_credential_mode(self) -> "CallImportEvaluationRetryRequest": + fields_set = self.model_fields_set + if ( + "provider" not in fields_set + and "telephony_integration_id" not in fields_set + ): + return self + has_provider = bool((self.provider or "").strip()) + has_integration = self.telephony_integration_id is not None + if has_provider != has_integration: + raise ValueError( + "provider and telephony_integration_id must both be provided " + "or both omitted for direct-URL retry." + ) + return self + + +class CallImportEvaluationRetrySkippedItem(BaseModel): + """One entry in the retry response's ``skipped`` list.""" + + eval_row_id: UUID + reason: str = Field( + ..., + description=( + "Why this row was not re-enqueued. Known values: " + "'unknown' (id not in this run), 'in_progress' " + "(status is pending/running), 'completed' (already " + "successful), 'source_row_missing'." + ), + ) + + +class CallImportEvaluationRetryResponse(BaseModel): + """Summary of a retry fan-out request.""" + + requeued: int = Field( + ..., + description="How many evaluation rows were reset and re-enqueued.", + ) + transcribe_requeued: int = Field( + default=0, + description=( + "Of those, how many were chained through a diarisation " + "task first because the diarised transcript was missing " + "(matches the auto-transcribe behavior of the create-run " + "endpoint)." + ), + ) + skipped: List[CallImportEvaluationRetrySkippedItem] = Field( + default_factory=list, + description="Rows the caller asked for that we did not re-enqueue.", + ) + + +class CallImportEvaluationBulkActionResponse(BaseModel): + """Acknowledgement for bulk cancel / force-fail requests accepted off-thread.""" + + accepted: bool = True + target_count: int = Field( + ..., + description="How many rows the background worker will process.", + ) + evaluation_id: UUID + + +class CallImportMetricSummary(BaseModel): + """Lightweight metric descriptor returned alongside an evaluation.""" + + id: UUID + name: str + metric_type: Optional[str] = None + description: Optional[str] = None + parent_metric_id: Optional[UUID] = None + selection_mode: Optional[SelectionMode] = None + # Surfaced so the Flow tab can decide whether to render the + # Discovered Labels panel next to a multi_label parent. Defaults to + # False to keep legacy clients (and standalone metrics) unaffected. + allow_discovery: bool = False + + model_config = ConfigDict(from_attributes=True) + + +class CallImportEvaluationResponse(BaseModel): + """Parent record describing one evaluation run over a batch.""" + + id: UUID + call_import_id: UUID + organization_id: UUID + name: Optional[str] = None + selected_metric_ids: List[UUID] = Field(default_factory=list) + # Parent UUID string -> [child UUID string]. Captured at run creation + # so the UI can rebuild the parent/child tree even after metrics are + # renamed or deleted. Empty / NULL = no hierarchy was used. + selected_metric_groups: Optional[Dict[str, List[str]]] = None + metrics: List[CallImportMetricSummary] = Field(default_factory=list) + status: str + total_rows: int + completed_rows: int + failed_rows: int + error_message: Optional[str] = None + llm_provider: Optional[str] = None + llm_model: Optional[str] = None + llm_credential_id: Optional[UUID] = None + llm_config: Optional[Dict[str, Any]] = None + metric_llm_overrides: Optional[Dict[str, Any]] = None + stt_provider: Optional[str] = None + stt_model: Optional[str] = None + stt_credential_id: Optional[UUID] = None + # Run-level LLM diariser config. Surfaced so the UI can show + # "Diarised via openai/gpt-4o-mini" on the evaluation header and + # pre-fill the retry modal with the previously-used prompt. + diarisation_llm_provider: Optional[str] = None + diarisation_llm_model: Optional[str] = None + diarisation_llm_credential_id: Optional[UUID] = None + diarisation_prompt: Optional[str] = None + # Diarisation pipeline shape this run was created with. ``stt_llm`` + # (default) is the legacy STT-then-LLM-diariser flow; ``llm_only`` + # means the audio was fed directly to a multimodal diariser LLM. + # Surfaced so the retry modal can preselect the right mode and the + # eval header can render "Diarised via LLM only (Gemini)" instead of + # an empty STT label. + transcribe_mode: Literal["stt_llm", "llm_only"] = "stt_llm" + # Which transcript column this run scored against. All current runs + # use diarised; legacy rows may still carry ``production``. + transcript_source: CallImportEvaluationTranscriptSource = "diarised" + # Sibling evaluation ids created in the same Run Evaluation request. + # Populated only on the POST response (and only when the user ticked + # both Production and Diarised in the modal G�� the backend creates + # one ``CallImportEvaluation`` per source and links them via this + # field so the frontend can deep-link to either run). Empty for all + # other reads. + sibling_evaluation_ids: List[UUID] = Field(default_factory=list) + started_at: Optional[datetime] = None + finished_at: Optional[datetime] = None + created_at: datetime + updated_at: datetime + created_by_email: Optional[str] = None + last_updated_by_email: Optional[str] = None + # Cached LLM-generated TLDR for the Visualizations tab. Lazily + # populated by ``POST /evaluations/{eval_id}/insights``; ``None`` + # for runs the user has not summarised yet. ``is_stale`` on the + # nested object is set by the route, not the model. + tldr_summary: Optional["EvaluationTldrSummary"] = None + # Cached LLM-generated user insights for External Audit PDF section 03. + user_insights: Optional["EvaluationUserInsightsState"] = None + # Cached per-metric failure clustering for internal diagnostics. + metric_clusters: Optional["EvaluationMetricClustersState"] = None + # True when the user opted into top-level metric discovery on the + # Run Evaluation modal. The frontend uses this to gate the + # "Discovered metrics" panel on the Flow tab. + discover_new_metrics: bool = False + bulk_operation: Optional[ + Literal["abort", "force_fail_pending", "retry"] + ] = Field( + default=None, + description=( + "When set, a bulk background operation (abort, force-fail pending, " + "or retry) is still running for this evaluation. Other mutating " + "actions are rejected until it completes." + ), + ) + + model_config = ConfigDict(from_attributes=True) + + +class CallImportEvaluationListResponse(BaseModel): + """Wrapper for listing evaluations on a single batch.""" + + items: List[CallImportEvaluationResponse] + total: int + + +class CallImportEvaluationRowResponse(BaseModel): + """Per-source-row evaluation output (one Metric set applied to one row). + + ``raw_columns``, ``recording_url`` and ``recording_s3_key`` come from + the parent ``CallImportRow`` so the row-detail panel can show the + full CSV row metadata + audio without a second round-trip. The UI + prefers ``recording_s3_key`` (resolved via a presigned URL) over + ``recording_url`` so playback uses our downloaded copy instead of + the raw provider URL, which is often expired/auth-gated. + """ + + id: UUID + evaluation_id: UUID + call_import_row_id: UUID + row_index: Optional[int] = None + # Renamed from ``external_call_id``; same value, mirrors the renamed + # ``call_import_rows.conversation_id`` column. + conversation_id: Optional[str] = None + transcript: Optional[str] = None + raw_columns: Optional[Dict[str, Any]] = None + recording_url: Optional[str] = None + recording_date: Optional[date] = None + recording_s3_key: Optional[str] = None + diarised_transcript_status: Optional[str] = None + diarised_transcript_error: Optional[str] = None + status: str + metric_scores: Dict[str, Any] = Field(default_factory=dict) + error_message: Optional[str] = None + started_at: Optional[datetime] = None + finished_at: Optional[datetime] = None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class CallImportEvaluationRowListResponse(BaseModel): + """Paginated per-row evaluation results.""" + + items: List[CallImportEvaluationRowResponse] + total: int + page: int + page_size: int + + +class CallImportRowBulkDelete(BaseModel): + """Request body for deleting multiple rows from a call-import batch.""" + + row_ids: List[UUID] = Field( + ..., + min_length=1, + description="Row ids to delete (must belong to the same call import).", + ) + + +class CallImportRowBulkDeleteResponse(BaseModel): + """Response after a bulk-delete pass over ``CallImportRow`` rows.""" + + deleted: int = Field( + ..., + description="How many rows were actually removed (unknown ids are skipped).", + ) + status: Literal["completed", "accepted"] = Field( + default="completed", + description=( + "``accepted`` when deletion was queued to run asynchronously; " + "``completed`` when rows were removed before the response." + ), + ) + + +class CallImportRetryFailedRowsRequest(BaseModel): + """Optional credential override when re-enqueueing failed import rows.""" + + provider: Optional[str] = Field( + default=None, + description=( + "Telephony provider key for this retry pass. Omit together with " + "``telephony_integration_id`` to download from CSV recording URLs." + ), + ) + telephony_integration_id: Optional[UUID] = Field( + default=None, + description=( + "Telephony credential to use for this retry pass. Omit together " + "with ``provider`` for direct-URL retry." + ), + ) + + @model_validator(mode="after") + def validate_credential_mode(self) -> "CallImportRetryFailedRowsRequest": + has_provider = bool((self.provider or "").strip()) + has_integration = self.telephony_integration_id is not None + if has_provider != has_integration: + raise ValueError( + "provider and telephony_integration_id must both be provided " + "or both omitted for direct-URL retry." + ) + return self + + +class CallImportRetryFailedRowsResponse(BaseModel): + """Summary of a retry pass over failed call-import rows.""" + + requeued: int = Field( + ..., + description=( + "Rows reset to pending and successfully re-enqueued on the " + "``imports`` worker queue." + ), + ) + enqueue_failed: int = Field( + default=0, + description=( + "Rows that were eligible for retry but failed to enqueue again. " + "These rows are left in ``failed`` with an enqueue error." + ), + ) + skipped: int = Field( + default=0, + description=( + "Rows skipped because they were no longer in ``failed`` at retry " + "time (for example, already retried from another tab)." + ), + ) + + +# --- Diarization / Transcription request/response shapes --- + + +class CallImportTranscribeRequest(BaseModel): + """Body for kicking off diarization for one or many call-import rows. + + The same shape powers both the per-row endpoint (where ``row_ids`` + is ignored) and the batch-level endpoint. ``only_missing`` is the + safe default G�� rows with an existing transcript are skipped unless + ``overwrite_existing`` is set. + + Two modes are supported: + + * ``mode="stt_llm"`` (default) G�� the legacy two-stage pipeline: STT + produces plain text, an LLM splits it into agent/user turns using + ``diarization_prompt``. ``stt_provider`` and ``stt_model`` are + required in this mode. + * ``mode="llm_only"`` G�� skip STT entirely and hand the recording's + audio bytes to a multimodal chat model along with + ``diarization_prompt``. The model both transcribes and diarises in + a single pass. The STT fields are ignored (and must be omitted / + null). Only providers whose chat API accepts audio input (OpenAI + ``gpt-4o-audio-*``, Google Gemini ``1.5/2.0``) are usable; other + providers will surface a typed error on the row. + """ + + mode: Literal["stt_llm", "llm_only"] = Field( + default="stt_llm", + description=( + "Pipeline shape. 'stt_llm' (default) runs STT then an LLM " + "diariser over the resulting text. 'llm_only' skips STT and " + "feeds the raw audio to a multimodal LLM together with " + "``diarization_prompt`` for a single-pass transcribe + " + "diarise." + ), + ) + stt_provider: Optional[str] = Field( + default=None, + max_length=50, + description=( + "STT provider key, e.g. 'deepgram' or 'openai'. Required when " + "``mode='stt_llm'``; must be omitted when ``mode='llm_only'``." + ), + ) + stt_model: Optional[str] = Field( + default=None, + max_length=100, + description=( + "STT model name, e.g. 'nova-2' or 'whisper-1'. Required when " + "``mode='stt_llm'``; must be omitted when ``mode='llm_only'``." + ), + ) + credential_id: Optional[UUID] = Field( + default=None, + description="Optional AIProvider/Integration row to pin for this run.", + ) + language: Optional[str] = Field( + default=None, + max_length=20, + description="Optional ISO language hint, e.g. 'en'.", + ) + only_missing: bool = Field( + default=True, + description=( + "When true, rows with an existing transcript are skipped (the " + "default safe behavior)." + ), + ) + overwrite_existing: bool = Field( + default=False, + description=( + "When true, existing transcripts are replaced. Mutually " + "exclusive with only_missing." + ), + ) + row_ids: Optional[List[UUID]] = Field( + default=None, + description=( + "Restrict the run to a specific subset of rows. NULL = every " + "row in the import (subject to only_missing)." + ), + ) + # --- LLM diariser config --- + # In ``stt_llm`` mode diarisation runs as a *second* step: STT + # produces plain text, then this LLM splits it into agent/user + # turns. In ``llm_only`` mode this same LLM directly receives the + # audio and the prompt. Both fields are always mandatory because + # there is no longer a pyannote fallback and ``llm_only`` cannot + # function without an LLM either. + diarization_llm_provider: str = Field( + ..., + max_length=50, + description=( + "LLM provider that diarises the call. In ``stt_llm`` it sees " + "the STT text; in ``llm_only`` it sees the raw audio." + ), + ) + diarization_llm_model: str = Field( + ..., + max_length=100, + description=( + "LLM model name. In ``llm_only`` mode this must be a model " + "that accepts audio input (e.g. 'gpt-4o-audio-preview', " + "'gemini-1.5-pro')." + ), + ) + diarization_llm_credential_id: Optional[UUID] = Field( + default=None, + description=( + "Optional AIProvider row to pin for the diarisation LLM." + ), + ) + diarization_prompt: Optional[str] = Field( + default=None, + max_length=10_000, + description=( + "Operator-supplied system prompt for the diariser LLM. " + "When NULL/empty the worker uses the canonical default " + "(see ``GET /api/v1/call-imports/diarisation-prompt-default``)." + ), + ) + + @model_validator(mode="after") + def _validate_mode_fields(self) -> "CallImportTranscribeRequest": + """Enforce STT-field presence rules based on ``mode``. + + ``stt_llm`` (default) requires both STT fields G�� the worker + cannot diarise without a transcript. ``llm_only`` forbids them + so the API contract makes it clear that the audio is going + straight to the LLM; passing both would be ambiguous about + which path the worker should take. + """ + stt_provider = (self.stt_provider or "").strip() if self.stt_provider else None + stt_model = (self.stt_model or "").strip() if self.stt_model else None + if self.mode == "stt_llm": + if not stt_provider or not stt_model: + raise ValueError( + "stt_provider and stt_model are required when " + "mode='stt_llm'." + ) + else: # llm_only + if stt_provider or stt_model: + raise ValueError( + "stt_provider/stt_model must be omitted when " + "mode='llm_only'; the LLM consumes the audio " + "directly." + ) + return self + + +class CallImportDiarisationPromptDefaultResponse(BaseModel): + """Wrapper for the canonical diariser-prompt fetched by the modal.""" + + prompt: str = Field( + ..., + description=( + "The exact prompt the worker falls back to when the caller " + "leaves ``diarization_prompt`` blank. The frontend pre-fills " + "the textarea with this value so the operator can edit it." + ), + ) + + +class CallImportRowIdsResponse(BaseModel): + """Flat row-id list for cross-page bulk selection. + + Powers the "Select all M rows in this import" affordance on the + detail page G�� returning only ids keeps the payload tiny so the UI + can hold the full set in memory even for batches with thousands + of rows. The frontend then passes those ids straight to the + existing bulk-delete / bulk-transcribe endpoints. + """ + + ids: List[UUID] = Field( + default_factory=list, + description=( + "Every ``CallImportRow.id`` that matches the ``q`` and " + "``diarised_status`` filters (or every row when neither is " + "supplied), sorted by ``row_index``." + ), + ) + total: int = Field( + ..., + description=( + "Length of ``ids``. Sent explicitly so callers can show a " + "count without re-measuring the array." + ), + ) + + +class CallImportTranscribeResponse(BaseModel): + """Summary of a transcribe fan-out request.""" + + queued: int = Field( + ..., + description=( + "How many rows were enqueued for diarization. Skipped rows " + "(missing recording, transcript already present, etc.) are " + "not counted." + ), + ) + skipped_rows: int = Field( + default=0, + description="Rows excluded by only_missing or because they had no recording.", + ) + skipped_reason_counts: Dict[str, int] = Field( + default_factory=dict, + description="Per-reason breakdown of skipped rows for the UI to surface.", + ) + accepted: bool = Field( + default=False, + description=( + "When true, diarization setup was queued to a background worker " + "and ``queued`` reflects zero until the worker finishes enqueue." + ), + ) + + +class CallImportCancelDiarisationRequest(BaseModel): + """Body for the batch cancel-diarisation endpoint. + + Omit ``row_ids`` (or pass ``null``) to cancel every row in the + import whose ``diarised_transcript_status`` is currently + ``pending`` or ``running``. Pass an explicit list to scope the + cancel to a subset (e.g. the rows the operator selected in the + UI). + """ + + row_ids: Optional[List[UUID]] = Field( + default=None, + description=( + "Optional subset of CallImportRow UUIDs. ``None`` cancels " + "every pending / running diarisation in the import." + ), + ) + + +class CallImportCancelDiarisationResponse(BaseModel): + """Summary of a cancel-diarisation request. + + ``cancelled`` counts rows that were actively pending / running + when the cancel landed and got flipped to ``failed`` with a + "Cancelled by user" error. ``skipped`` counts rows that were + requested (or matched the implicit "all rows" filter) but were + not in a cancellable state G�� typically because they had already + finished or were never queued for diarisation in the first place. + """ + + cancelled: int = Field( + ..., + description=( + "Rows whose in-flight Celery task was revoked and whose " + "``diarised_transcript_status`` was flipped to ``failed`` " + "with a 'Cancelled by user' error message." + ), + ) + skipped: int = Field( + default=0, + description=( + "Rows that were requested but not in a cancellable state " + "(idle / completed / already failed)." + ), + ) + + +# --- Per-run aggregation / visualization payloads --- + + +class CallImportMetricHistogramBucket(BaseModel): + """One bin of a numeric metric histogram.""" + + x0: float + x1: float + count: int + + +class CallImportMetricValueCount(BaseModel): + """One row of a categorical metric's value frequency table.""" + + label: str + count: int + + +class CallImportMetricLabelPair(BaseModel): + """One unordered pair-count cell of a multi-label parent's + co-occurrence matrix. + + ``a`` and ``b`` are child label names; ``count`` is the number of + rows on which both labels fired together (intersection size). + Pairs are emitted with ``a < b`` lexicographically so the matrix + can be reconstructed without duplicates on the frontend. + """ + + a: str + b: str + count: int + + +class CallImportMetricAggregate(BaseModel): + """Per-metric aggregate computed from an evaluation run's rows. + + Numeric metrics return summary statistics + histogram buckets; + categorical / pass-fail / text metrics return the top value counts. + Both shapes can coexist if a metric mixes types G�� the UI prefers + histogram when present, falls back to value_counts otherwise. + """ + + metric_id: str + metric_name: str + metric_type: Optional[str] = None + metric_category: str = "quality" + # True when this aggregate represents a multi-label parent metric + # (selection_mode == "multi_label" with no parent_metric_id). For + # those, ``value_counts`` lists per-child label tallies and the + # rows scored != sum(value_counts.count). The UI uses this flag to + # force a horizontal bar layout (slices wouldn't sum to 100%) and + # to label the n-badge as rows scored, not label occurrences. + is_multi_label_parent: bool = False + count: int = 0 + skipped_count: int = 0 + error_count: int = 0 + # Numeric stats (None when no numeric values were observed) + mean: Optional[float] = None + median: Optional[float] = None + p25: Optional[float] = None + p75: Optional[float] = None + p95: Optional[float] = None + min: Optional[float] = None + max: Optional[float] = None + stddev: Optional[float] = None + histogram_buckets: List[CallImportMetricHistogramBucket] = Field( + default_factory=list + ) + value_counts: List[CallImportMetricValueCount] = Field(default_factory=list) + # Pairwise label intersections for multi-label parent metrics. + # Empty for everything else. The frontend reconstructs a square + # symmetric matrix from these unordered pairs and renders the + # co-occurrence heatmap chart type. + co_occurrence: List[CallImportMetricLabelPair] = Field(default_factory=list) + + +class MetricPeriodDelta(BaseModel): + """Week-over-week (or baseline-run) delta for one metric.""" + + label: str + detail: str + why: Optional[str] = None + + +class CallImportEvaluationAggregateResponse(BaseModel): + """Aggregated metric distributions for a single evaluation run.""" + + evaluation_id: UUID + total_rows: int + completed_rows: int + failed_rows: int + metrics: List[CallImportMetricAggregate] = Field(default_factory=list) + period_deltas: Dict[str, MetricPeriodDelta] = Field(default_factory=dict) + baseline_evaluation_id: Optional[UUID] = None + failure_policies_source: Optional[Literal["inferred", "user"]] = Field( + default=None, + description=( + "Whether flagged-rate semantics use user-confirmed failure policies " + "or inferred defaults from the Failure diagnostics flow." + ), + ) + + +# --- LLM-generated TLDR for the Visualizations tab --- + + + +class EvaluatorResultsAggregateResponse(BaseModel): + """Chart-friendly metric rollups for evaluator results in a suite or scenario scope.""" + + scope: str + suite_id: Optional[UUID] = None + agent_id: Optional[UUID] = None + scenario_id: Optional[UUID] = None + total_rows: int = 0 + completed_rows: int = 0 + failed_rows: int = 0 + metrics: List[CallImportMetricAggregate] = Field(default_factory=list) + + +# --- LLM-generated TLDR for the Visualizations tab --- + + + +class EvaluationTldrSummary(BaseModel): + """Cached LLM-generated narrative + bullet patterns for an eval run. + + Persisted on ``CallImportEvaluation.tldr_summary`` (JSONB) and + rendered above the per-metric charts. ``generated_at_completed_rows`` + is the snapshot of ``completed_rows`` at the time the summary was + written; the API compares it against the current count to flag + ``is_stale`` so the UI can prompt for a regenerate. + """ + + narrative: str + patterns: List[str] = Field(default_factory=list) + metric_insights: Dict[str, str] = Field(default_factory=dict) + generated_at: datetime + generated_at_completed_rows: int = 0 + provider: Optional[str] = None + model: Optional[str] = None + is_stale: bool = False + + +class EvaluationInsightsRequest(BaseModel): + """Body for ``POST /evaluations/{eval_id}/insights``. + + All fields are optional. When ``provider``/``model`` are unset the + backend resolves the org's first active OpenAI/Anthropic/Google + provider (mirroring the Prompt Partials AI-generate flow) so + callers that don't care can simply post ``{}``. + """ + + regenerate: bool = False + provider: Optional[str] = None + model: Optional[str] = Field(default=None, min_length=1) + credential_id: Optional[UUID] = None + max_llm_calls: Optional[int] = Field( + default=None, + ge=20, + le=500, + description=( + "Max LLM calls for user-insights sampling (extraction + synthesis). " + "Defaults to 200 when omitted." + ), + ) + + +class UserInsightCategory(BaseModel): + label: str + count: int + share_pct: float + + +class UserInsightEvidenceTurn(BaseModel): + speaker: str + text: str + + +class UserInsightEvidence(BaseModel): + conversation_id: Optional[str] = None + quote: str + turns: List[UserInsightEvidenceTurn] = Field(default_factory=list) + + +class EvaluationUserInsightItem(BaseModel): + id: str + title: str + categories: List[UserInsightCategory] = Field(default_factory=list) + observation: str + evidence: UserInsightEvidence + + +class EvaluationUserInsightsState(BaseModel): + """Cached map-reduce LLM user insights for an evaluation run.""" + + status: Literal["idle", "running", "completed", "failed"] = "idle" + insights: List[EvaluationUserInsightItem] = Field(default_factory=list) + overview: Optional[str] = None + generated_at: Optional[datetime] = None + generated_at_completed_rows: int = 0 + progress: Optional[Dict[str, int]] = None + provider: Optional[str] = None + model: Optional[str] = None + llm_calls_used: int = 0 + max_llm_calls: Optional[int] = None + error_message: Optional[str] = None + is_stale: bool = False + + +class EvaluationUserInsightsRequest(BaseModel): + """Body for ``POST /evaluations/{eval_id}/user-insights``.""" + + regenerate: bool = False + force: bool = False + provider: Optional[str] = None + model: Optional[str] = Field(default=None, min_length=1) + credential_id: Optional[UUID] = None + max_llm_calls: Optional[int] = Field(default=None, ge=20, le=500) + + +MetricClusterGapLabel = Literal[ + "LOGIC_GAP", + "UNDERSPEC", + "EXISTS_NO_TRIGGER", + "MISSING", +] + +FailurePolicyNumericOp = Literal["lt", "lte", "gt", "gte"] + + +class MetricFailurePolicy(BaseModel): + """Per-metric definition of which scores count as failures for this evaluation.""" + + metric_id: str + failure_values: List[str] = Field( + default_factory=list, + description="Normalized lowercase labels that count as failure (single-choice, enum, boolean-as-category).", + ) + failure_child_names: List[str] = Field( + default_factory=list, + description="Child label names that count as failure for multi_label parents.", + ) + numeric_rule: Optional[Dict[str, Any]] = Field( + default=None, + description='Numeric failure rule, e.g. {"op": "lt", "threshold": 0.5}.', + ) + + +class MetricFailurePolicyValueCount(BaseModel): + label: str + count: int = 0 + + +class MetricFailurePolicyMetricPreview(BaseModel): + metric_id: str + metric_name: str + metric_type: Optional[str] = None + selection_mode: Optional[str] = None + is_multi_label_parent: bool = False + value_counts: List[MetricFailurePolicyValueCount] = Field(default_factory=list) + child_names: List[str] = Field(default_factory=list) + row_count_by_value: Dict[str, int] = Field(default_factory=dict) + suggested_policy: MetricFailurePolicy + effective_policy: MetricFailurePolicy + + +class MetricFailurePoliciesResponse(BaseModel): + previews: List[MetricFailurePolicyMetricPreview] = Field(default_factory=list) + policies: Dict[str, MetricFailurePolicy] = Field(default_factory=dict) + source: Literal["inferred", "user"] = "inferred" + updated_at: Optional[datetime] = None + + +class MetricFailurePoliciesSaveRequest(BaseModel): + policies: Dict[str, MetricFailurePolicy] = Field(default_factory=dict) + source: Literal["user"] = "user" + + +class MetricClusterEvidenceTurn(BaseModel): + speaker: str + text: str + + +class MetricClusterEvidence(BaseModel): + conversation_id: Optional[str] = None + evaluation_row_id: Optional[UUID] = None + quote: str = "" + turns: List[MetricClusterEvidenceTurn] = Field(default_factory=list) + + +class MetricSubCluster(BaseModel): + label: str + count: int = 0 + share_pct: float = 0.0 + + +class MetricCluster(BaseModel): + id: str + label: str + gap_label: MetricClusterGapLabel + level: int = 1 + count: int = 0 + share_pct: float = 0.0 + sub_clusters: List[MetricSubCluster] = Field(default_factory=list) + observation: str = "" + failure_reason: str = "" + evidence: MetricClusterEvidence = Field(default_factory=MetricClusterEvidence) + is_discovered: bool = False + + +class MetricClusterGroup(BaseModel): + metric_id: str + metric_name: str + flagged_count: int = 0 + failure_reason: str = "" + clusters: List[MetricCluster] = Field(default_factory=list) + + +class DiscoveredProblemCluster(BaseModel): + id: str + label: str + gap_label: MetricClusterGapLabel + count: int = 0 + share_pct: float = 0.0 + observation: str = "" + failure_reason: str = "" + evidence: MetricClusterEvidence = Field(default_factory=MetricClusterEvidence) + + +class RcaRepeatedPatternRow(BaseModel): + metric_id: str + metric_name: str + top_rca_patterns: str = "" + evidence_share_pct: float = 0.0 + evidence_calls: int = 0 + evidence_cluster_count: int = 0 + failure_reason: str = "" + + +class RcaMetricHotspotRow(BaseModel): + metric_id: str + metric_name: str + description: str = "" + metric_rate_pct: float = 0.0 + flagged_calls: int = 0 + + +class RcaPromptAreaRow(BaseModel): + label: str + share_pct: float = 0.0 + gap_label: MetricClusterGapLabel + + +class MetricClustersRcaSummary(BaseModel): + total_clusters: int = 0 + total_clustered_instances: int = 0 + total_flagged_instances: int = 0 + analysed_calls: int = 0 + repeated_patterns: List[RcaRepeatedPatternRow] = Field(default_factory=list) + metric_hotspots: List[RcaMetricHotspotRow] = Field(default_factory=list) + prompt_areas: List[RcaPromptAreaRow] = Field(default_factory=list) + + +class EvaluationMetricClustersState(BaseModel): + """Cached per-metric failure clustering for internal diagnostics.""" + + status: Literal["idle", "running", "completed", "failed", "cancelled"] = "idle" + groups: List[MetricClusterGroup] = Field(default_factory=list) + discovered_problems: List[DiscoveredProblemCluster] = Field( + default_factory=list + ) + overview: Optional[str] = None + generated_at: Optional[datetime] = None + generated_at_completed_rows: int = 0 + progress: Optional[Dict[str, int]] = None + provider: Optional[str] = None + model: Optional[str] = None + llm_calls_used: int = 0 + max_llm_calls: Optional[int] = None + error_message: Optional[str] = None + is_stale: bool = False + selected_evaluation_row_ids: List[str] = Field( + default_factory=list, + description="Evaluation row IDs included in the last clustering run.", + ) + failure_policies: Dict[str, MetricFailurePolicy] = Field(default_factory=dict) + failure_policies_source: Literal["inferred", "user"] = "inferred" + failure_policies_updated_at: Optional[datetime] = None + rca_summary: Optional[MetricClustersRcaSummary] = None + + +class MetricClusterEligibleRow(BaseModel): + """Completed evaluation row with at least one flagged quality metric.""" + + evaluation_row_id: UUID + conversation_id: Optional[str] = None + row_index: Optional[int] = None + flagged_metric_names: List[str] = Field(default_factory=list) + + +class MetricClusterEligibleRowsResponse(BaseModel): + items: List[MetricClusterEligibleRow] = Field(default_factory=list) + total: int = 0 + + +class PromptImprovementSuggestion(BaseModel): + """One LLM-generated prompt edit to address a failure cluster.""" + + id: str + metric_id: str + metric_name: str + cluster_id: str + cluster_label: str + gap_label: MetricClusterGapLabel + share_pct: float = 0.0 + priority: Literal["high", "medium", "low"] = "medium" + change_type: Literal["edit", "add"] = "add" + target_section: str = "" + anchor_excerpt: str = "" + current_gap: str = "" + suggested_text: str = "" + rationale: str = "" + flow_node_id: str = "" + flow_node_label: str = "" + + +class EvaluationPromptImprovementsState(BaseModel): + """Cached prompt improvement suggestions for an evaluation run.""" + + status: Literal["idle", "running", "completed", "failed"] = "idle" + imported_agent_id: Optional[str] = None + imported_agent_name: Optional[str] = None + suggestions: List[PromptImprovementSuggestion] = Field(default_factory=list) + overview: Optional[str] = None + generated_at: Optional[datetime] = None + generated_at_completed_rows: int = 0 + provider: Optional[str] = None + model: Optional[str] = None + error_message: Optional[str] = None + is_stale: bool = False + + +class EvaluationPromptImprovementsRequest(BaseModel): + """Body for ``POST /evaluations/{eval_id}/prompt-improvements``.""" + + imported_agent_id: UUID + regenerate: bool = False + force: bool = False + provider: Optional[str] = None + model: Optional[str] = None + credential_id: Optional[UUID] = None + + +class EvaluationMetricClustersRequest(BaseModel): + """Body for ``POST /evaluations/{eval_id}/metric-clusters``.""" + + regenerate: bool = False + force: bool = False + provider: Optional[str] = None + model: Optional[str] = Field(default=None, min_length=1) + credential_id: Optional[UUID] = None + max_llm_calls: Optional[int] = Field(default=None, ge=20, le=500) + evaluation_row_ids: Optional[List[UUID]] = Field( + default=None, + description=( + "Subset of completed evaluation row IDs to cluster. When omitted, " + "all completed rows with at least one flagged quality metric are used." + ), + ) + row_limit: Optional[int] = Field( + default=None, + ge=1, + description=( + "Use the first N eligible rows (by row order). Mutually exclusive " + "with evaluation_row_ids." + ), + ) + failure_policies: Optional[Dict[str, MetricFailurePolicy]] = Field( + default=None, + description="Per-metric failure policies confirmed in the cluster modal.", + ) + + +# Resolve the forward reference on ``CallImportEvaluationResponse`` +# (defined further up the file) now that ``EvaluationTldrSummary`` +# exists. Without this Pydantic raises at first ``.model_validate`` +# because the string annotation can't be evaluated. +CallImportEvaluationResponse.model_rebuild() + + +# --- Cross-run insights for a CallImport batch --- + + +class CallImportInsightsRunPoint(BaseModel): + """One run's mean for a metric, used to render trend lines.""" + + evaluation_id: UUID + name: Optional[str] = None + created_at: datetime + mean: Optional[float] = None + completed_rows: int = 0 + + +class CallImportInsightsMetric(BaseModel): + """Per-metric history across every evaluation run on this import.""" + + metric_id: str + metric_name: str + metric_type: Optional[str] = None + latest: Optional[CallImportMetricAggregate] = None + trend: List[CallImportInsightsRunPoint] = Field(default_factory=list) + + +class CallImportInsightsResponse(BaseModel): + """Aggregated cross-run signals for a single call-import batch.""" + + call_import_id: UUID + total_rows: int + rows_with_transcript: int + rows_without_transcript: int + transcript_source_counts: Dict[str, int] = Field(default_factory=dict) + evaluation_count: int = 0 + metrics: List[CallImportInsightsMetric] = Field(default_factory=list) + + +# --- Flow chart visualization for hierarchical metrics --- + + +class MetricFlowNode(BaseModel): + """One step in the LLM-inferred temporal flow for a parent metric. + + Represents a child sub-metric label. ``count`` is the number of rows + in the evaluation where this child appears anywhere in its + ``sequence`` array. ``is_terminal`` is set when the child is the + last entry in a meaningful fraction of those sequences. + + ``is_discovered`` is set when the node represents an LLM-discovered + candidate (parent has ``allow_discovery=true``) rather than a + user-defined child. The id of a discovered node is prefixed with + ``disc:`` so it can't collide with real child UUIDs. + """ + + id: str + label: str + count: int = 0 + is_terminal: bool = False + is_discovered: bool = False + + +class MetricFlowEdge(BaseModel): + """One directed transition between two children across all rows. + + ``count`` is the number of rows where ``source`` immediately + precedes ``target`` in the sequence. The synthetic ``START`` node + is used as the ``source`` for the first child in every sequence. + """ + + source: str + target: str + count: int = 0 + + +class MetricFlowResponse(BaseModel): + """Aggregate flow diagram payload for a single parent metric.""" + + parent_metric_id: str + parent_metric_name: str + selection_mode: Optional[SelectionMode] = None + nodes: List[MetricFlowNode] = Field(default_factory=list) + edges: List[MetricFlowEdge] = Field(default_factory=list) + total_rows: int = 0 + rows_with_sequence: int = 0 + + +class DiscoveredLabelItem(BaseModel): + """One LLM-discovered candidate sub-label aggregated across rows. + + ``key`` is the slugified label identifier (matches what appears in + ``sequence`` entries). ``count`` is the number of rows in the + evaluation that emitted this slug. ``sample_rationale`` is the + first non-empty rationale captured from any row (back-compat + field, identical to ``examples[0]`` when present). ``examples`` + holds up to 3 distinct rationales G�� the UI surfaces 2 of them as + ``Examples:`` in the rubric on Promote, with the third kept as + headroom in case the first is unhelpful. + """ + + key: str + name: str + description: Optional[str] = None + sample_rationale: Optional[str] = None + examples: List[str] = Field(default_factory=list, max_length=3) + count: int = 0 + + +class DiscoveredLabelsResponse(BaseModel): + """List of discovered candidate sub-labels for a parent metric.""" + + parent_metric_id: str + items: List[DiscoveredLabelItem] = Field(default_factory=list) + + +class DiscoveredLabelMergeRequest(BaseModel): + """Body for POST /evaluations/{eval_id}/discovered-labels/merge. + + Rewrites every row's ``metric_scores[parent_id].discovered_labels`` + entries whose key is ``from_key`` to use ``to_key`` instead, so the + user can collapse near-duplicate candidates ("On Hold" / "Customer + Put On Hold") into a single promoted child. + """ + + parent_metric_id: UUID + from_key: str = Field(..., min_length=1, max_length=120) + to_key: str = Field(..., min_length=1, max_length=120) + + +class DiscoveredLabelDeleteRequest(BaseModel): + """Body for POST /evaluations/{eval_id}/discovered-labels/delete. + + Strips a candidate sub-label from every row's + ``discovered_labels`` list AND from each row's ``sequence`` array, + then tombstones the slug at the evaluation level so workers + finishing later can't re-introduce it. Use for gibberish or + irrelevant candidates the LLM proposed; for near-duplicates that + you want to keep but unify, use the merge endpoint instead. + """ + + parent_metric_id: UUID + key: str = Field(..., min_length=1, max_length=120) + + +class PromoteDiscoveredChildRequest(BaseModel): + """Body for POST /metrics/{parent_id}/children/from-discovered. + + ``key`` is the slug under which the candidate is currently stored + on per-row ``metric_scores``. The newly-created child Metric's + name is normalized so ``slugify(name) == key``, which keeps every + already-scored row's ``sequence`` array resolvable against the + promoted child without a backfill. + """ + + key: str = Field(..., min_length=1, max_length=120) + name: str = Field(..., min_length=1, max_length=120) + description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) + # Default True: when promoting a discovered label we want the new + # sub-metric to always capture rationales going forward, since the + # candidate was itself proposed *with* a rationale and the user + # almost always wants to see why future rows hit it. Explicit False + # keeps the original opt-in behavior available for callers that + # don't care about rationales. + capture_rationale: bool = True + + +# --- Discovered top-level metrics (per-evaluation discovery) --- +# +# Parallel to ``DiscoveredLabelItem`` / merge / delete / promote G�� but +# scoped to the evaluation as a whole, not to a parent category metric. +# Used by the "Discovered metrics" panel at the top of the evaluation +# detail Flow tab when ``CallImportEvaluation.discover_new_metrics`` +# is true. + + +# The promote endpoint accepts these three suggested types; "category" +# creates a parent (no children yet) that the user can later extend in +# the Metrics page. +DiscoveredMetricSuggestedType = Literal["boolean", "rating", "category"] + + +class DiscoveredMetricItem(BaseModel): + """One LLM-discovered candidate top-level metric aggregated across rows. + + Mirrors :class:`DiscoveredLabelItem` but at the evaluation level + (no ``parent_metric_id``). ``suggested_type`` is the LLM's guess at + the best representation; the promote flow lets the user override + it before creating the real :class:`Metric` row. + """ + + key: str + name: str + description: Optional[str] = None + suggested_type: DiscoveredMetricSuggestedType = "boolean" + sample_rationale: Optional[str] = None + examples: List[str] = Field(default_factory=list, max_length=3) + count: int = 0 + + +class DiscoveredMetricsResponse(BaseModel): + """List of discovered candidate top-level metrics for an evaluation.""" + + evaluation_id: UUID + items: List[DiscoveredMetricItem] = Field(default_factory=list) + + +class DiscoveredMetricMergeRequest(BaseModel): + """Body for POST /evaluations/{eval_id}/discovered-metrics/merge. + + Rewrites every row's ``metric_scores["__discovered_metrics__"]`` + entries whose key is ``from_key`` to use ``to_key`` instead, and + records the redirect in ``CallImportEvaluation.discovered_metric_aliases`` + so workers finishing later can't resurrect the merged-out slug. + """ + + from_key: str = Field(..., min_length=1, max_length=120) + to_key: str = Field(..., min_length=1, max_length=120) + + +class DiscoveredMetricDeleteRequest(BaseModel): + """Body for POST /evaluations/{eval_id}/discovered-metrics/delete. + + Strips a candidate from every row's + ``metric_scores["__discovered_metrics__"]`` list and tombstones + the slug at the evaluation level (empty-string alias) so workers + finishing later can't re-introduce it. + """ + + key: str = Field(..., min_length=1, max_length=120) + + +class PromoteDiscoveredMetricRequest(BaseModel): + """Body for POST /metrics/from-discovered. + + Creates a standalone :class:`Metric` (``parent_metric_id=None``) + from an LLM-discovered candidate. The new metric's name is + normalized so ``slugify(name) == key`` to keep already-scored row + payloads resolvable against the promoted metric. ``metric_type`` + selects how the new metric will be scored on future runs; + ``"category"`` creates a ``multi_label`` parent with no children + (the user adds children via the existing Metrics page). + """ + + key: str = Field(..., min_length=1, max_length=120) + name: str = Field(..., min_length=1, max_length=120) + description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) + metric_type: DiscoveredMetricSuggestedType = "boolean" + capture_rationale: bool = True + # Optional per-type config knobs passed through to ``Metric.custom_config``. + # For ``rating`` the frontend can supply {"min": 1, "max": 5}; for + # ``boolean`` / ``category`` the field is typically empty. + custom_config: Optional[Dict[str, Any]] = None + + +# --- Workspace Schemas --- + + +class WorkspaceBase(BaseModel): + """Shared fields for workspace create/update payloads.""" + + name: str = Field(..., min_length=1, max_length=255) + + +class WorkspaceCreate(WorkspaceBase): + """Body for POST /workspaces.""" + + # Optional: derived from name when omitted; uniqueness is per-org. + slug: Optional[str] = Field( + default=None, min_length=1, max_length=255 + ) + + +class WorkspaceUpdate(BaseModel): + """Body for PATCH /workspaces/{id} (rename and/or org-admin activation).""" + + name: Optional[str] = Field(default=None, min_length=1, max_length=255) + is_active: Optional[bool] = None + + +class WorkspaceResponse(BaseModel): + """Response schema for a single workspace.""" + + id: UUID + organization_id: UUID + name: str + slug: str + is_default: bool + is_active: bool = True + created_at: datetime + updated_at: datetime + role_id: Optional[UUID] = None + role_name: Optional[str] = None + capabilities: List[str] = Field(default_factory=list) + + model_config = ConfigDict(from_attributes=True) + + +class WorkspaceRoleBase(BaseModel): + name: str = Field(..., min_length=1, max_length=255) + description: Optional[str] = None + capabilities: List[str] = Field(default_factory=list) + + +class WorkspaceRoleCreate(WorkspaceRoleBase): + pass + + +class WorkspaceRoleUpdate(BaseModel): + name: Optional[str] = Field(default=None, min_length=1, max_length=255) + description: Optional[str] = None + capabilities: Optional[List[str]] = None + + +class WorkspaceRoleResponse(BaseModel): + id: UUID + organization_id: UUID + name: str + description: Optional[str] = None + capabilities: List[str] + is_system: bool + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class WorkspaceMemberResponse(BaseModel): + id: UUID + workspace_id: UUID + user_id: UUID + role_id: UUID + role_name: str + user_email: str + user_name: Optional[str] = None + added_by_user_id: Optional[UUID] = None + created_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class WorkspaceMemberCreate(BaseModel): + user_id: UUID + role_id: UUID + + +class WorkspaceMemberUpdate(BaseModel): + role_id: UUID + + +class CapabilityInfoResponse(BaseModel): + key: str + label: str + + +class CapabilityDomainResponse(BaseModel): + key: str + label: str + capabilities: List[CapabilityInfoResponse] diff --git a/app/services/ai/llm_gateway.py b/app/services/ai/llm_gateway.py index b0731840..fc2b3a71 100644 --- a/app/services/ai/llm_gateway.py +++ b/app/services/ai/llm_gateway.py @@ -547,27 +547,38 @@ def resolve_llm_gateway( return config +def _credential_routing_context(credential: Any) -> CredentialRoutingContext: + """Build a full routing context from a row, enum, or context object.""" + if isinstance(credential, CredentialRoutingContext): + return credential + if hasattr(credential, "provider"): + return routing_context_from_ai_provider(credential) + if hasattr(credential, "platform"): + return routing_context_from_integration(credential) + return CredentialRoutingContext( + routing_mode=_normalize_routing_mode(credential), + ) + + def get_credential_effective_gateway_interface( organization_id: UUID, db: Session, - gateway_interface: Optional[str], + credential: Any, ) -> GatewayInterface: """Resolved Bifrost API surface for a credential (for API responses).""" - credential = CredentialRoutingContext( - gateway_interface=_normalize_gateway_interface(gateway_interface or "inherit"), - ) + ctx = _credential_routing_context(credential) org = _get_org_raw_settings(organization_id, db) platform = _platform_config() - return _resolve_gateway_interface(credential, org, platform) + return _resolve_gateway_interface(ctx, org, platform) def get_credential_effective_routing_label( organization_id: UUID, db: Session, - routing_mode: Any, + credential: Any, ) -> EffectiveRouting: """Resolved routing label for API responses.""" - ctx = CredentialRoutingContext(routing_mode=_normalize_routing_mode(routing_mode)) + ctx = _credential_routing_context(credential) _, effective = resolve_effective_routing(organization_id, db, ctx) return effective diff --git a/app/services/ai/transcription_service.py b/app/services/ai/transcription_service.py index fbe87c7c..955e1a7b 100644 --- a/app/services/ai/transcription_service.py +++ b/app/services/ai/transcription_service.py @@ -1,735 +1,735 @@ -""" -Transcription service for converting audio to text using various STT providers. - -Response Format: -All providers return a standardized format: -{ - "text": str, # Full transcript text - "language": str, # Language code (e.g., "en", "es") - "segments": [ # List of segments with timestamps - { - "start": float, # Start time in seconds - "end": float, # End time in seconds - "text": str # Text for this segment - } - ], - "speaker_segments": [ # Segments with speaker labels (if diarization enabled) - { - "speaker": str, # Speaker label (e.g., "Speaker 1") - "text": str, - "start": float, - "end": float - } - ], - "processing_time": float, - "raw_output": dict # Original provider response -} - -Provider-Specific Formats: -- OpenAI Whisper API: Uses verbose_json format which includes segments with timestamps -- Local Whisper: Returns segments by default with word-level timestamps available -- Google/Azure/AWS: Formats documented but not yet implemented - -Speaker Diarization: -- Whisper does NOT provide speaker diarization natively (only transcription) -- We use pyannote.audio (speaker-diarization-3.1) for accurate ML-based diarization -- Whisper provides word-level timestamps, pyannote identifies speakers, then we align -- Requires: pyannote.audio installed + diarization.huggingface_token in config.yml -- Falls back to unreliable gap-based heuristics if pyannote is unavailable -""" - -import time -import tempfile -import os -import logging -from typing import Optional, Dict, Any, List -from uuid import UUID -from pathlib import Path - -from app.models.database import ModelProvider, AIProvider, Integration -from app.core.encryption import decrypt_api_key -from app.services.credentials import resolve_ai_provider, resolve_integration -from app.services.storage.s3_service import s3_service -from app.core.exceptions import StorageError -from sqlalchemy.orm import Session - -logger = logging.getLogger(__name__) - - -class TranscriptionService: - """Service for transcribing audio files using various STT providers.""" - - def __init__(self): - """Initialize transcription service.""" - self._pyannote_pipeline = None - - def _get_ai_provider( - self, - provider: ModelProvider, - db: Session, - organization_id: UUID, - credential_id: Optional[UUID] = None, - ) -> Optional[AIProvider]: - """Resolve the AIProvider row to use, honoring ``credential_id``.""" - return resolve_ai_provider( - provider, db, organization_id, credential_id=credential_id - ) - - def _get_api_key_for_provider( - self, - provider: ModelProvider, - db: Session, - organization_id: UUID, - credential_id: Optional[UUID] = None, - ) -> Optional[str]: - """Resolve and decrypt API key from AIProvider or Integration tables. - - Checks AIProvider first (LLM-style providers like OpenAI), then falls - back to the Integration table (voice platforms like Deepgram, - ElevenLabs). When ``credential_id`` is given the explicit row is - preferred in either table; otherwise the row marked ``is_default`` - wins, with a back-compat fallback to the most recent active row. - """ - from app.services.ai.llm_gateway import ( - resolve_litellm_api_key, - routing_context_from_ai_provider, - ) - - ai_provider = self._get_ai_provider( - provider, db, organization_id, credential_id=credential_id - ) - if ai_provider: - credential_ctx = routing_context_from_ai_provider(ai_provider) - return resolve_litellm_api_key( - organization_id, - db, - ai_provider, - credential=credential_ctx, - ) - - integration = resolve_integration( - provider, db, organization_id, credential_id=credential_id - ) - if integration: - return decrypt_api_key(integration.api_key) - - return None - - def _get_credential_context_for_provider( - self, - provider: ModelProvider, - db: Session, - organization_id: UUID, - credential_id: Optional[UUID] = None, - ): - from app.services.ai.llm_gateway import routing_context_from_ai_provider - - ai_provider = self._get_ai_provider( - provider, db, organization_id, credential_id=credential_id - ) - if ai_provider: - return routing_context_from_ai_provider(ai_provider) - return None - - def _download_audio_to_temp(self, audio_file_key: str, db: Optional[Session] = None) -> str: - """ - Download audio from S3 to temporary file, or use local file if S3 is not available. - """ - import os - - # First, try S3 if enabled - if s3_service.is_enabled(): - try: - # Download from S3 - audio_bytes = s3_service.download_file_by_key(audio_file_key) - - # Determine file extension from key - file_ext = Path(audio_file_key).suffix.lstrip(".") or "wav" - - # Create temporary file - with tempfile.NamedTemporaryFile(delete=False, suffix=f".{file_ext}") as temp_file: - temp_file.write(audio_bytes) - return temp_file.name - except Exception as e: - # If S3 download fails, fall through to local file check - logger.warning(f"S3 download failed for {audio_file_key}: {e}, trying local file...") - - # Fallback: Try to find local file - # Check if it's already a local file path - if os.path.exists(audio_file_key): - # It's a local file path, return it directly - return audio_file_key - - # Try to look up in database if db session is provided - if db: - try: - from app.models.database import AudioFile - # Try to find by S3 key or file path - audio_file = db.query(AudioFile).filter( - (AudioFile.file_path == audio_file_key) | (AudioFile.file_path.like(f"%{audio_file_key}%")) - ).first() - - if audio_file and os.path.exists(audio_file.file_path): - return audio_file.file_path - except Exception as e: - logger.warning(f"Database lookup failed for {audio_file_key}: {e}") - - # If all else fails, raise error - raise StorageError(f"Failed to download audio file: Cloud blob storage is not enabled and local file not found for key: {audio_file_key}") - - # Provider-specific transcription is delegated to app.services.ai.stt_clients - - @staticmethod - def _transcribe_with_whisper_local(audio_file_path: str, model_name: str = "base") -> Dict[str, Any]: - """Transcribe audio using local Whisper model (not a remote API).""" - try: - import whisper - - model = whisper.load_model(model_name) - result = model.transcribe(audio_file_path) - - return { - "text": result.get("text", ""), - "language": result.get("language", "en"), - "segments": [ - {"start": seg.get("start", 0), "end": seg.get("end", 0), "text": seg.get("text", "")} - for seg in result.get("segments", []) - ], - } - except ImportError: - raise RuntimeError("Whisper library not installed. Install with: pip install openai-whisper") - except Exception as e: - raise RuntimeError(f"Whisper transcription failed: {str(e)}") - - def _get_pyannote_pipeline(self): - """Load and cache the pyannote diarization pipeline.""" - if self._pyannote_pipeline is not None: - return self._pyannote_pipeline - - # Compatibility shim: list_audio_backends was removed in torchaudio 2.4+ - import torchaudio - if not hasattr(torchaudio, "list_audio_backends"): - torchaudio.list_audio_backends = lambda: ["ffmpeg"] - - from pyannote.audio import Pipeline - from app.config import settings - - hf_token = settings.HUGGINGFACE_TOKEN - if not hf_token: - raise RuntimeError( - "HUGGINGFACE_TOKEN not configured. Set it under 'diarization.huggingface_token' " - "in config.yml. Required for pyannote speaker diarization." - ) - - logger.info("Loading pyannote speaker-diarization-3.1 pipeline (first call, will be cached)...") - self._pyannote_pipeline = Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", token=hf_token) - logger.info("Pyannote pipeline loaded successfully") - return self._pyannote_pipeline - - def _detect_speakers_with_pyannote( - self, - audio_file_path: str, - segments: List[Dict[str, Any]], - words: Optional[List[Dict[str, Any]]] = None, - num_speakers: Optional[int] = 2, - min_speakers: Optional[int] = None, - max_speakers: Optional[int] = None, - ) -> List[Dict[str, Any]]: - """ - Use pyannote.audio for ML-based speaker diarization, aligned with - Whisper word timestamps for accurate speaker-text mapping. - """ - pipeline = self._get_pyannote_pipeline() - - # Build pipeline kwargs for speaker count hints - pipeline_kwargs = {} - if num_speakers is not None: - pipeline_kwargs["num_speakers"] = num_speakers - if min_speakers is not None: - pipeline_kwargs["min_speakers"] = min_speakers - if max_speakers is not None: - pipeline_kwargs["max_speakers"] = max_speakers - - logger.info( - f"Running pyannote diarization on {audio_file_path} " f"(speaker hints: {pipeline_kwargs or 'auto'})" - ) - raw_output = pipeline(audio_file_path, **pipeline_kwargs) - - # Handle different return types across pyannote versions: - # - Older versions: Annotation directly (has itertracks) - # - Newer versions: DiarizeOutput with .speaker_diarization attribute - if hasattr(raw_output, "itertracks"): - annotation = raw_output - elif hasattr(raw_output, "speaker_diarization"): - annotation = raw_output.speaker_diarization - elif hasattr(raw_output, "annotation"): - annotation = raw_output.annotation - elif isinstance(raw_output, tuple): - annotation = raw_output[0] - else: - attrs = [a for a in dir(raw_output) if not a.startswith("_")] - raise TypeError( - f"Unexpected pyannote output type: {type(raw_output).__name__}. " f"Available attributes: {attrs}" - ) - - # Collect diarization turns as a sorted list for fast lookup - diar_turns = [] - raw_labels = set() - for turn, _, speaker_label in annotation.itertracks(yield_label=True): - diar_turns.append((turn.start, turn.end, speaker_label)) - raw_labels.add(speaker_label) - - if not diar_turns: - logger.warning("Pyannote returned no speaker turns") - return [] - - sorted_labels = sorted(raw_labels) - label_map = {lbl: f"Speaker {i + 1}" for i, lbl in enumerate(sorted_labels)} - logger.info(f"Pyannote detected {len(sorted_labels)} speakers, {len(diar_turns)} turns") - - def find_speaker(midpoint: float) -> str: - """Find which speaker is active at a given timestamp.""" - for t_start, t_end, lbl in diar_turns: - if t_start <= midpoint <= t_end: - return label_map[lbl] - # No exact match -- find the closest turn - min_dist = float("inf") - closest_label = label_map[diar_turns[0][2]] - for t_start, t_end, lbl in diar_turns: - dist = min(abs(midpoint - t_start), abs(midpoint - t_end)) - if dist < min_dist: - min_dist = dist - closest_label = label_map[lbl] - return closest_label - - if words and len(words) > 0: - speaker_segments = [] - current_speaker = None - current_words: List[str] = [] - current_start = 0.0 - current_end = 0.0 - - for w in words: - word_text = w.get("word", "").strip() - w_start = w.get("start", 0) - w_end = w.get("end", 0) - if not word_text: - continue - - midpoint = (w_start + w_end) / 2.0 - speaker = find_speaker(midpoint) - - if speaker != current_speaker: - if current_words and current_speaker: - speaker_segments.append( - { - "speaker": current_speaker, - "text": " ".join(current_words).strip(), - "start": round(current_start, 3), - "end": round(current_end, 3), - } - ) - current_speaker = speaker - current_words = [word_text] - current_start = w_start - current_end = w_end - else: - current_words.append(word_text) - current_end = w_end - - if current_words and current_speaker: - speaker_segments.append( - { - "speaker": current_speaker, - "text": " ".join(current_words).strip(), - "start": round(current_start, 3), - "end": round(current_end, 3), - } - ) - - return speaker_segments - - # Fallback: align at segment level when word timestamps are not available - speaker_segments = [] - for seg in segments: - seg_start = seg.get("start", 0) - seg_end = seg.get("end", 0) - seg_text = seg.get("text", "").strip() - if not seg_text: - continue - - midpoint = (seg_start + seg_end) / 2.0 - speaker = find_speaker(midpoint) - - speaker_segments.append( - { - "speaker": speaker, - "text": seg_text, - "start": round(seg_start, 3), - "end": round(seg_end, 3), - } - ) - - return speaker_segments - - def _detect_speakers_heuristic(self, segments: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Heuristic-based speaker diarization fallback. Uses gap-based detection - which is unreliable -- pyannote.audio should be used for accurate results. - """ - logger.warning( - "Using heuristic speaker diarization (unreliable). For accurate results, " - "install pyannote.audio and configure diarization.huggingface_token in config.yml" - ) - if not segments: - return [] - - speaker_segments = [] - current_speaker = "Speaker 1" - - # Thresholds for speaker change detection - GAP_THRESHOLD_LARGE = 0.8 # seconds - large gap definitely suggests change - GAP_THRESHOLD_MEDIUM = 0.4 # seconds - medium gap suggests change - GAP_THRESHOLD_SMALL = 0.2 # seconds - small gap, but combined with other factors - MIN_SEGMENT_DURATION = 0.2 # seconds - minimum segment to consider - - # Calculate average gap size to adapt thresholds - gaps = [] - for i in range(1, len(segments)): - gap = segments[i].get("start", 0) - segments[i - 1].get("end", 0) - if gap > 0: - gaps.append(gap) - - avg_gap = sum(gaps) / len(gaps) if gaps else 0.5 - # Adaptive threshold based on conversation pace - adaptive_threshold = min(max(avg_gap * 1.5, 0.3), 1.0) - - # Track recent speaker assignments for pattern detection - recent_assignments = [] - assignment_window = 5 # Look at last N segments for patterns - - for i, seg in enumerate(segments): - seg_start = seg.get("start", 0) - seg_end = seg.get("end", 0) - seg_duration = seg_end - seg_start - seg_text = seg.get("text", "").strip() - - # Skip very short segments (likely noise or artifacts) - if seg_duration < MIN_SEGMENT_DURATION or not seg_text: - continue - - # Check for speaker change indicators - should_switch = False - gap = 0 - - if i > 0: - prev_seg = segments[i - 1] - gap = seg_start - prev_seg.get("end", 0) - - # Large gap definitely suggests speaker change - if gap > GAP_THRESHOLD_LARGE: - should_switch = True - # Medium gap suggests change - elif gap > GAP_THRESHOLD_MEDIUM: - should_switch = True - # Small gap but check for alternating pattern - elif gap > GAP_THRESHOLD_SMALL: - # If we've been alternating, continue the pattern - if len(recent_assignments) >= 2: - # Check if last two were the same speaker (suggests we should switch) - if recent_assignments[-1] == recent_assignments[-2] == current_speaker: - should_switch = True - # Or if we see a pattern of quick back-and-forth - elif len(recent_assignments) >= 3: - # If pattern is A-A-A, switch to B - if all(a == current_speaker for a in recent_assignments[-3:]): - should_switch = True - # Very small or no gap - use alternating pattern if established - elif gap >= 0: - # If we have an established alternating pattern, continue it - if len(recent_assignments) >= 2: - # Check if we should alternate based on recent pattern - if recent_assignments[-1] == current_speaker: - # If last segment was same speaker, consider switching - # But only if we have a pattern suggesting alternation - if len(recent_assignments) >= 4: - # Check for A-B-A-B pattern - pattern = recent_assignments[-4:] - if pattern[0] != pattern[1] and pattern[1] != pattern[2] and pattern[2] != pattern[3]: - # We have alternating pattern, continue it - should_switch = True - - # Additional heuristics for first few segments - if i < 3 and i > 0: - # Early in conversation, be more aggressive about switching - if gap > 0.1: # Any noticeable gap - should_switch = True - - # If we have multiple consecutive segments from same speaker, force alternation - if len(recent_assignments) >= 2: - # If last 2 segments were both the same speaker (and it's the current speaker), switch - # This prevents one speaker from getting too many consecutive segments - if recent_assignments[-1] == current_speaker and recent_assignments[-2] == current_speaker: - should_switch = True - - # Switch speaker if needed - if should_switch: - current_speaker = "Speaker 2" if current_speaker == "Speaker 1" else "Speaker 1" - - speaker_segments.append( - { - "speaker": current_speaker, - "text": seg_text, - "start": seg_start, - "end": seg_end, - } - ) - - # Track recent assignments for pattern detection - recent_assignments.append(current_speaker) - if len(recent_assignments) > assignment_window: - recent_assignments.pop(0) - - # Post-process: Balance speakers if one dominates too much - speaker_1_count = sum(1 for s in speaker_segments if s["speaker"] == "Speaker 1") - speaker_2_count = sum(1 for s in speaker_segments if s["speaker"] == "Speaker 2") - total_segments = len(speaker_segments) - - # If one speaker has more than 70% of segments, redistribute by alternating - if total_segments > 0: - speaker_1_ratio = speaker_1_count / total_segments - if speaker_1_ratio > 0.7: - # Redistribute: alternate segments starting from index 1 - # This assumes the first speaker is correct, then alternates - for i in range(1, len(speaker_segments)): - expected_speaker = "Speaker 2" if i % 2 == 1 else "Speaker 1" - if speaker_segments[i]["speaker"] != expected_speaker: - speaker_segments[i]["speaker"] = expected_speaker - elif speaker_1_ratio < 0.3: - # Speaker 2 dominates, redistribute - for i in range(1, len(speaker_segments)): - expected_speaker = "Speaker 1" if i % 2 == 1 else "Speaker 2" - if speaker_segments[i]["speaker"] != expected_speaker: - speaker_segments[i]["speaker"] = expected_speaker - - return speaker_segments - - def transcribe_text_only( - self, - audio_file_path: str, - stt_provider: ModelProvider, - stt_model: str, - organization_id: UUID, - db: Session, - language: Optional[str] = None, - credential_id: Optional[UUID] = None, - ) -> Optional[str]: - """Transcribe a local audio file and return just the text. - - Lightweight alternative to `transcribe()` -- skips S3 download, - diarization, and segment extraction. Designed for WER/CER - evaluation where only the transcript string is needed. - """ - api_key = self._get_api_key_for_provider( - stt_provider, db, organization_id, credential_id=credential_id - ) - if not api_key and stt_provider != ModelProvider.GOOGLE: - logger.warning( - f"[TranscriptionService] No API key found for {stt_provider} " - f"(checked AIProvider and Integration tables) for org {organization_id}" - ) - return None - - credential_ctx = ( - self._get_credential_context_for_provider( - stt_provider, db, organization_id, credential_id=credential_id - ) - if stt_provider == ModelProvider.GOOGLE - else None - ) - - from app.services.ai.stt_clients import ( - transcribe_openai, - transcribe_deepgram, - transcribe_elevenlabs, - transcribe_google, - transcribe_sarvam, - transcribe_smallest, - ) - - try: - if stt_provider == ModelProvider.OPENAI: - result = transcribe_openai(audio_file_path, stt_model, api_key, language) - elif stt_provider == ModelProvider.DEEPGRAM: - result = transcribe_deepgram(audio_file_path, stt_model, api_key, language) - elif stt_provider == ModelProvider.ELEVENLABS: - result = transcribe_elevenlabs(audio_file_path, stt_model, api_key, language) - elif stt_provider == ModelProvider.GOOGLE: - result = transcribe_google( - audio_file_path, stt_model, api_key, language, - organization_id=organization_id, db=db, - credential=credential_ctx, - ) - elif stt_provider == ModelProvider.SARVAM: - result = transcribe_sarvam(audio_file_path, stt_model, api_key, language) - elif stt_provider == ModelProvider.SMALLEST: - result = transcribe_smallest(audio_file_path, stt_model, api_key, language) - else: - logger.warning(f"[TranscriptionService] Unsupported STT provider for text-only: {stt_provider}") - return None - - text = (result.get("text") or "").strip() - return text or None - except Exception as e: - logger.error(f"[TranscriptionService] text-only transcription failed ({stt_provider}/{stt_model}): {e}") - return None - - def transcribe( - self, - audio_file_key: str, - stt_provider: ModelProvider, - stt_model: str, - organization_id: UUID, - db: Session, - language: Optional[str] = None, - enable_speaker_diarization: bool = True, - credential_id: Optional[UUID] = None, - ) -> Dict[str, Any]: - """ - Transcribe audio file from S3. - """ - start_time = time.time() - temp_file_path = None - - try: - # Download audio to temporary file - temp_file_path = self._download_audio_to_temp(audio_file_key, db=db) - - api_key = self._get_api_key_for_provider( - stt_provider, db, organization_id, credential_id=credential_id - ) - credential_ctx = self._get_credential_context_for_provider( - stt_provider, db, organization_id, credential_id=credential_id - ) - if not api_key and stt_provider != ModelProvider.GOOGLE: - raise RuntimeError( - f"No API key found for {stt_provider} (checked AIProvider and Integration tables). " - f"Please configure the provider in Settings." - ) - - from app.services.ai.stt_clients import ( - transcribe_openai, - transcribe_deepgram, - transcribe_elevenlabs, - transcribe_google, - transcribe_sarvam, - transcribe_smallest, - ) - - if stt_provider == ModelProvider.OPENAI: - # All OpenAI STT models in app/config/models.json - # (``whisper-1``, ``gpt-4o-transcribe``, - # ``gpt-4o-mini-transcribe``) are hosted-API models, so - # always go through the OpenAI client. ``transcribe_openai`` - # handles the per-model differences in ``response_format`` - # / ``timestamp_granularities`` internally. Local Whisper - # is reserved for the unknown-provider fallback below. - result = transcribe_openai(temp_file_path, stt_model, api_key, language) - elif stt_provider == ModelProvider.DEEPGRAM: - result = transcribe_deepgram(temp_file_path, stt_model, api_key, language) - elif stt_provider == ModelProvider.ELEVENLABS: - result = transcribe_elevenlabs(temp_file_path, stt_model, api_key, language) - elif stt_provider == ModelProvider.SARVAM: - result = transcribe_sarvam(temp_file_path, stt_model, api_key, language) - elif stt_provider == ModelProvider.SMALLEST: - result = transcribe_smallest(temp_file_path, stt_model, api_key, language) - elif stt_provider == ModelProvider.GOOGLE: - # Gemini STT models (``gemini-2.5-pro-stt``, - # ``gemini-2.5-flash-stt``, ``gemini-2.5-flash-lite-stt``) - # are routed through LiteLLM as multimodal completions. - # Legacy ``google-speech-v2`` would also land here, but - # we don't yet have a Cloud Speech client wired up. - result = transcribe_google( - temp_file_path, stt_model, api_key, language, - organization_id=organization_id, db=db, - credential=credential_ctx, - ) - elif stt_provider == ModelProvider.AZURE: - raise NotImplementedError("Azure Speech Services not yet implemented") - elif stt_provider == ModelProvider.AWS: - raise NotImplementedError("AWS Transcribe not yet implemented") - else: - result = self._transcribe_with_whisper_local(temp_file_path, "base") - - # Apply speaker diarization if enabled - speaker_segments = None - if enable_speaker_diarization: - segments = result.get("segments", []) - - # If no segments from transcription, create a single segment from the full text. - # The local audio file is bound to ``temp_file_path`` (downloaded - # from S3 a few lines above); the previous name ``audio_file_path`` - # was a typo that NameError'd as soon as a provider that doesn't - # surface segments (e.g. Gemini STT) was used with diarisation - # enabled. - if not segments and result.get("text"): - estimated_duration = 0.0 - if hasattr(result, "duration") and result.get("duration"): - estimated_duration = result.get("duration") - elif temp_file_path and os.path.exists(temp_file_path): - try: - import librosa - duration = librosa.get_duration(path=temp_file_path) - estimated_duration = duration - except Exception: - pass - - segments = [ - { - "start": 0.0, - "end": estimated_duration if estimated_duration > 0 else 10.0, # Default to 10s if unknown - "text": result.get("text", ""), - } - ] - - if segments: - words = result.get("words", []) - # Check if words actually have valid timestamps - valid_word_count = sum(1 for w in words if w.get("start", 0) > 0 or w.get("end", 0) > 0) - if words and valid_word_count == 0: - words = [] - - try: - from app.config import settings as app_settings - num_spk = getattr(app_settings, "DIARIZATION_NUM_SPEAKERS", 2) - speaker_segments = self._detect_speakers_with_pyannote( - temp_file_path, segments, words, num_speakers=num_spk - ) - if not speaker_segments: - speaker_segments = self._detect_speakers_heuristic(segments) - except Exception as e: - logger.warning(f"Pyannote diarization failed: {e}, falling back to heuristic") - speaker_segments = self._detect_speakers_heuristic(segments) - - processing_time = time.time() - start_time - - return { - "transcript": result["text"], - "language": result.get("language", language), - "speaker_segments": speaker_segments, - "segments": result.get("segments", []), - "processing_time": processing_time, - "raw_output": result, - } - - finally: - # Clean up temporary file - if temp_file_path and os.path.exists(temp_file_path): - try: - os.unlink(temp_file_path) - except Exception: - pass - - -# Singleton instance -transcription_service = TranscriptionService() +""" +Transcription service for converting audio to text using various STT providers. + +Response Format: +All providers return a standardized format: +{ + "text": str, # Full transcript text + "language": str, # Language code (e.g., "en", "es") + "segments": [ # List of segments with timestamps + { + "start": float, # Start time in seconds + "end": float, # End time in seconds + "text": str # Text for this segment + } + ], + "speaker_segments": [ # Segments with speaker labels (if diarization enabled) + { + "speaker": str, # Speaker label (e.g., "Speaker 1") + "text": str, + "start": float, + "end": float + } + ], + "processing_time": float, + "raw_output": dict # Original provider response +} + +Provider-Specific Formats: +- OpenAI Whisper API: Uses verbose_json format which includes segments with timestamps +- Local Whisper: Returns segments by default with word-level timestamps available +- Google/Azure/AWS: Formats documented but not yet implemented + +Speaker Diarization: +- Whisper does NOT provide speaker diarization natively (only transcription) +- We use pyannote.audio (speaker-diarization-3.1) for accurate ML-based diarization +- Whisper provides word-level timestamps, pyannote identifies speakers, then we align +- Requires: pyannote.audio installed + diarization.huggingface_token in config.yml +- Falls back to unreliable gap-based heuristics if pyannote is unavailable +""" + +import time +import tempfile +import os +import logging +from typing import Optional, Dict, Any, List +from uuid import UUID +from pathlib import Path + +from app.models.database import ModelProvider, AIProvider, Integration +from app.core.encryption import decrypt_api_key +from app.services.credentials import resolve_ai_provider, resolve_integration +from app.services.storage.s3_service import s3_service +from app.core.exceptions import StorageError +from sqlalchemy.orm import Session + +logger = logging.getLogger(__name__) + + +class TranscriptionService: + """Service for transcribing audio files using various STT providers.""" + + def __init__(self): + """Initialize transcription service.""" + self._pyannote_pipeline = None + + def _get_ai_provider( + self, + provider: ModelProvider, + db: Session, + organization_id: UUID, + credential_id: Optional[UUID] = None, + ) -> Optional[AIProvider]: + """Resolve the AIProvider row to use, honoring ``credential_id``.""" + return resolve_ai_provider( + provider, db, organization_id, credential_id=credential_id + ) + + def _get_api_key_for_provider( + self, + provider: ModelProvider, + db: Session, + organization_id: UUID, + credential_id: Optional[UUID] = None, + ) -> Optional[str]: + """Resolve and decrypt API key from AIProvider or Integration tables. + + Checks AIProvider first (LLM-style providers like OpenAI), then falls + back to the Integration table (voice platforms like Deepgram, + ElevenLabs). When ``credential_id`` is given the explicit row is + preferred in either table; otherwise the row marked ``is_default`` + wins, with a back-compat fallback to the most recent active row. + """ + from app.services.ai.llm_gateway import ( + resolve_litellm_api_key, + routing_context_from_ai_provider, + ) + + ai_provider = self._get_ai_provider( + provider, db, organization_id, credential_id=credential_id + ) + if ai_provider: + credential_ctx = routing_context_from_ai_provider(ai_provider) + return resolve_litellm_api_key( + organization_id, + db, + ai_provider, + credential=credential_ctx, + ) + + integration = resolve_integration( + provider, db, organization_id, credential_id=credential_id + ) + if integration: + return decrypt_api_key(integration.api_key) + + return None + + def _get_credential_context_for_provider( + self, + provider: ModelProvider, + db: Session, + organization_id: UUID, + credential_id: Optional[UUID] = None, + ): + from app.services.ai.llm_gateway import routing_context_from_ai_provider + + ai_provider = self._get_ai_provider( + provider, db, organization_id, credential_id=credential_id + ) + if ai_provider: + return routing_context_from_ai_provider(ai_provider) + return None + + def _download_audio_to_temp(self, audio_file_key: str, db: Optional[Session] = None) -> str: + """ + Download audio from S3 to temporary file, or use local file if S3 is not available. + """ + import os + + # First, try S3 if enabled + if s3_service.is_enabled(): + try: + # Download from S3 + audio_bytes = s3_service.download_file_by_key(audio_file_key) + + # Determine file extension from key + file_ext = Path(audio_file_key).suffix.lstrip(".") or "wav" + + # Create temporary file + with tempfile.NamedTemporaryFile(delete=False, suffix=f".{file_ext}") as temp_file: + temp_file.write(audio_bytes) + return temp_file.name + except Exception as e: + # If S3 download fails, fall through to local file check + logger.warning(f"S3 download failed for {audio_file_key}: {e}, trying local file...") + + # Fallback: Try to find local file + # Check if it's already a local file path + if os.path.exists(audio_file_key): + # It's a local file path, return it directly + return audio_file_key + + # Try to look up in database if db session is provided + if db: + try: + from app.models.database import AudioFile + # Try to find by S3 key or file path + audio_file = db.query(AudioFile).filter( + (AudioFile.file_path == audio_file_key) | (AudioFile.file_path.like(f"%{audio_file_key}%")) + ).first() + + if audio_file and os.path.exists(audio_file.file_path): + return audio_file.file_path + except Exception as e: + logger.warning(f"Database lookup failed for {audio_file_key}: {e}") + + # If all else fails, raise error + raise StorageError(f"Failed to download audio file: Cloud blob storage is not enabled and local file not found for key: {audio_file_key}") + + # Provider-specific transcription is delegated to app.services.ai.stt_clients + + @staticmethod + def _transcribe_with_whisper_local(audio_file_path: str, model_name: str = "base") -> Dict[str, Any]: + """Transcribe audio using local Whisper model (not a remote API).""" + try: + import whisper + + model = whisper.load_model(model_name) + result = model.transcribe(audio_file_path) + + return { + "text": result.get("text", ""), + "language": result.get("language", "en"), + "segments": [ + {"start": seg.get("start", 0), "end": seg.get("end", 0), "text": seg.get("text", "")} + for seg in result.get("segments", []) + ], + } + except ImportError: + raise RuntimeError("Whisper library not installed. Install with: pip install openai-whisper") + except Exception as e: + raise RuntimeError(f"Whisper transcription failed: {str(e)}") + + def _get_pyannote_pipeline(self): + """Load and cache the pyannote diarization pipeline.""" + if self._pyannote_pipeline is not None: + return self._pyannote_pipeline + + # Compatibility shim: list_audio_backends was removed in torchaudio 2.4+ + import torchaudio + if not hasattr(torchaudio, "list_audio_backends"): + torchaudio.list_audio_backends = lambda: ["ffmpeg"] + + from pyannote.audio import Pipeline + from app.config import settings + + hf_token = settings.HUGGINGFACE_TOKEN + if not hf_token: + raise RuntimeError( + "HUGGINGFACE_TOKEN not configured. Set it under 'diarization.huggingface_token' " + "in config.yml. Required for pyannote speaker diarization." + ) + + logger.info("Loading pyannote speaker-diarization-3.1 pipeline (first call, will be cached)...") + self._pyannote_pipeline = Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", token=hf_token) + logger.info("Pyannote pipeline loaded successfully") + return self._pyannote_pipeline + + def _detect_speakers_with_pyannote( + self, + audio_file_path: str, + segments: List[Dict[str, Any]], + words: Optional[List[Dict[str, Any]]] = None, + num_speakers: Optional[int] = 2, + min_speakers: Optional[int] = None, + max_speakers: Optional[int] = None, + ) -> List[Dict[str, Any]]: + """ + Use pyannote.audio for ML-based speaker diarization, aligned with + Whisper word timestamps for accurate speaker-text mapping. + """ + pipeline = self._get_pyannote_pipeline() + + # Build pipeline kwargs for speaker count hints + pipeline_kwargs = {} + if num_speakers is not None: + pipeline_kwargs["num_speakers"] = num_speakers + if min_speakers is not None: + pipeline_kwargs["min_speakers"] = min_speakers + if max_speakers is not None: + pipeline_kwargs["max_speakers"] = max_speakers + + logger.info( + f"Running pyannote diarization on {audio_file_path} " f"(speaker hints: {pipeline_kwargs or 'auto'})" + ) + raw_output = pipeline(audio_file_path, **pipeline_kwargs) + + # Handle different return types across pyannote versions: + # - Older versions: Annotation directly (has itertracks) + # - Newer versions: DiarizeOutput with .speaker_diarization attribute + if hasattr(raw_output, "itertracks"): + annotation = raw_output + elif hasattr(raw_output, "speaker_diarization"): + annotation = raw_output.speaker_diarization + elif hasattr(raw_output, "annotation"): + annotation = raw_output.annotation + elif isinstance(raw_output, tuple): + annotation = raw_output[0] + else: + attrs = [a for a in dir(raw_output) if not a.startswith("_")] + raise TypeError( + f"Unexpected pyannote output type: {type(raw_output).__name__}. " f"Available attributes: {attrs}" + ) + + # Collect diarization turns as a sorted list for fast lookup + diar_turns = [] + raw_labels = set() + for turn, _, speaker_label in annotation.itertracks(yield_label=True): + diar_turns.append((turn.start, turn.end, speaker_label)) + raw_labels.add(speaker_label) + + if not diar_turns: + logger.warning("Pyannote returned no speaker turns") + return [] + + sorted_labels = sorted(raw_labels) + label_map = {lbl: f"Speaker {i + 1}" for i, lbl in enumerate(sorted_labels)} + logger.info(f"Pyannote detected {len(sorted_labels)} speakers, {len(diar_turns)} turns") + + def find_speaker(midpoint: float) -> str: + """Find which speaker is active at a given timestamp.""" + for t_start, t_end, lbl in diar_turns: + if t_start <= midpoint <= t_end: + return label_map[lbl] + # No exact match -- find the closest turn + min_dist = float("inf") + closest_label = label_map[diar_turns[0][2]] + for t_start, t_end, lbl in diar_turns: + dist = min(abs(midpoint - t_start), abs(midpoint - t_end)) + if dist < min_dist: + min_dist = dist + closest_label = label_map[lbl] + return closest_label + + if words and len(words) > 0: + speaker_segments = [] + current_speaker = None + current_words: List[str] = [] + current_start = 0.0 + current_end = 0.0 + + for w in words: + word_text = w.get("word", "").strip() + w_start = w.get("start", 0) + w_end = w.get("end", 0) + if not word_text: + continue + + midpoint = (w_start + w_end) / 2.0 + speaker = find_speaker(midpoint) + + if speaker != current_speaker: + if current_words and current_speaker: + speaker_segments.append( + { + "speaker": current_speaker, + "text": " ".join(current_words).strip(), + "start": round(current_start, 3), + "end": round(current_end, 3), + } + ) + current_speaker = speaker + current_words = [word_text] + current_start = w_start + current_end = w_end + else: + current_words.append(word_text) + current_end = w_end + + if current_words and current_speaker: + speaker_segments.append( + { + "speaker": current_speaker, + "text": " ".join(current_words).strip(), + "start": round(current_start, 3), + "end": round(current_end, 3), + } + ) + + return speaker_segments + + # Fallback: align at segment level when word timestamps are not available + speaker_segments = [] + for seg in segments: + seg_start = seg.get("start", 0) + seg_end = seg.get("end", 0) + seg_text = seg.get("text", "").strip() + if not seg_text: + continue + + midpoint = (seg_start + seg_end) / 2.0 + speaker = find_speaker(midpoint) + + speaker_segments.append( + { + "speaker": speaker, + "text": seg_text, + "start": round(seg_start, 3), + "end": round(seg_end, 3), + } + ) + + return speaker_segments + + def _detect_speakers_heuristic(self, segments: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Heuristic-based speaker diarization fallback. Uses gap-based detection + which is unreliable -- pyannote.audio should be used for accurate results. + """ + logger.warning( + "Using heuristic speaker diarization (unreliable). For accurate results, " + "install pyannote.audio and configure diarization.huggingface_token in config.yml" + ) + if not segments: + return [] + + speaker_segments = [] + current_speaker = "Speaker 1" + + # Thresholds for speaker change detection + GAP_THRESHOLD_LARGE = 0.8 # seconds - large gap definitely suggests change + GAP_THRESHOLD_MEDIUM = 0.4 # seconds - medium gap suggests change + GAP_THRESHOLD_SMALL = 0.2 # seconds - small gap, but combined with other factors + MIN_SEGMENT_DURATION = 0.2 # seconds - minimum segment to consider + + # Calculate average gap size to adapt thresholds + gaps = [] + for i in range(1, len(segments)): + gap = segments[i].get("start", 0) - segments[i - 1].get("end", 0) + if gap > 0: + gaps.append(gap) + + avg_gap = sum(gaps) / len(gaps) if gaps else 0.5 + # Adaptive threshold based on conversation pace + adaptive_threshold = min(max(avg_gap * 1.5, 0.3), 1.0) + + # Track recent speaker assignments for pattern detection + recent_assignments = [] + assignment_window = 5 # Look at last N segments for patterns + + for i, seg in enumerate(segments): + seg_start = seg.get("start", 0) + seg_end = seg.get("end", 0) + seg_duration = seg_end - seg_start + seg_text = seg.get("text", "").strip() + + # Skip very short segments (likely noise or artifacts) + if seg_duration < MIN_SEGMENT_DURATION or not seg_text: + continue + + # Check for speaker change indicators + should_switch = False + gap = 0 + + if i > 0: + prev_seg = segments[i - 1] + gap = seg_start - prev_seg.get("end", 0) + + # Large gap definitely suggests speaker change + if gap > GAP_THRESHOLD_LARGE: + should_switch = True + # Medium gap suggests change + elif gap > GAP_THRESHOLD_MEDIUM: + should_switch = True + # Small gap but check for alternating pattern + elif gap > GAP_THRESHOLD_SMALL: + # If we've been alternating, continue the pattern + if len(recent_assignments) >= 2: + # Check if last two were the same speaker (suggests we should switch) + if recent_assignments[-1] == recent_assignments[-2] == current_speaker: + should_switch = True + # Or if we see a pattern of quick back-and-forth + elif len(recent_assignments) >= 3: + # If pattern is A-A-A, switch to B + if all(a == current_speaker for a in recent_assignments[-3:]): + should_switch = True + # Very small or no gap - use alternating pattern if established + elif gap >= 0: + # If we have an established alternating pattern, continue it + if len(recent_assignments) >= 2: + # Check if we should alternate based on recent pattern + if recent_assignments[-1] == current_speaker: + # If last segment was same speaker, consider switching + # But only if we have a pattern suggesting alternation + if len(recent_assignments) >= 4: + # Check for A-B-A-B pattern + pattern = recent_assignments[-4:] + if pattern[0] != pattern[1] and pattern[1] != pattern[2] and pattern[2] != pattern[3]: + # We have alternating pattern, continue it + should_switch = True + + # Additional heuristics for first few segments + if i < 3 and i > 0: + # Early in conversation, be more aggressive about switching + if gap > 0.1: # Any noticeable gap + should_switch = True + + # If we have multiple consecutive segments from same speaker, force alternation + if len(recent_assignments) >= 2: + # If last 2 segments were both the same speaker (and it's the current speaker), switch + # This prevents one speaker from getting too many consecutive segments + if recent_assignments[-1] == current_speaker and recent_assignments[-2] == current_speaker: + should_switch = True + + # Switch speaker if needed + if should_switch: + current_speaker = "Speaker 2" if current_speaker == "Speaker 1" else "Speaker 1" + + speaker_segments.append( + { + "speaker": current_speaker, + "text": seg_text, + "start": seg_start, + "end": seg_end, + } + ) + + # Track recent assignments for pattern detection + recent_assignments.append(current_speaker) + if len(recent_assignments) > assignment_window: + recent_assignments.pop(0) + + # Post-process: Balance speakers if one dominates too much + speaker_1_count = sum(1 for s in speaker_segments if s["speaker"] == "Speaker 1") + speaker_2_count = sum(1 for s in speaker_segments if s["speaker"] == "Speaker 2") + total_segments = len(speaker_segments) + + # If one speaker has more than 70% of segments, redistribute by alternating + if total_segments > 0: + speaker_1_ratio = speaker_1_count / total_segments + if speaker_1_ratio > 0.7: + # Redistribute: alternate segments starting from index 1 + # This assumes the first speaker is correct, then alternates + for i in range(1, len(speaker_segments)): + expected_speaker = "Speaker 2" if i % 2 == 1 else "Speaker 1" + if speaker_segments[i]["speaker"] != expected_speaker: + speaker_segments[i]["speaker"] = expected_speaker + elif speaker_1_ratio < 0.3: + # Speaker 2 dominates, redistribute + for i in range(1, len(speaker_segments)): + expected_speaker = "Speaker 1" if i % 2 == 1 else "Speaker 2" + if speaker_segments[i]["speaker"] != expected_speaker: + speaker_segments[i]["speaker"] = expected_speaker + + return speaker_segments + + def transcribe_text_only( + self, + audio_file_path: str, + stt_provider: ModelProvider, + stt_model: str, + organization_id: UUID, + db: Session, + language: Optional[str] = None, + credential_id: Optional[UUID] = None, + ) -> Optional[str]: + """Transcribe a local audio file and return just the text. + + Lightweight alternative to `transcribe()` -- skips S3 download, + diarization, and segment extraction. Designed for WER/CER + evaluation where only the transcript string is needed. + """ + api_key = self._get_api_key_for_provider( + stt_provider, db, organization_id, credential_id=credential_id + ) + if not api_key and stt_provider != ModelProvider.GOOGLE: + logger.warning( + f"[TranscriptionService] No API key found for {stt_provider} " + f"(checked AIProvider and Integration tables) for org {organization_id}" + ) + return None + + credential_ctx = ( + self._get_credential_context_for_provider( + stt_provider, db, organization_id, credential_id=credential_id + ) + if stt_provider == ModelProvider.GOOGLE + else None + ) + + from app.services.ai.stt_clients import ( + transcribe_openai, + transcribe_deepgram, + transcribe_elevenlabs, + transcribe_google, + transcribe_sarvam, + transcribe_smallest, + ) + + try: + if stt_provider == ModelProvider.OPENAI: + result = transcribe_openai(audio_file_path, stt_model, api_key, language) + elif stt_provider == ModelProvider.DEEPGRAM: + result = transcribe_deepgram(audio_file_path, stt_model, api_key, language) + elif stt_provider == ModelProvider.ELEVENLABS: + result = transcribe_elevenlabs(audio_file_path, stt_model, api_key, language) + elif stt_provider == ModelProvider.GOOGLE: + result = transcribe_google( + audio_file_path, stt_model, api_key, language, + organization_id=organization_id, db=db, + credential=credential_ctx, + ) + elif stt_provider == ModelProvider.SARVAM: + result = transcribe_sarvam(audio_file_path, stt_model, api_key, language) + elif stt_provider == ModelProvider.SMALLEST: + result = transcribe_smallest(audio_file_path, stt_model, api_key, language) + else: + logger.warning(f"[TranscriptionService] Unsupported STT provider for text-only: {stt_provider}") + return None + + text = (result.get("text") or "").strip() + return text or None + except Exception as e: + logger.error(f"[TranscriptionService] text-only transcription failed ({stt_provider}/{stt_model}): {e}") + return None + + def transcribe( + self, + audio_file_key: str, + stt_provider: ModelProvider, + stt_model: str, + organization_id: UUID, + db: Session, + language: Optional[str] = None, + enable_speaker_diarization: bool = True, + credential_id: Optional[UUID] = None, + ) -> Dict[str, Any]: + """ + Transcribe audio file from S3. + """ + start_time = time.time() + temp_file_path = None + + try: + # Download audio to temporary file + temp_file_path = self._download_audio_to_temp(audio_file_key, db=db) + + api_key = self._get_api_key_for_provider( + stt_provider, db, organization_id, credential_id=credential_id + ) + credential_ctx = self._get_credential_context_for_provider( + stt_provider, db, organization_id, credential_id=credential_id + ) + if not api_key and stt_provider != ModelProvider.GOOGLE: + raise RuntimeError( + f"No API key found for {stt_provider} (checked AIProvider and Integration tables). " + f"Please configure the provider in Settings." + ) + + from app.services.ai.stt_clients import ( + transcribe_openai, + transcribe_deepgram, + transcribe_elevenlabs, + transcribe_google, + transcribe_sarvam, + transcribe_smallest, + ) + + if stt_provider == ModelProvider.OPENAI: + # All OpenAI STT models in app/config/models.json + # (``whisper-1``, ``gpt-4o-transcribe``, + # ``gpt-4o-mini-transcribe``) are hosted-API models, so + # always go through the OpenAI client. ``transcribe_openai`` + # handles the per-model differences in ``response_format`` + # / ``timestamp_granularities`` internally. Local Whisper + # is reserved for the unknown-provider fallback below. + result = transcribe_openai(temp_file_path, stt_model, api_key, language) + elif stt_provider == ModelProvider.DEEPGRAM: + result = transcribe_deepgram(temp_file_path, stt_model, api_key, language) + elif stt_provider == ModelProvider.ELEVENLABS: + result = transcribe_elevenlabs(temp_file_path, stt_model, api_key, language) + elif stt_provider == ModelProvider.SARVAM: + result = transcribe_sarvam(temp_file_path, stt_model, api_key, language) + elif stt_provider == ModelProvider.SMALLEST: + result = transcribe_smallest(temp_file_path, stt_model, api_key, language) + elif stt_provider == ModelProvider.GOOGLE: + # Gemini STT models (``gemini-2.5-pro-stt``, + # ``gemini-2.5-flash-stt``, ``gemini-2.5-flash-lite-stt``) + # are routed through LiteLLM as multimodal completions. + # Legacy ``google-speech-v2`` would also land here, but + # we don't yet have a Cloud Speech client wired up. + result = transcribe_google( + temp_file_path, stt_model, api_key, language, + organization_id=organization_id, db=db, + credential=credential_ctx, + ) + elif stt_provider == ModelProvider.AZURE: + raise NotImplementedError("Azure Speech Services not yet implemented") + elif stt_provider == ModelProvider.AWS: + raise NotImplementedError("AWS Transcribe not yet implemented") + else: + result = self._transcribe_with_whisper_local(temp_file_path, "base") + + # Apply speaker diarization if enabled + speaker_segments = None + if enable_speaker_diarization: + segments = result.get("segments", []) + + # If no segments from transcription, create a single segment from the full text. + # The local audio file is bound to ``temp_file_path`` (downloaded + # from S3 a few lines above); the previous name ``audio_file_path`` + # was a typo that NameError'd as soon as a provider that doesn't + # surface segments (e.g. Gemini STT) was used with diarisation + # enabled. + if not segments and result.get("text"): + estimated_duration = 0.0 + if hasattr(result, "duration") and result.get("duration"): + estimated_duration = result.get("duration") + elif temp_file_path and os.path.exists(temp_file_path): + try: + import librosa + duration = librosa.get_duration(path=temp_file_path) + estimated_duration = duration + except Exception: + pass + + segments = [ + { + "start": 0.0, + "end": estimated_duration if estimated_duration > 0 else 10.0, # Default to 10s if unknown + "text": result.get("text", ""), + } + ] + + if segments: + words = result.get("words", []) + # Check if words actually have valid timestamps + valid_word_count = sum(1 for w in words if w.get("start", 0) > 0 or w.get("end", 0) > 0) + if words and valid_word_count == 0: + words = [] + + try: + from app.config import settings as app_settings + num_spk = getattr(app_settings, "DIARIZATION_NUM_SPEAKERS", 2) + speaker_segments = self._detect_speakers_with_pyannote( + temp_file_path, segments, words, num_speakers=num_spk + ) + if not speaker_segments: + speaker_segments = self._detect_speakers_heuristic(segments) + except Exception as e: + logger.warning(f"Pyannote diarization failed: {e}, falling back to heuristic") + speaker_segments = self._detect_speakers_heuristic(segments) + + processing_time = time.time() - start_time + + return { + "transcript": result["text"], + "language": result.get("language", language), + "speaker_segments": speaker_segments, + "segments": result.get("segments", []), + "processing_time": processing_time, + "raw_output": result, + } + + finally: + # Clean up temporary file + if temp_file_path and os.path.exists(temp_file_path): + try: + os.unlink(temp_file_path) + except Exception: + pass + + +# Singleton instance +transcription_service = TranscriptionService() diff --git a/app/services/ai/tts_service.py b/app/services/ai/tts_service.py index 640c682c..60d41255 100644 --- a/app/services/ai/tts_service.py +++ b/app/services/ai/tts_service.py @@ -1,312 +1,312 @@ -""" -TTS service for converting text to speech using various TTS providers. -""" - -import time -from typing import Optional, Dict, Any, Tuple -from uuid import UUID - -from app.models.database import ModelProvider, AIProvider, Integration -from app.services.storage.s3_service import s3_service -from efficientai.services.cartesia.http_tts import synthesize_cartesia_bytes -from efficientai.services.deepgram.http_tts import synthesize_deepgram_bytes -from efficientai.services.elevenlabs.http_tts import synthesize_elevenlabs_bytes -from efficientai.services.google.http_tts import synthesize_google_bytes -from efficientai.services.murf.tts import synthesize_murf_stream_bytes -from efficientai.services.openai.http_tts import synthesize_openai_bytes -from efficientai.services.sarvam.http_tts import synthesize_sarvam_bytes -from efficientai.services.smallest.http_tts import synthesize_smallest_bytes -from efficientai.services.voicemaker.http_tts import synthesize_voicemaker_bytes -from sqlalchemy.orm import Session - - -ELEVENLABS_HZ_TO_OUTPUT_FORMAT = { - 8000: "pcm_8000", - 16000: "pcm_16000", - 22050: "mp3_22050_32", - 24000: "pcm_24000", - 44100: "mp3_44100_128", -} - -PROVIDER_SUPPORTED_SAMPLE_RATES: Dict[str, list] = { - "elevenlabs": [8000, 16000, 22050, 24000, 44100], - "cartesia": [8000, 16000, 22050, 24000, 44100], - "deepgram": [8000, 16000, 24000, 48000], - "sarvam": [8000, 16000, 22050], - "murf": [8000, 16000, 24000, 44100, 48000], - "smallest": [8000, 16000, 24000], - "voicemaker": [8000, 16000, 22050, 24000, 44100, 48000], -} - - -def get_audio_file_extension(provider: str, sample_rate_hz: Optional[int] = None) -> str: - """Determine audio file extension based on provider and requested sample rate.""" - if provider == "sarvam": - # Sarvam HTTP TTS returns base64 WAV audio. - return "wav" - if provider == "smallest": - return "wav" - if provider == "elevenlabs" and sample_rate_hz: - fmt = ELEVENLABS_HZ_TO_OUTPUT_FORMAT.get(sample_rate_hz, "") - if fmt.startswith(("pcm_", "ulaw_")): - return "wav" - return "mp3" - - -class TTSService: - """Service for converting text to speech using various TTS providers.""" - - def __init__(self): - self._provider_handlers: Dict[str, Any] = {} - - def _get_ai_provider(self, provider: ModelProvider, db: Session, organization_id: UUID) -> Optional[AIProvider]: - """Get AI provider configuration from database.""" - from sqlalchemy import func - - provider_value = provider.value if hasattr(provider, "value") else provider - - ai_provider = db.query(AIProvider).filter( - AIProvider.provider == provider_value, - AIProvider.organization_id == organization_id, - AIProvider.is_active == True, - ).first() - - if not ai_provider: - ai_provider = db.query(AIProvider).filter( - func.lower(AIProvider.provider) == provider_value.lower(), - AIProvider.organization_id == organization_id, - AIProvider.is_active == True, - ).first() - - return ai_provider - - def _get_api_key_for_provider( - self, provider: ModelProvider, db: Session, organization_id: UUID - ) -> str: - """Resolve and decrypt API key from AIProvider or Integration tables.""" - from app.core.encryption import decrypt_api_key - from sqlalchemy import func - - ai_provider = self._get_ai_provider(provider, db, organization_id) - if ai_provider: - return decrypt_api_key(ai_provider.api_key) - - # Fallback: check Integration table for cartesia/elevenlabs/deepgram - provider_value = provider.value if hasattr(provider, "value") else provider - integration = db.query(Integration).filter( - func.lower(Integration.platform) == provider_value.lower(), - Integration.organization_id == organization_id, - Integration.is_active == True, - ).first() - if integration: - return decrypt_api_key(integration.api_key) - - raise RuntimeError(f"No API key configured for provider {provider_value}") - - # ------------------------------------------------------------------ - # OpenAI - # ------------------------------------------------------------------ - - def _synthesize_with_openai( - self, text: str, model: str, api_key: str, - voice: Optional[str] = "alloy", config: Optional[Dict[str, Any]] = None - ) -> Tuple[bytes, float]: - return synthesize_openai_bytes(text=text, model=model, api_key=api_key, voice=voice, config=config) - - # ------------------------------------------------------------------ - # Google (unary RPC - no streaming; TTFB ~= total API time) - # ------------------------------------------------------------------ - - def _synthesize_with_google( - self, text: str, model: str, api_key: str, - voice: Optional[str] = None, config: Optional[Dict[str, Any]] = None - ) -> Tuple[bytes, float]: - return synthesize_google_bytes(text=text, model=model, api_key=api_key, voice=voice, config=config) - - # ------------------------------------------------------------------ - # ElevenLabs - # ------------------------------------------------------------------ - - def _synthesize_with_elevenlabs( - self, text: str, model: str, api_key: str, - voice: Optional[str] = None, config: Optional[Dict[str, Any]] = None - ) -> Tuple[bytes, float]: - return synthesize_elevenlabs_bytes(text=text, model=model, api_key=api_key, voice=voice, config=config) - - # ------------------------------------------------------------------ - # Cartesia - # ------------------------------------------------------------------ - - def _synthesize_with_cartesia( - self, text: str, model: str, api_key: str, - voice: Optional[str] = None, config: Optional[Dict[str, Any]] = None - ) -> Tuple[bytes, float]: - return synthesize_cartesia_bytes(text=text, model=model, api_key=api_key, voice=voice, config=config) - - # ------------------------------------------------------------------ - # Deepgram - # ------------------------------------------------------------------ - - def _synthesize_with_deepgram( - self, text: str, model: str, api_key: str, - voice: Optional[str] = None, config: Optional[Dict[str, Any]] = None - ) -> Tuple[bytes, float]: - return synthesize_deepgram_bytes(text=text, model=model, api_key=api_key, voice=voice, config=config) - - # ------------------------------------------------------------------ - # Sarvam - # ------------------------------------------------------------------ - - def _synthesize_with_sarvam( - self, text: str, model: str, api_key: str, - voice: Optional[str] = None, config: Optional[Dict[str, Any]] = None - ) -> Tuple[bytes, float]: - return synthesize_sarvam_bytes(text=text, model=model, api_key=api_key, voice=voice, config=config) - - # ------------------------------------------------------------------ - # VoiceMaker - # ------------------------------------------------------------------ - - def _synthesize_with_voicemaker( - self, text: str, model: str, api_key: str, - voice: Optional[str] = None, config: Optional[Dict[str, Any]] = None - ) -> Tuple[bytes, float]: - return synthesize_voicemaker_bytes(text=text, model=model, api_key=api_key, voice=voice, config=config) - - # ------------------------------------------------------------------ - # Smallest.ai - # ------------------------------------------------------------------ - - def _synthesize_with_smallest( - self, text: str, model: str, api_key: str, - voice: Optional[str] = None, config: Optional[Dict[str, Any]] = None - ) -> Tuple[bytes, float]: - return synthesize_smallest_bytes(text=text, model=model, api_key=api_key, voice=voice, config=config) - - # ------------------------------------------------------------------ - # Main synthesis entry point - # ------------------------------------------------------------------ - - def register_tts_provider(self, provider: str, handler: Any) -> None: - """Register/override a TTS provider handler dynamically.""" - self._provider_handlers[provider.strip().lower()] = handler - - def _get_tts_handler(self, tts_provider: ModelProvider): - provider_key = (tts_provider.value if hasattr(tts_provider, "value") else str(tts_provider)).lower() - - # Prefer explicitly registered handlers. - handler = self._provider_handlers.get(provider_key) - if handler: - return handler - - # Fallback convention: _synthesize_with_. - handler = getattr(self, f"_synthesize_with_{provider_key}", None) - if callable(handler): - # Cache resolved handler to avoid repeated getattr lookups. - self._provider_handlers[provider_key] = handler - return handler - - if not handler: - raise NotImplementedError(f"TTS provider {tts_provider} not supported") - return handler - - def _synthesize_with_murf( - self, - text: str, - model: str, - api_key: str, - voice: Optional[str] = None, - config: Optional[Dict[str, Any]] = None - ) -> Tuple[bytes, float]: - """Delegate Murf synthesis to shared efficientai Murf service helper.""" - try: - return synthesize_murf_stream_bytes( - text=text, - model=model, - api_key=api_key, - voice=voice, - config=config, - ) - except ImportError: - raise RuntimeError("requests library not installed. Install with: pip install requests") - except Exception as e: - import traceback - error_details = traceback.format_exc() - raise RuntimeError(f"Murf TTS synthesis failed: {str(e)}\nDetails: {error_details}") - - def synthesize( - self, - text: str, - tts_provider: ModelProvider, - tts_model: str, - organization_id: UUID, - db: Session, - voice: Optional[str] = None, - config: Optional[Dict[str, Any]] = None, - ) -> bytes: - """ - Synthesize speech from text. - - Args: - text: Text to convert to speech - tts_provider: TTS provider to use - tts_model: TTS model name - organization_id: Organization ID - db: Database session - voice: Voice selection (if applicable) - config: Additional provider-specific configuration - - Returns: - Audio bytes (MP3 format) - """ - api_key = self._get_api_key_for_provider(tts_provider, db, organization_id) - handler = self._get_tts_handler(tts_provider) - audio_bytes, _ttfb_ms = handler(text, tts_model, api_key, voice, config) - return audio_bytes - - def synthesize_timed( - self, - text: str, - tts_provider: ModelProvider, - tts_model: str, - organization_id: UUID, - db: Session, - voice: Optional[str] = None, - config: Optional[Dict[str, Any]] = None, - ) -> Tuple[bytes, float, float]: - """Synthesize and return (audio_bytes, total_latency_ms, ttfb_ms).""" - api_key = self._get_api_key_for_provider(tts_provider, db, organization_id) - handler = self._get_tts_handler(tts_provider) - start = time.time() - audio_bytes, ttfb_ms = handler(text, tts_model, api_key, voice, config) - total_latency_ms = (time.time() - start) * 1000 - return audio_bytes, total_latency_ms, ttfb_ms - - def synthesize_and_upload( - self, - text: str, - tts_provider: ModelProvider, - tts_model: str, - organization_id: UUID, - db: Session, - voice: Optional[str] = None, - config: Optional[Dict[str, Any]] = None, - file_prefix: str = "tts_output", - ) -> str: - """Synthesize speech and upload to S3. Returns S3 key.""" - audio_bytes = self.synthesize(text, tts_provider, tts_model, organization_id, db, voice, config) - - import uuid as _uuid - - file_id = _uuid.uuid4() - s3_key = s3_service.upload_file( - file_id=file_id, - file_content=audio_bytes, - file_format="mp3", - organization_id=organization_id, - ) - return s3_key - - -# Singleton instance -tts_service = TTSService() +""" +TTS service for converting text to speech using various TTS providers. +""" + +import time +from typing import Optional, Dict, Any, Tuple +from uuid import UUID + +from app.models.database import ModelProvider, AIProvider, Integration +from app.services.storage.s3_service import s3_service +from efficientai.services.cartesia.http_tts import synthesize_cartesia_bytes +from efficientai.services.deepgram.http_tts import synthesize_deepgram_bytes +from efficientai.services.elevenlabs.http_tts import synthesize_elevenlabs_bytes +from efficientai.services.google.http_tts import synthesize_google_bytes +from efficientai.services.murf.tts import synthesize_murf_stream_bytes +from efficientai.services.openai.http_tts import synthesize_openai_bytes +from efficientai.services.sarvam.http_tts import synthesize_sarvam_bytes +from efficientai.services.smallest.http_tts import synthesize_smallest_bytes +from efficientai.services.voicemaker.http_tts import synthesize_voicemaker_bytes +from sqlalchemy.orm import Session + + +ELEVENLABS_HZ_TO_OUTPUT_FORMAT = { + 8000: "pcm_8000", + 16000: "pcm_16000", + 22050: "mp3_22050_32", + 24000: "pcm_24000", + 44100: "mp3_44100_128", +} + +PROVIDER_SUPPORTED_SAMPLE_RATES: Dict[str, list] = { + "elevenlabs": [8000, 16000, 22050, 24000, 44100], + "cartesia": [8000, 16000, 22050, 24000, 44100], + "deepgram": [8000, 16000, 24000, 48000], + "sarvam": [8000, 16000, 22050], + "murf": [8000, 16000, 24000, 44100, 48000], + "smallest": [8000, 16000, 24000], + "voicemaker": [8000, 16000, 22050, 24000, 44100, 48000], +} + + +def get_audio_file_extension(provider: str, sample_rate_hz: Optional[int] = None) -> str: + """Determine audio file extension based on provider and requested sample rate.""" + if provider == "sarvam": + # Sarvam HTTP TTS returns base64 WAV audio. + return "wav" + if provider == "smallest": + return "wav" + if provider == "elevenlabs" and sample_rate_hz: + fmt = ELEVENLABS_HZ_TO_OUTPUT_FORMAT.get(sample_rate_hz, "") + if fmt.startswith(("pcm_", "ulaw_")): + return "wav" + return "mp3" + + +class TTSService: + """Service for converting text to speech using various TTS providers.""" + + def __init__(self): + self._provider_handlers: Dict[str, Any] = {} + + def _get_ai_provider(self, provider: ModelProvider, db: Session, organization_id: UUID) -> Optional[AIProvider]: + """Get AI provider configuration from database.""" + from sqlalchemy import func + + provider_value = provider.value if hasattr(provider, "value") else provider + + ai_provider = db.query(AIProvider).filter( + AIProvider.provider == provider_value, + AIProvider.organization_id == organization_id, + AIProvider.is_active == True, + ).first() + + if not ai_provider: + ai_provider = db.query(AIProvider).filter( + func.lower(AIProvider.provider) == provider_value.lower(), + AIProvider.organization_id == organization_id, + AIProvider.is_active == True, + ).first() + + return ai_provider + + def _get_api_key_for_provider( + self, provider: ModelProvider, db: Session, organization_id: UUID + ) -> str: + """Resolve and decrypt API key from AIProvider or Integration tables.""" + from app.core.encryption import decrypt_api_key + from sqlalchemy import func + + ai_provider = self._get_ai_provider(provider, db, organization_id) + if ai_provider: + return decrypt_api_key(ai_provider.api_key) + + # Fallback: check Integration table for cartesia/elevenlabs/deepgram + provider_value = provider.value if hasattr(provider, "value") else provider + integration = db.query(Integration).filter( + func.lower(Integration.platform) == provider_value.lower(), + Integration.organization_id == organization_id, + Integration.is_active == True, + ).first() + if integration: + return decrypt_api_key(integration.api_key) + + raise RuntimeError(f"No API key configured for provider {provider_value}") + + # ------------------------------------------------------------------ + # OpenAI + # ------------------------------------------------------------------ + + def _synthesize_with_openai( + self, text: str, model: str, api_key: str, + voice: Optional[str] = "alloy", config: Optional[Dict[str, Any]] = None + ) -> Tuple[bytes, float]: + return synthesize_openai_bytes(text=text, model=model, api_key=api_key, voice=voice, config=config) + + # ------------------------------------------------------------------ + # Google (unary RPC - no streaming; TTFB ~= total API time) + # ------------------------------------------------------------------ + + def _synthesize_with_google( + self, text: str, model: str, api_key: str, + voice: Optional[str] = None, config: Optional[Dict[str, Any]] = None + ) -> Tuple[bytes, float]: + return synthesize_google_bytes(text=text, model=model, api_key=api_key, voice=voice, config=config) + + # ------------------------------------------------------------------ + # ElevenLabs + # ------------------------------------------------------------------ + + def _synthesize_with_elevenlabs( + self, text: str, model: str, api_key: str, + voice: Optional[str] = None, config: Optional[Dict[str, Any]] = None + ) -> Tuple[bytes, float]: + return synthesize_elevenlabs_bytes(text=text, model=model, api_key=api_key, voice=voice, config=config) + + # ------------------------------------------------------------------ + # Cartesia + # ------------------------------------------------------------------ + + def _synthesize_with_cartesia( + self, text: str, model: str, api_key: str, + voice: Optional[str] = None, config: Optional[Dict[str, Any]] = None + ) -> Tuple[bytes, float]: + return synthesize_cartesia_bytes(text=text, model=model, api_key=api_key, voice=voice, config=config) + + # ------------------------------------------------------------------ + # Deepgram + # ------------------------------------------------------------------ + + def _synthesize_with_deepgram( + self, text: str, model: str, api_key: str, + voice: Optional[str] = None, config: Optional[Dict[str, Any]] = None + ) -> Tuple[bytes, float]: + return synthesize_deepgram_bytes(text=text, model=model, api_key=api_key, voice=voice, config=config) + + # ------------------------------------------------------------------ + # Sarvam + # ------------------------------------------------------------------ + + def _synthesize_with_sarvam( + self, text: str, model: str, api_key: str, + voice: Optional[str] = None, config: Optional[Dict[str, Any]] = None + ) -> Tuple[bytes, float]: + return synthesize_sarvam_bytes(text=text, model=model, api_key=api_key, voice=voice, config=config) + + # ------------------------------------------------------------------ + # VoiceMaker + # ------------------------------------------------------------------ + + def _synthesize_with_voicemaker( + self, text: str, model: str, api_key: str, + voice: Optional[str] = None, config: Optional[Dict[str, Any]] = None + ) -> Tuple[bytes, float]: + return synthesize_voicemaker_bytes(text=text, model=model, api_key=api_key, voice=voice, config=config) + + # ------------------------------------------------------------------ + # Smallest.ai + # ------------------------------------------------------------------ + + def _synthesize_with_smallest( + self, text: str, model: str, api_key: str, + voice: Optional[str] = None, config: Optional[Dict[str, Any]] = None + ) -> Tuple[bytes, float]: + return synthesize_smallest_bytes(text=text, model=model, api_key=api_key, voice=voice, config=config) + + # ------------------------------------------------------------------ + # Main synthesis entry point + # ------------------------------------------------------------------ + + def register_tts_provider(self, provider: str, handler: Any) -> None: + """Register/override a TTS provider handler dynamically.""" + self._provider_handlers[provider.strip().lower()] = handler + + def _get_tts_handler(self, tts_provider: ModelProvider): + provider_key = (tts_provider.value if hasattr(tts_provider, "value") else str(tts_provider)).lower() + + # Prefer explicitly registered handlers. + handler = self._provider_handlers.get(provider_key) + if handler: + return handler + + # Fallback convention: _synthesize_with_. + handler = getattr(self, f"_synthesize_with_{provider_key}", None) + if callable(handler): + # Cache resolved handler to avoid repeated getattr lookups. + self._provider_handlers[provider_key] = handler + return handler + + if not handler: + raise NotImplementedError(f"TTS provider {tts_provider} not supported") + return handler + + def _synthesize_with_murf( + self, + text: str, + model: str, + api_key: str, + voice: Optional[str] = None, + config: Optional[Dict[str, Any]] = None + ) -> Tuple[bytes, float]: + """Delegate Murf synthesis to shared efficientai Murf service helper.""" + try: + return synthesize_murf_stream_bytes( + text=text, + model=model, + api_key=api_key, + voice=voice, + config=config, + ) + except ImportError: + raise RuntimeError("requests library not installed. Install with: pip install requests") + except Exception as e: + import traceback + error_details = traceback.format_exc() + raise RuntimeError(f"Murf TTS synthesis failed: {str(e)}\nDetails: {error_details}") + + def synthesize( + self, + text: str, + tts_provider: ModelProvider, + tts_model: str, + organization_id: UUID, + db: Session, + voice: Optional[str] = None, + config: Optional[Dict[str, Any]] = None, + ) -> bytes: + """ + Synthesize speech from text. + + Args: + text: Text to convert to speech + tts_provider: TTS provider to use + tts_model: TTS model name + organization_id: Organization ID + db: Database session + voice: Voice selection (if applicable) + config: Additional provider-specific configuration + + Returns: + Audio bytes (MP3 format) + """ + api_key = self._get_api_key_for_provider(tts_provider, db, organization_id) + handler = self._get_tts_handler(tts_provider) + audio_bytes, _ttfb_ms = handler(text, tts_model, api_key, voice, config) + return audio_bytes + + def synthesize_timed( + self, + text: str, + tts_provider: ModelProvider, + tts_model: str, + organization_id: UUID, + db: Session, + voice: Optional[str] = None, + config: Optional[Dict[str, Any]] = None, + ) -> Tuple[bytes, float, float]: + """Synthesize and return (audio_bytes, total_latency_ms, ttfb_ms).""" + api_key = self._get_api_key_for_provider(tts_provider, db, organization_id) + handler = self._get_tts_handler(tts_provider) + start = time.time() + audio_bytes, ttfb_ms = handler(text, tts_model, api_key, voice, config) + total_latency_ms = (time.time() - start) * 1000 + return audio_bytes, total_latency_ms, ttfb_ms + + def synthesize_and_upload( + self, + text: str, + tts_provider: ModelProvider, + tts_model: str, + organization_id: UUID, + db: Session, + voice: Optional[str] = None, + config: Optional[Dict[str, Any]] = None, + file_prefix: str = "tts_output", + ) -> str: + """Synthesize speech and upload to S3. Returns S3 key.""" + audio_bytes = self.synthesize(text, tts_provider, tts_model, organization_id, db, voice, config) + + import uuid as _uuid + + file_id = _uuid.uuid4() + s3_key = s3_service.upload_file( + file_id=file_id, + file_content=audio_bytes, + file_format="mp3", + organization_id=organization_id, + ) + return s3_key + + +# Singleton instance +tts_service = TTSService() diff --git a/app/services/call_imports/audit.py b/app/services/call_imports/audit.py index cc940562..a30c72ef 100644 --- a/app/services/call_imports/audit.py +++ b/app/services/call_imports/audit.py @@ -1,103 +1,103 @@ -"""Actor stamping and email resolution for call import audit fields.""" - -from __future__ import annotations - -from typing import Dict, Iterable, Optional, Set, Tuple -from uuid import UUID - -from sqlalchemy.orm import Session - -from app.core.auth.principal import Principal -from app.models.database import CallImport, CallImportEvaluation, User - - -def stamp_call_import_actor( - call_import: CallImport, - principal: Principal, - *, - creating: bool = False, -) -> None: - if creating and principal.user_id is not None: - call_import.created_by_user_id = principal.user_id - if principal.user_id is not None: - call_import.last_updated_by_user_id = principal.user_id - - -def stamp_evaluation_actor( - evaluation: CallImportEvaluation, - principal: Principal, - *, - creating: bool = False, -) -> None: - if creating and principal.user_id is not None: - evaluation.created_by_user_id = principal.user_id - if principal.user_id is not None: - evaluation.last_updated_by_user_id = principal.user_id - - -def user_ids_from_call_imports(imports: Iterable[CallImport]) -> Set[UUID]: - ids: Set[UUID] = set() - for row in imports: - created_by = getattr(row, "created_by_user_id", None) - updated_by = getattr(row, "last_updated_by_user_id", None) - if created_by is not None: - ids.add(created_by) - if updated_by is not None: - ids.add(updated_by) - return ids - - -def user_ids_from_evaluations( - evaluations: Iterable[CallImportEvaluation], -) -> Set[UUID]: - ids: Set[UUID] = set() - for row in evaluations: - created_by = getattr(row, "created_by_user_id", None) - updated_by = getattr(row, "last_updated_by_user_id", None) - if created_by is not None: - ids.add(created_by) - if updated_by is not None: - ids.add(updated_by) - return ids - - -def emails_for_user_ids(db: Session, user_ids: Iterable[UUID]) -> Dict[UUID, str]: - unique = {uid for uid in user_ids if uid is not None} - if not unique: - return {} - rows = db.query(User.id, User.email).filter(User.id.in_(unique)).all() - return {row.id: row.email for row in rows if row.email} - - -def actor_emails_for_call_import( - call_import: CallImport, - email_by_id: Dict[UUID, str], -) -> Tuple[Optional[str], Optional[str]]: - created = ( - email_by_id.get(call_import.created_by_user_id) - if call_import.created_by_user_id - else None - ) - updated = ( - email_by_id.get(call_import.last_updated_by_user_id) - if call_import.last_updated_by_user_id - else None - ) - return created, updated - - -def actor_emails_for_evaluation( - evaluation: CallImportEvaluation, - email_by_id: Dict[UUID, str], -) -> Tuple[Optional[str], Optional[str]]: - created = ( - email_by_id.get(evaluation.created_by_user_id) - if evaluation.created_by_user_id - else None - ) - updated = ( - email_by_id.get(evaluation.last_updated_by_user_id) - if evaluation.last_updated_by_user_id - else None - ) - return created, updated +"""Actor stamping and email resolution for call import audit fields.""" + +from __future__ import annotations + +from typing import Dict, Iterable, Optional, Set, Tuple +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.core.auth.principal import Principal +from app.models.database import CallImport, CallImportEvaluation, User + + +def stamp_call_import_actor( + call_import: CallImport, + principal: Principal, + *, + creating: bool = False, +) -> None: + if creating and principal.user_id is not None: + call_import.created_by_user_id = principal.user_id + if principal.user_id is not None: + call_import.last_updated_by_user_id = principal.user_id + + +def stamp_evaluation_actor( + evaluation: CallImportEvaluation, + principal: Principal, + *, + creating: bool = False, +) -> None: + if creating and principal.user_id is not None: + evaluation.created_by_user_id = principal.user_id + if principal.user_id is not None: + evaluation.last_updated_by_user_id = principal.user_id + + +def user_ids_from_call_imports(imports: Iterable[CallImport]) -> Set[UUID]: + ids: Set[UUID] = set() + for row in imports: + created_by = getattr(row, "created_by_user_id", None) + updated_by = getattr(row, "last_updated_by_user_id", None) + if created_by is not None: + ids.add(created_by) + if updated_by is not None: + ids.add(updated_by) + return ids + + +def user_ids_from_evaluations( + evaluations: Iterable[CallImportEvaluation], +) -> Set[UUID]: + ids: Set[UUID] = set() + for row in evaluations: + created_by = getattr(row, "created_by_user_id", None) + updated_by = getattr(row, "last_updated_by_user_id", None) + if created_by is not None: + ids.add(created_by) + if updated_by is not None: + ids.add(updated_by) + return ids + + +def emails_for_user_ids(db: Session, user_ids: Iterable[UUID]) -> Dict[UUID, str]: + unique = {uid for uid in user_ids if uid is not None} + if not unique: + return {} + rows = db.query(User.id, User.email).filter(User.id.in_(unique)).all() + return {row.id: row.email for row in rows if row.email} + + +def actor_emails_for_call_import( + call_import: CallImport, + email_by_id: Dict[UUID, str], +) -> Tuple[Optional[str], Optional[str]]: + created = ( + email_by_id.get(call_import.created_by_user_id) + if call_import.created_by_user_id + else None + ) + updated = ( + email_by_id.get(call_import.last_updated_by_user_id) + if call_import.last_updated_by_user_id + else None + ) + return created, updated + + +def actor_emails_for_evaluation( + evaluation: CallImportEvaluation, + email_by_id: Dict[UUID, str], +) -> Tuple[Optional[str], Optional[str]]: + created = ( + email_by_id.get(evaluation.created_by_user_id) + if evaluation.created_by_user_id + else None + ) + updated = ( + email_by_id.get(evaluation.last_updated_by_user_id) + if evaluation.last_updated_by_user_id + else None + ) + return created, updated diff --git a/app/services/call_imports/bulk_ops.py b/app/services/call_imports/bulk_ops.py index 69e2b302..09c1acee 100644 --- a/app/services/call_imports/bulk_ops.py +++ b/app/services/call_imports/bulk_ops.py @@ -998,6 +998,7 @@ def rollup_call_import_batch_status(db: Session, call_import: CallImport) -> Non CallImportEvaluationRow.id, CallImportEvaluationRow.status, CallImportEvaluationRow.celery_task_id, + CallImportEvaluationRow.call_import_row_id, ) @@ -1094,6 +1095,9 @@ def execute_evaluation_cancel( mode=mode, ) + if mode == "abort": + _sweep_evaluation_diarisation_cancel(db, evaluation_id) + db.refresh(evaluation) _rollup_evaluation_status(evaluation, db) db.commit() @@ -1111,21 +1115,91 @@ def execute_evaluation_cancel( clear_evaluation_bulk_operation(evaluation_id) +def _sweep_evaluation_diarisation_cancel( + catalog_db: Session, + evaluation_id: UUID, +) -> int: + """Fail any remaining in-flight diarisation for rows in this evaluation run.""" + from app.api.v1.routes.call_imports import _apply_diarisation_cancel + + cancelled_total = 0 + if is_sharding_enabled(): + from app.db_sharding.pool_manager import db_pool_manager + + router = db_pool_manager.router + assert router is not None + for shard_id in router.shard_ids: + factory = db_pool_manager.shard_session_factory(shard_id) + shard_db = factory() + try: + source_rows = ( + shard_db.query(CallImportRow) + .join( + CallImportEvaluationRow, + CallImportEvaluationRow.call_import_row_id + == CallImportRow.id, + ) + .filter( + CallImportEvaluationRow.evaluation_id == evaluation_id, + CallImportRow.diarised_transcript_status.in_( + ("pending", "running") + ), + ) + .all() + ) + if not source_rows: + continue + cancelled, _ = _apply_diarisation_cancel(source_rows) + if cancelled: + shard_db.commit() + cancelled_total += cancelled + finally: + shard_db.close() + return cancelled_total + + source_rows = ( + catalog_db.query(CallImportRow) + .join( + CallImportEvaluationRow, + CallImportEvaluationRow.call_import_row_id == CallImportRow.id, + ) + .filter( + CallImportEvaluationRow.evaluation_id == evaluation_id, + CallImportRow.diarised_transcript_status.in_(("pending", "running")), + ) + .all() + ) + if not source_rows: + return 0 + cancelled, _ = _apply_diarisation_cancel(source_rows) + if cancelled: + catalog_db.commit() + return cancelled + + def _cancel_evaluation_rows_on_session( db: Session, evaluation_id: UUID, *, mode: Literal["abort", "force_fail_pending"], ) -> int: - from app.api.v1.routes.call_import_evaluations import EVAL_CANCELLED_BY_USER_ERROR + from app.api.v1.routes.call_import_evaluations import ( + _cancel_eval_row_with_source_diarisation, + ) cancelled_total = 0 + cascade_diarisation = mode == "abort" while True: - query = ( - db.query(CallImportEvaluationRow) - .options(load_only(*_EVAL_CANCEL_COLUMNS)) - .filter(CallImportEvaluationRow.evaluation_id == evaluation_id) + query = db.query(CallImportEvaluationRow).filter( + CallImportEvaluationRow.evaluation_id == evaluation_id ) + if cascade_diarisation: + query = query.add_columns(CallImportRow).join( + CallImportRow, + CallImportRow.id == CallImportEvaluationRow.call_import_row_id, + ) + else: + query = query.options(load_only(*_EVAL_CANCEL_COLUMNS)) if mode == "abort": query = query.filter( CallImportEvaluationRow.status.in_(("pending", "running")) @@ -1133,24 +1207,37 @@ def _cancel_evaluation_rows_on_session( else: query = query.filter(CallImportEvaluationRow.status == "pending") - rows = ( - query.order_by(CallImportEvaluationRow.id.asc()) - .limit(_BULK_INSERT_CHUNK) - .all() - ) - if not rows: - break + if cascade_diarisation: + rows = ( + query.order_by(CallImportEvaluationRow.id.asc()) + .limit(_BULK_INSERT_CHUNK) + .all() + ) + if not rows: + break + pairs = [(eval_row, source_row) for eval_row, source_row in rows] + else: + eval_rows = ( + query.order_by(CallImportEvaluationRow.id.asc()) + .limit(_BULK_INSERT_CHUNK) + .all() + ) + if not eval_rows: + break + pairs = [(eval_row, None) for eval_row in eval_rows] task_ids: List[str] = [] now = datetime.now(timezone.utc) - for row in rows: - task_id = (row.celery_task_id or "").strip() - if task_id: - task_ids.append(task_id) - row.status = "failed" - row.error_message = EVAL_CANCELLED_BY_USER_ERROR - row.finished_at = now - row.celery_task_id = None + for eval_row, source_row in pairs: + row_task_ids = _cancel_eval_row_with_source_diarisation( + eval_row, + source_row, + cascade_diarisation=cascade_diarisation, + now=now, + ) + for task_id in row_task_ids: + if task_id not in task_ids: + task_ids.append(task_id) cancelled_total += 1 _batch_revoke_celery_task_ids(task_ids, terminate=True) diff --git a/app/services/metric_studio/__init__.py b/app/services/metric_studio/__init__.py new file mode 100644 index 00000000..eabcd651 --- /dev/null +++ b/app/services/metric_studio/__init__.py @@ -0,0 +1 @@ +"""Metrics Studio services.""" diff --git a/app/services/metric_studio/metric_selection.py b/app/services/metric_studio/metric_selection.py new file mode 100644 index 00000000..78cddf92 --- /dev/null +++ b/app/services/metric_studio/metric_selection.py @@ -0,0 +1,100 @@ +"""Metric selection helpers for Metrics Studio runs.""" + +from __future__ import annotations + +from typing import Dict, List, Tuple +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models.database import Metric + + +def expand_studio_metric_selection( + db: Session, + org_id: UUID, + selected_ids: List[UUID], +) -> Tuple[List[Metric], Dict[UUID, List[Metric]]]: + """Like call-import metric expansion but allows draft/disabled metrics.""" + if not selected_ids: + return [], {} + + requested = list(selected_ids) + initial_rows = ( + db.query(Metric) + .filter( + Metric.organization_id == org_id, + Metric.id.in_(requested), + ) + .all() + ) + initial_by_id = {row.id: row for row in initial_rows} + + parent_ids_requested = { + m.id for m in initial_rows if m.selection_mode and not m.parent_metric_id + } + explicit_children_by_parent: Dict[UUID, List[Metric]] = {} + for m in initial_rows: + if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: + explicit_children_by_parent.setdefault(m.parent_metric_id, []).append(m) + + parents_needing_full_expansion = [ + pid for pid in parent_ids_requested if pid not in explicit_children_by_parent + ] + auto_expanded_children: Dict[UUID, List[Metric]] = {} + if parents_needing_full_expansion: + for pid in parents_needing_full_expansion: + child_rows = ( + db.query(Metric) + .filter( + Metric.organization_id == org_id, + Metric.parent_metric_id == pid, + ) + .order_by(Metric.created_at.asc()) + .all() + ) + auto_expanded_children[pid] = child_rows + + parent_to_children: Dict[UUID, List[Metric]] = {} + for pid in parent_ids_requested: + children = explicit_children_by_parent.get(pid) or auto_expanded_children.get( + pid, [] + ) + parent_to_children[pid] = list(children) + + effective: List[Metric] = [] + seen: set[UUID] = set() + for mid in requested: + m = initial_by_id.get(mid) + if m is None: + continue + if m.selection_mode and not m.parent_metric_id: + for child in parent_to_children.get(m.id, []): + if child.id in seen: + continue + seen.add(child.id) + effective.append(child) + elif m.parent_metric_id is None or m.parent_metric_id not in parent_ids_requested: + if m.id in seen: + continue + seen.add(m.id) + effective.append(m) + + return effective, parent_to_children + + +def load_studio_run_metrics( + db: Session, + organization_id: UUID, + metric_ids: List[UUID], +) -> List[Metric]: + if not metric_ids: + return [] + return ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.id.in_(metric_ids), + ) + .all() + ) diff --git a/app/services/metric_studio/source_resolver.py b/app/services/metric_studio/source_resolver.py new file mode 100644 index 00000000..c659e8cf --- /dev/null +++ b/app/services/metric_studio/source_resolver.py @@ -0,0 +1,323 @@ +"""Resolve heterogeneous call sources into a common evaluation sample.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, Optional +from uuid import UUID + +from fastapi import HTTPException +from sqlalchemy.orm import Session + +from app.api.v1.routes.playground import extract_transcript_from_call_data +from app.models.database import ( + Agent, + CallImport, + CallImportRow, + CallRecording, + Evaluator, + EvaluatorResult, + Persona, + Scenario, +) + + +def _basename_from_s3_key(key: Optional[str]) -> Optional[str]: + if not key: + return None + parts = key.strip().split("/") + name = parts[-1].strip() if parts else "" + return name or None + + +@dataclass +class ResolvedCallSample: + source_kind: str + source_ref: str + label: str + transcript: Optional[str] + diarised_transcript: Optional[str] + audio_s3_key: Optional[str] + call_data: Optional[dict] + agent_id: Optional[UUID] + metadata: Dict[str, Any] = field(default_factory=dict) + + +def _resolve_call_import_row( + db: Session, + *, + organization_id: UUID, + workspace_id: UUID, + source_ref: str, + display_label: Optional[str], +) -> ResolvedCallSample: + try: + row_id = UUID(source_ref) + except ValueError as exc: + raise HTTPException(status_code=400, detail="Invalid call_import_row id.") from exc + + from app.db_sharding.sessions import is_sharding_enabled + from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row + + row_db = None + extra_catalog = None + close_located_sessions = False + try: + if is_sharding_enabled(): + try: + row_db, located_catalog, row, _shard_id = locate_call_import_row(row_id) + extra_catalog = located_catalog if located_catalog is not row_db else None + close_located_sessions = True + except LookupError as exc: + raise HTTPException(status_code=404, detail="Call import row not found.") from exc + else: + row = db.query(CallImportRow).filter(CallImportRow.id == row_id).first() + if row is None: + raise HTTPException(status_code=404, detail="Call import row not found.") + row_db = db + + if row.organization_id != organization_id: + raise HTTPException(status_code=404, detail="Call import row not found.") + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == row.call_import_id, + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + .first() + ) + if not call_import: + raise HTTPException(status_code=404, detail="Call import row not found.") + + recording_filename = _basename_from_s3_key(row.recording_s3_key) + label = ( + display_label + or recording_filename + or row.conversation_id + or f"Import row {row.row_index}" + ) + return ResolvedCallSample( + source_kind="call_import_row", + source_ref=str(row.id), + label=label, + transcript=(row.transcript or "").strip() or None, + diarised_transcript=(row.diarised_transcript or "").strip() or None, + audio_s3_key=(row.recording_s3_key or "").strip() or None, + call_data=None, + agent_id=None, + metadata={ + "call_import_id": str(row.call_import_id), + "call_import_name": getattr(call_import, "name", None), + "original_filename": getattr(call_import, "original_filename", None), + "recording_filename": recording_filename, + "recording_s3_key": (row.recording_s3_key or "").strip() or None, + "row_index": row.row_index, + "conversation_id": row.conversation_id, + }, + ) + finally: + if close_located_sessions and row_db is not None: + close_row_sessions(row_db, extra_catalog) + + +def _resolve_call_recording( + db: Session, + *, + organization_id: UUID, + workspace_id: UUID, + source_ref: str, + display_label: Optional[str], +) -> ResolvedCallSample: + recording = ( + db.query(CallRecording) + .filter( + CallRecording.call_short_id == source_ref, + CallRecording.organization_id == organization_id, + CallRecording.workspace_id == workspace_id, + ) + .first() + ) + if not recording: + raise HTTPException(status_code=404, detail="Call recording not found.") + + call_data = recording.call_data if isinstance(recording.call_data, dict) else {} + platform = (recording.provider_platform or "").lower() + transcript_text, _ = extract_transcript_from_call_data(call_data, platform) + audio_s3_key = None + evaluator_result_name = None + if recording.evaluator_result_id: + result = ( + db.query(EvaluatorResult) + .filter(EvaluatorResult.id == recording.evaluator_result_id) + .first() + ) + if result: + if result.audio_s3_key: + audio_s3_key = result.audio_s3_key + evaluator_result_name = result.name or result.result_id + + agent_name = None + if recording.agent_id: + agent = db.query(Agent).filter(Agent.id == recording.agent_id).first() + agent_name = agent.name if agent else None + + label = ( + display_label + or evaluator_result_name + or agent_name + or recording.call_short_id + ) + recording_url = None + if isinstance(call_data, dict): + recording_url = call_data.get("recording_url") + if not recording_url and isinstance(call_data.get("recording_urls"), dict): + urls = call_data["recording_urls"] + recording_url = urls.get("combined_url") or urls.get("mono_url") + return ResolvedCallSample( + source_kind="call_recording", + source_ref=recording.call_short_id, + label=label, + transcript=transcript_text or None, + diarised_transcript=transcript_text or None, + audio_s3_key=audio_s3_key, + call_data=call_data or None, + agent_id=recording.agent_id, + metadata={ + "call_short_id": recording.call_short_id, + "provider_platform": recording.provider_platform, + "source": getattr(recording.source, "value", recording.source), + "agent_name": agent_name, + "evaluator_result_name": evaluator_result_name, + "audio_s3_key": audio_s3_key, + "recording_url": recording_url, + }, + ) + + +def _resolve_evaluator_result( + db: Session, + *, + organization_id: UUID, + workspace_id: UUID, + source_ref: str, + display_label: Optional[str], +) -> ResolvedCallSample: + result = None + try: + result_uuid = UUID(source_ref) + result = ( + db.query(EvaluatorResult) + .filter( + EvaluatorResult.id == result_uuid, + EvaluatorResult.organization_id == organization_id, + EvaluatorResult.workspace_id == workspace_id, + ) + .first() + ) + except ValueError: + result = ( + db.query(EvaluatorResult) + .filter( + EvaluatorResult.result_id == source_ref, + EvaluatorResult.organization_id == organization_id, + EvaluatorResult.workspace_id == workspace_id, + ) + .first() + ) + + if not result: + raise HTTPException(status_code=404, detail="Evaluator result not found.") + + persona_name = None + scenario_name = None + evaluator_name = None + if result.persona_id: + persona = db.query(Persona).filter(Persona.id == result.persona_id).first() + persona_name = persona.name if persona else None + if result.scenario_id: + scenario = db.query(Scenario).filter(Scenario.id == result.scenario_id).first() + scenario_name = scenario.name if scenario else None + if result.evaluator_id: + evaluator = db.query(Evaluator).filter(Evaluator.id == result.evaluator_id).first() + evaluator_name = evaluator.name if evaluator else None + + label = display_label or result.name or result.result_id + return ResolvedCallSample( + source_kind="evaluator_result", + source_ref=str(result.id), + label=label, + transcript=(result.transcription or "").strip() or None, + diarised_transcript=(result.transcription or "").strip() or None, + audio_s3_key=(result.audio_s3_key or "").strip() or None, + call_data=result.call_data if isinstance(result.call_data, dict) else None, + agent_id=result.agent_id, + metadata={ + "result_id": result.result_id, + "evaluator_id": str(result.evaluator_id) if result.evaluator_id else None, + "evaluator_name": evaluator_name, + "persona_id": str(result.persona_id) if result.persona_id else None, + "persona_name": persona_name, + "scenario_id": str(result.scenario_id) if result.scenario_id else None, + "scenario_name": scenario_name, + "agent_id": str(result.agent_id) if result.agent_id else None, + "audio_s3_key": (result.audio_s3_key or "").strip() or None, + }, + ) + + +def resolve_source( + db: Session, + *, + organization_id: UUID, + workspace_id: UUID, + source_kind: str, + source_ref: str, + display_label: Optional[str] = None, +) -> ResolvedCallSample: + if source_kind == "call_import_row": + return _resolve_call_import_row( + db, + organization_id=organization_id, + workspace_id=workspace_id, + source_ref=source_ref, + display_label=display_label, + ) + if source_kind == "call_recording": + return _resolve_call_recording( + db, + organization_id=organization_id, + workspace_id=workspace_id, + source_ref=source_ref, + display_label=display_label, + ) + if source_kind == "evaluator_result": + return _resolve_evaluator_result( + db, + organization_id=organization_id, + workspace_id=workspace_id, + source_ref=source_ref, + display_label=display_label, + ) + raise HTTPException(status_code=400, detail=f"Unknown source_kind: {source_kind}") + + +def preview_source_label( + db: Session, + *, + organization_id: UUID, + workspace_id: UUID, + source_kind: str, + source_ref: str, + display_label: Optional[str] = None, +) -> str: + sample = resolve_source( + db, + organization_id=organization_id, + workspace_id=workspace_id, + source_kind=source_kind, + source_ref=source_ref, + display_label=display_label, + ) + return sample.label diff --git a/app/services/signup_reference_codes.py b/app/services/signup_reference_codes.py new file mode 100644 index 00000000..c163248b --- /dev/null +++ b/app/services/signup_reference_codes.py @@ -0,0 +1,61 @@ +"""Signup reference code hashing and validation.""" + +from __future__ import annotations + +import hashlib +from datetime import datetime, timezone +from typing import Optional + +from fastapi import HTTPException, status +from sqlalchemy.orm import Session + +from app.config import settings +from app.models.database import SignupReferenceCode + + +def hash_reference_code(code: str) -> str: + normalized = code.strip().upper() + payload = f"{settings.SECRET_KEY}:signup_ref:{normalized}" + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _to_aware_utc(dt: datetime) -> datetime: + if dt.tzinfo is None: + return dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) + + +def _code_is_usable(row: SignupReferenceCode, *, now: datetime) -> bool: + if not row.is_active: + return False + if row.expires_at is not None and _to_aware_utc(row.expires_at) <= now: + return False + if row.max_uses is not None and row.use_count >= row.max_uses: + return False + return True + + +def validate_reference_code_for_signup(db: Session, code: Optional[str]) -> SignupReferenceCode: + if not code or not code.strip(): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="A valid reference code is required to sign up.", + ) + + code_hash = hash_reference_code(code) + row = ( + db.query(SignupReferenceCode) + .filter(SignupReferenceCode.code_hash == code_hash) + .first() + ) + now = datetime.now(timezone.utc) + if row is None or not _code_is_usable(row, now=now): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="A valid reference code is required to sign up.", + ) + return row + + +def consume_reference_code(db: Session, row: SignupReferenceCode) -> None: + row.use_count = (row.use_count or 0) + 1 diff --git a/app/services/storage/gcs_service.py b/app/services/storage/gcs_service.py index 07213e62..f78acb66 100644 --- a/app/services/storage/gcs_service.py +++ b/app/services/storage/gcs_service.py @@ -1,606 +1,606 @@ -"""GCS service for handling audio file storage and retrieval from Google Cloud Storage.""" - -import os -import uuid -from datetime import timedelta -from pathlib import Path -from typing import Any, List, Optional, Tuple - -from app.config import settings -from app.core.exceptions import StorageError -from app.services.storage.blob_paths import ( - build_object_key, - content_type_for_format, - get_organization_root_prefix, - normalize_prefix, -) - -_GCS_LIBS: Optional[ - Tuple[Any, Any, Any, Any] -] = None - - -_GCS_EXCEPTION_LIBS: Optional[Tuple[Any, Any, Any]] = None - - -def _get_gcs_exception_types() -> Tuple[Any, Any, Any]: - """Import GCS exception types without requiring google.cloud.storage.""" - global _GCS_EXCEPTION_LIBS - if _GCS_EXCEPTION_LIBS is None: - try: - from google.api_core.exceptions import Forbidden, NotFound - except ImportError as exc: - raise ImportError( - "google-cloud-storage is required for GCS blob storage. " - "Install with: pip install 'google-cloud-storage>=2.14.0'" - ) from exc - try: - from google.cloud.exceptions import GoogleCloudError - except ImportError: - GoogleCloudError = Exception - _GCS_EXCEPTION_LIBS = (Forbidden, NotFound, GoogleCloudError) - return _GCS_EXCEPTION_LIBS - -def _get_gcs_libs() -> Tuple[Any, Any, Any, Any]: - """Import google-cloud-storage lazily so S3-only installs can start.""" - global _GCS_LIBS - if _GCS_LIBS is None: - try: - from google.cloud import storage - Forbidden, NotFound, GoogleCloudError = _get_gcs_exception_types() - except ImportError as exc: - raise ImportError( - "google-cloud-storage is required for GCS blob storage. " - "Install with: pip install 'google-cloud-storage>=2.14.0'" - ) from exc - _GCS_LIBS = (storage, Forbidden, NotFound, GoogleCloudError) - return _GCS_LIBS - - -def _resolve_credentials_path() -> Optional[str]: - """Resolve GCS credentials path from config or env (supports relative paths).""" - raw = settings.GCS_CREDENTIALS_PATH or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") - if not raw: - return None - path = Path(raw) - if not path.is_absolute(): - path = Path.cwd() / path - resolved = path.resolve() - return str(resolved) if resolved.exists() else str(path) - - -_GCS_SIGNING_UNAVAILABLE_MSG = ( - "GCS signed URLs require signing credentials. Provide a service account JSON " - "(gcs.credentials_path or GOOGLE_APPLICATION_CREDENTIALS), or configure " - "GKE Workload Identity with the IAM Credentials API enabled and grant " - "roles/iam.serviceAccountTokenCreator to the workload service account on itself. " - "Optional: set gcs.signing_service_account_email when the service account email " - "is not detected from ADC." -) - - -class GcsService: - """Service for managing GCS file storage.""" - - def __init__(self): - """Initialize GCS service with configuration.""" - self.gcs_client = None - self.bucket = None - self._initialization_error = None - self._signing_credentials = None - - @property - def enabled(self) -> bool: - """Get GCS enabled status from settings.""" - return settings.GCS_ENABLED - - @property - def bucket_name(self) -> Optional[str]: - """Get GCS bucket name from settings.""" - return settings.GCS_BUCKET_NAME - - @property - def prefix(self) -> str: - """Get GCS prefix from settings.""" - return normalize_prefix(settings.GCS_PREFIX) - - def _build_client(self): - """Create a GCS client using configured or default credentials.""" - storage, _, _, _ = _get_gcs_libs() - creds_path = _resolve_credentials_path() - if creds_path and Path(creds_path).exists(): - return storage.Client.from_service_account_json( - creds_path, - project=settings.GCS_PROJECT_ID, - ) - if settings.GCS_PROJECT_ID: - return storage.Client(project=settings.GCS_PROJECT_ID) - return storage.Client() - - def _get_signing_credentials(self): - """Return credentials that can sign V4 URLs (service account with private key).""" - if self._signing_credentials is not None: - return self._signing_credentials - - creds_path = _resolve_credentials_path() - if creds_path and Path(creds_path).exists(): - from google.oauth2 import service_account - - self._signing_credentials = service_account.Credentials.from_service_account_file( - creds_path - ) - return self._signing_credentials - - if self.gcs_client is not None: - creds = getattr(self.gcs_client, "_credentials", None) - if creds is not None and getattr(creds, "signer", None) is not None: - self._signing_credentials = creds - return creds - - return None - - def _get_iam_signing_params(self) -> Optional[Tuple[str, str]]: - """Return (service_account_email, access_token) for IAM signBlob URL signing.""" - if self.gcs_client is None: - return None - - creds = getattr(self.gcs_client, "_credentials", None) - if creds is None: - return None - - sa_email = settings.GCS_SIGNING_SERVICE_ACCOUNT_EMAIL or getattr( - creds, "service_account_email", None - ) or getattr(creds, "signer_email", None) - if not sa_email: - return None - - try: - from google.auth.transport import requests as auth_requests - - auth_request = auth_requests.Request() - if not creds.valid: - creds.refresh(auth_request) - token = creds.token - if token: - return sa_email, token - except Exception: - return None - - return None - - def _ensure_initialized(self): - """Lazily initialize GCS client if not already initialized.""" - if self.gcs_client is not None and self.bucket is not None: - return - - if not self.enabled: - return - - if not self.bucket_name: - self._initialization_error = "GCS is enabled but bucket_name is not configured" - return - - try: - self.gcs_client = self._build_client() - self.bucket = self.gcs_client.bucket(self.bucket_name) - - if not self.bucket.exists(): - self._initialization_error = f"GCS bucket '{self.bucket_name}' does not exist" - self.gcs_client = None - self.bucket = None - return - except ImportError as exc: - self._initialization_error = str(exc) - self.gcs_client = None - self.bucket = None - return - except Exception as e: - try: - Forbidden, _, GoogleCloudError = _get_gcs_exception_types() - except ImportError: - Forbidden = () - GoogleCloudError = () - if isinstance(e, Forbidden): - self._initialization_error = ( - f"Access denied to GCS bucket '{self.bucket_name}'. Check credentials." - ) - elif isinstance(e, GoogleCloudError): - self._initialization_error = f"Failed to connect to GCS bucket: {str(e)}" - else: - self._initialization_error = f"Failed to initialize GCS service: {str(e)}" - self.gcs_client = None - self.bucket = None - - def reset_connection(self): - """Reset lazy initialization state (for connection tests).""" - self.gcs_client = None - self.bucket = None - self._initialization_error = None - self._signing_credentials = None - - def is_enabled(self) -> bool: - """Check if GCS is enabled and configured.""" - if not self.enabled: - return False - self._ensure_initialized() - return self.gcs_client is not None and self.bucket is not None - - def get_status_message(self) -> Optional[str]: - """Get status message if there's an initialization error.""" - if not self.enabled: - return None - self._ensure_initialized() - return self._initialization_error - - def _get_key( - self, - file_id: uuid.UUID, - file_format: str, - organization_id: Optional[str] = None, - evaluator_id: Optional[str] = None, - meaningful_id: Optional[str] = None, - ) -> str: - """Generate GCS object key for a file.""" - return build_object_key( - file_id, - file_format, - settings.GCS_PREFIX, - organization_id, - evaluator_id, - meaningful_id, - ) - - def upload_file( - self, - file_content: bytes, - file_id: uuid.UUID, - file_format: str, - organization_id: Optional[str] = None, - evaluator_id: Optional[str] = None, - meaningful_id: Optional[str] = None, - ) -> str: - """Upload file to GCS.""" - self._ensure_initialized() - if not self.is_enabled(): - error_msg = self._initialization_error or "GCS is not enabled or not configured" - raise StorageError(error_msg) - - _, NotFound, GoogleCloudError = _get_gcs_exception_types() - try: - key = self._get_key( - file_id, file_format, organization_id, evaluator_id, meaningful_id - ) - content_type = content_type_for_format(file_format) - blob = self.bucket.blob(key) - blob.upload_from_string(file_content, content_type=content_type) - return key - except GoogleCloudError as e: - raise StorageError(f"Failed to upload file to GCS: {str(e)}") - except Exception as e: - raise StorageError(f"Unexpected error uploading file to GCS: {str(e)}") - - def upload_file_by_key( - self, file_content: bytes, key: str, content_type: str = "audio/mpeg" - ) -> str: - """Upload file to GCS using an explicit key path.""" - self._ensure_initialized() - if not self.is_enabled(): - error_msg = self._initialization_error or "GCS is not enabled or not configured" - raise StorageError(error_msg) - - _, NotFound, GoogleCloudError = _get_gcs_exception_types() - - try: - blob = self.bucket.blob(key) - blob.upload_from_string(file_content, content_type=content_type) - return key - except GoogleCloudError as e: - raise StorageError(f"Failed to upload file to GCS: {str(e)}") - except Exception as e: - raise StorageError(f"Unexpected error uploading file to GCS: {str(e)}") - - def download_file(self, file_id: uuid.UUID, file_format: str) -> bytes: - """Download file from GCS.""" - self._ensure_initialized() - if not self.is_enabled(): - error_msg = self._initialization_error or "GCS is not enabled or not configured" - raise StorageError(error_msg) - - key = self._get_key(file_id, file_format) - return self.download_file_by_key(key) - - def download_file_by_key(self, key: str) -> bytes: - """Download file content from GCS using an explicit key path.""" - self._ensure_initialized() - if not self.is_enabled(): - error_msg = self._initialization_error or "GCS is not enabled or not configured" - raise StorageError(error_msg) - - _, NotFound, GoogleCloudError = _get_gcs_exception_types() - - try: - blob = self.bucket.blob(key) - return blob.download_as_bytes() - except NotFound: - raise StorageError(f"File not found in GCS: {key}") - except GoogleCloudError as e: - raise StorageError(f"Failed to download file from GCS: {str(e)}") - except Exception as e: - raise StorageError(f"Unexpected error downloading file from GCS: {str(e)}") - - def delete_file(self, file_id: uuid.UUID, file_format: str) -> bool: - """Delete file from GCS.""" - self._ensure_initialized() - if not self.is_enabled(): - error_msg = self._initialization_error or "GCS is not enabled or not configured" - raise StorageError(error_msg) - - _, NotFound, GoogleCloudError = _get_gcs_exception_types() - - try: - key = self._get_key(file_id, file_format) - return self.delete_file_by_key(key) - except StorageError: - raise - except GoogleCloudError as e: - raise StorageError(f"Failed to delete file from GCS: {str(e)}") - except Exception as e: - raise StorageError(f"Unexpected error deleting file from GCS: {str(e)}") - - def delete_file_by_key(self, key: str) -> bool: - """Delete file from GCS by key.""" - self._ensure_initialized() - if not self.is_enabled(): - error_msg = self._initialization_error or "GCS is not enabled or not configured" - raise StorageError(error_msg) - - _, NotFound, GoogleCloudError = _get_gcs_exception_types() - - try: - blob = self.bucket.blob(key) - if not blob.exists(): - return False - blob.delete() - return True - except GoogleCloudError as e: - raise StorageError(f"Failed to delete file from GCS: {str(e)}") - except Exception as e: - raise StorageError(f"Unexpected error deleting file from GCS: {str(e)}") - - def delete_keys(self, keys: List[str]) -> tuple[int, List[dict]]: - """Bulk-delete a list of GCS object keys. Returns (deleted_count, errors).""" - if not keys: - return 0, [] - - self._ensure_initialized() - if not self.is_enabled(): - error_msg = self._initialization_error or "GCS is not enabled or not configured" - raise StorageError(error_msg) - - _, NotFound, GoogleCloudError = _get_gcs_exception_types() - deduped = list({k for k in keys if k}) - deleted = 0 - errors: List[dict] = [] - - for key in deduped: - try: - blob = self.bucket.blob(key) - blob.delete() - deleted += 1 - except NotFound: - deleted += 1 - except GoogleCloudError as e: - errors.append({"Key": key, "Code": "DeleteError", "Message": str(e)}) - except Exception as e: - errors.append({"Key": key, "Code": "DeleteError", "Message": str(e)}) - - return deleted, errors - - def delete_keys_by_prefix(self, prefix: str) -> tuple[int, List[dict]]: - """List and bulk-delete every object whose key starts with ``prefix``.""" - if not prefix: - return 0, [] - - self._ensure_initialized() - if not self.is_enabled(): - error_msg = self._initialization_error or "GCS is not enabled or not configured" - raise StorageError(error_msg) - - _, NotFound, GoogleCloudError = _get_gcs_exception_types() - keys: List[str] = [] - try: - for blob in self.gcs_client.list_blobs(self.bucket_name, prefix=prefix): - if blob.name: - keys.append(blob.name) - except GoogleCloudError as e: - raise StorageError(f"Failed to list GCS objects under {prefix!r}: {e}") - - if not keys: - return 0, [] - - return self.delete_keys(keys) - - def file_exists(self, file_id: uuid.UUID, file_format: str) -> bool: - """Check if file exists in GCS.""" - self._ensure_initialized() - if not self.is_enabled(): - return False - - try: - key = self._get_key(file_id, file_format) - return self.bucket.blob(key).exists() - except Exception: - return False - - def list_audio_files( - self, - prefix: Optional[str] = None, - max_keys: int = 1000, - organization_id: Optional[str] = None, - ) -> List[dict]: - """List audio files in GCS bucket.""" - self._ensure_initialized() - if not self.is_enabled(): - error_msg = self._initialization_error or "GCS is not enabled or not configured" - raise StorageError(error_msg) - - _, NotFound, GoogleCloudError = _get_gcs_exception_types() - - try: - if organization_id: - search_prefix = f"{self.prefix}organizations/{organization_id}/audio/" - else: - search_prefix = prefix if prefix else self.prefix - - files = [] - for blob in self.gcs_client.list_blobs( - self.bucket_name, prefix=search_prefix, max_results=max_keys - ): - key = blob.name - if any( - key.lower().endswith(f".{fmt}") - for fmt in settings.ALLOWED_AUDIO_FORMATS - ): - updated = blob.updated or blob.time_created - files.append( - { - "key": key, - "size": blob.size or 0, - "last_modified": updated.isoformat() if updated else "", - "filename": Path(key).name, - } - ) - - return files - except GoogleCloudError as e: - raise StorageError(f"Failed to list files in GCS: {str(e)}") - except Exception as e: - raise StorageError(f"Unexpected error listing files in GCS: {str(e)}") - - def get_organization_root_prefix(self, organization_id: str) -> str: - """Get the root GCS prefix for a given organization.""" - return get_organization_root_prefix(settings.GCS_PREFIX, organization_id) - - def browse_folder( - self, - organization_id: str, - path: str = "", - max_keys: int = 1000, - ) -> dict: - """Browse a folder within an organization's GCS namespace.""" - self._ensure_initialized() - if not self.is_enabled(): - error_msg = self._initialization_error or "GCS is not enabled or not configured" - raise StorageError(error_msg) - - org_root = self.get_organization_root_prefix(organization_id) - full_prefix = f"{org_root}{path}" - if full_prefix and not full_prefix.endswith("/"): - full_prefix += "/" - - _, NotFound, GoogleCloudError = _get_gcs_exception_types() - - try: - iterator = self.gcs_client.list_blobs( - self.bucket_name, - prefix=full_prefix, - delimiter="/", - max_results=max_keys, - ) - - folders = [] - files = [] - for page in iterator.pages: - for prefix_name in page.prefixes: - relative = prefix_name[len(org_root):] - folder_name = relative.rstrip("/").rsplit("/", 1)[-1] - folders.append({"name": folder_name, "path": relative}) - - for blob in page: - key = blob.name - if key == full_prefix: - continue - updated = blob.updated or blob.time_created - files.append( - { - "key": key, - "filename": Path(key).name, - "size": blob.size or 0, - "last_modified": updated.isoformat() if updated else "", - } - ) - - return { - "folders": folders, - "files": files, - "current_path": path, - "organization_id": organization_id, - } - except GoogleCloudError as e: - raise StorageError(f"Failed to browse GCS folder: {str(e)}") - except Exception as e: - raise StorageError(f"Unexpected error browsing GCS folder: {str(e)}") - - def generate_presigned_url( - self, file_id: uuid.UUID, file_format: str, expiration: int = 3600 - ) -> str: - """Generate a signed URL for temporary file access.""" - self._ensure_initialized() - if not self.is_enabled(): - error_msg = self._initialization_error or "GCS is not enabled or not configured" - raise StorageError(error_msg) - - key = self._get_key(file_id, file_format) - return self.generate_presigned_url_by_key(key, expiration=expiration) - - def generate_presigned_url_by_key( - self, - key: str, - expiration: int = 3600, - *, - response_content_disposition: str | None = None, - ) -> str: - """Generate a signed URL for temporary file access by key.""" - self._ensure_initialized() - if not self.is_enabled(): - error_msg = self._initialization_error or "GCS is not enabled or not configured" - raise StorageError(error_msg) - - _, NotFound, GoogleCloudError = _get_gcs_exception_types() - - credentials = self._get_signing_credentials() - iam_params = None if credentials is not None else self._get_iam_signing_params() - if credentials is None and iam_params is None: - raise StorageError(_GCS_SIGNING_UNAVAILABLE_MSG) - - signed_url_kwargs: dict = { - "version": "v4", - "expiration": timedelta(seconds=expiration), - "method": "GET", - } - if response_content_disposition: - signed_url_kwargs["response_disposition"] = response_content_disposition - - try: - blob = self.bucket.blob(key) - if credentials is not None: - url = blob.generate_signed_url( - credentials=credentials, - **signed_url_kwargs, - ) - else: - sa_email, access_token = iam_params - url = blob.generate_signed_url( - service_account_email=sa_email, - access_token=access_token, - **signed_url_kwargs, - ) - return url - except GoogleCloudError as e: - raise StorageError(f"Failed to generate signed URL: {str(e)}") - except Exception as e: - raise StorageError(f"Unexpected error generating signed URL: {str(e)}") - - -# Singleton instance -gcs_service = GcsService() +"""GCS service for handling audio file storage and retrieval from Google Cloud Storage.""" + +import os +import uuid +from datetime import timedelta +from pathlib import Path +from typing import Any, List, Optional, Tuple + +from app.config import settings +from app.core.exceptions import StorageError +from app.services.storage.blob_paths import ( + build_object_key, + content_type_for_format, + get_organization_root_prefix, + normalize_prefix, +) + +_GCS_LIBS: Optional[ + Tuple[Any, Any, Any, Any] +] = None + + +_GCS_EXCEPTION_LIBS: Optional[Tuple[Any, Any, Any]] = None + + +def _get_gcs_exception_types() -> Tuple[Any, Any, Any]: + """Import GCS exception types without requiring google.cloud.storage.""" + global _GCS_EXCEPTION_LIBS + if _GCS_EXCEPTION_LIBS is None: + try: + from google.api_core.exceptions import Forbidden, NotFound + except ImportError as exc: + raise ImportError( + "google-cloud-storage is required for GCS blob storage. " + "Install with: pip install 'google-cloud-storage>=2.14.0'" + ) from exc + try: + from google.cloud.exceptions import GoogleCloudError + except ImportError: + GoogleCloudError = Exception + _GCS_EXCEPTION_LIBS = (Forbidden, NotFound, GoogleCloudError) + return _GCS_EXCEPTION_LIBS + +def _get_gcs_libs() -> Tuple[Any, Any, Any, Any]: + """Import google-cloud-storage lazily so S3-only installs can start.""" + global _GCS_LIBS + if _GCS_LIBS is None: + try: + from google.cloud import storage + Forbidden, NotFound, GoogleCloudError = _get_gcs_exception_types() + except ImportError as exc: + raise ImportError( + "google-cloud-storage is required for GCS blob storage. " + "Install with: pip install 'google-cloud-storage>=2.14.0'" + ) from exc + _GCS_LIBS = (storage, Forbidden, NotFound, GoogleCloudError) + return _GCS_LIBS + + +def _resolve_credentials_path() -> Optional[str]: + """Resolve GCS credentials path from config or env (supports relative paths).""" + raw = settings.GCS_CREDENTIALS_PATH or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + if not raw: + return None + path = Path(raw) + if not path.is_absolute(): + path = Path.cwd() / path + resolved = path.resolve() + return str(resolved) if resolved.exists() else str(path) + + +_GCS_SIGNING_UNAVAILABLE_MSG = ( + "GCS signed URLs require signing credentials. Provide a service account JSON " + "(gcs.credentials_path or GOOGLE_APPLICATION_CREDENTIALS), or configure " + "GKE Workload Identity with the IAM Credentials API enabled and grant " + "roles/iam.serviceAccountTokenCreator to the workload service account on itself. " + "Optional: set gcs.signing_service_account_email when the service account email " + "is not detected from ADC." +) + + +class GcsService: + """Service for managing GCS file storage.""" + + def __init__(self): + """Initialize GCS service with configuration.""" + self.gcs_client = None + self.bucket = None + self._initialization_error = None + self._signing_credentials = None + + @property + def enabled(self) -> bool: + """Get GCS enabled status from settings.""" + return settings.GCS_ENABLED + + @property + def bucket_name(self) -> Optional[str]: + """Get GCS bucket name from settings.""" + return settings.GCS_BUCKET_NAME + + @property + def prefix(self) -> str: + """Get GCS prefix from settings.""" + return normalize_prefix(settings.GCS_PREFIX) + + def _build_client(self): + """Create a GCS client using configured or default credentials.""" + storage, _, _, _ = _get_gcs_libs() + creds_path = _resolve_credentials_path() + if creds_path and Path(creds_path).exists(): + return storage.Client.from_service_account_json( + creds_path, + project=settings.GCS_PROJECT_ID, + ) + if settings.GCS_PROJECT_ID: + return storage.Client(project=settings.GCS_PROJECT_ID) + return storage.Client() + + def _get_signing_credentials(self): + """Return credentials that can sign V4 URLs (service account with private key).""" + if self._signing_credentials is not None: + return self._signing_credentials + + creds_path = _resolve_credentials_path() + if creds_path and Path(creds_path).exists(): + from google.oauth2 import service_account + + self._signing_credentials = service_account.Credentials.from_service_account_file( + creds_path + ) + return self._signing_credentials + + if self.gcs_client is not None: + creds = getattr(self.gcs_client, "_credentials", None) + if creds is not None and getattr(creds, "signer", None) is not None: + self._signing_credentials = creds + return creds + + return None + + def _get_iam_signing_params(self) -> Optional[Tuple[str, str]]: + """Return (service_account_email, access_token) for IAM signBlob URL signing.""" + if self.gcs_client is None: + return None + + creds = getattr(self.gcs_client, "_credentials", None) + if creds is None: + return None + + sa_email = settings.GCS_SIGNING_SERVICE_ACCOUNT_EMAIL or getattr( + creds, "service_account_email", None + ) or getattr(creds, "signer_email", None) + if not sa_email: + return None + + try: + from google.auth.transport import requests as auth_requests + + auth_request = auth_requests.Request() + if not creds.valid: + creds.refresh(auth_request) + token = creds.token + if token: + return sa_email, token + except Exception: + return None + + return None + + def _ensure_initialized(self): + """Lazily initialize GCS client if not already initialized.""" + if self.gcs_client is not None and self.bucket is not None: + return + + if not self.enabled: + return + + if not self.bucket_name: + self._initialization_error = "GCS is enabled but bucket_name is not configured" + return + + try: + self.gcs_client = self._build_client() + self.bucket = self.gcs_client.bucket(self.bucket_name) + + if not self.bucket.exists(): + self._initialization_error = f"GCS bucket '{self.bucket_name}' does not exist" + self.gcs_client = None + self.bucket = None + return + except ImportError as exc: + self._initialization_error = str(exc) + self.gcs_client = None + self.bucket = None + return + except Exception as e: + try: + Forbidden, _, GoogleCloudError = _get_gcs_exception_types() + except ImportError: + Forbidden = () + GoogleCloudError = () + if isinstance(e, Forbidden): + self._initialization_error = ( + f"Access denied to GCS bucket '{self.bucket_name}'. Check credentials." + ) + elif isinstance(e, GoogleCloudError): + self._initialization_error = f"Failed to connect to GCS bucket: {str(e)}" + else: + self._initialization_error = f"Failed to initialize GCS service: {str(e)}" + self.gcs_client = None + self.bucket = None + + def reset_connection(self): + """Reset lazy initialization state (for connection tests).""" + self.gcs_client = None + self.bucket = None + self._initialization_error = None + self._signing_credentials = None + + def is_enabled(self) -> bool: + """Check if GCS is enabled and configured.""" + if not self.enabled: + return False + self._ensure_initialized() + return self.gcs_client is not None and self.bucket is not None + + def get_status_message(self) -> Optional[str]: + """Get status message if there's an initialization error.""" + if not self.enabled: + return None + self._ensure_initialized() + return self._initialization_error + + def _get_key( + self, + file_id: uuid.UUID, + file_format: str, + organization_id: Optional[str] = None, + evaluator_id: Optional[str] = None, + meaningful_id: Optional[str] = None, + ) -> str: + """Generate GCS object key for a file.""" + return build_object_key( + file_id, + file_format, + settings.GCS_PREFIX, + organization_id, + evaluator_id, + meaningful_id, + ) + + def upload_file( + self, + file_content: bytes, + file_id: uuid.UUID, + file_format: str, + organization_id: Optional[str] = None, + evaluator_id: Optional[str] = None, + meaningful_id: Optional[str] = None, + ) -> str: + """Upload file to GCS.""" + self._ensure_initialized() + if not self.is_enabled(): + error_msg = self._initialization_error or "GCS is not enabled or not configured" + raise StorageError(error_msg) + + _, NotFound, GoogleCloudError = _get_gcs_exception_types() + try: + key = self._get_key( + file_id, file_format, organization_id, evaluator_id, meaningful_id + ) + content_type = content_type_for_format(file_format) + blob = self.bucket.blob(key) + blob.upload_from_string(file_content, content_type=content_type) + return key + except GoogleCloudError as e: + raise StorageError(f"Failed to upload file to GCS: {str(e)}") + except Exception as e: + raise StorageError(f"Unexpected error uploading file to GCS: {str(e)}") + + def upload_file_by_key( + self, file_content: bytes, key: str, content_type: str = "audio/mpeg" + ) -> str: + """Upload file to GCS using an explicit key path.""" + self._ensure_initialized() + if not self.is_enabled(): + error_msg = self._initialization_error or "GCS is not enabled or not configured" + raise StorageError(error_msg) + + _, NotFound, GoogleCloudError = _get_gcs_exception_types() + + try: + blob = self.bucket.blob(key) + blob.upload_from_string(file_content, content_type=content_type) + return key + except GoogleCloudError as e: + raise StorageError(f"Failed to upload file to GCS: {str(e)}") + except Exception as e: + raise StorageError(f"Unexpected error uploading file to GCS: {str(e)}") + + def download_file(self, file_id: uuid.UUID, file_format: str) -> bytes: + """Download file from GCS.""" + self._ensure_initialized() + if not self.is_enabled(): + error_msg = self._initialization_error or "GCS is not enabled or not configured" + raise StorageError(error_msg) + + key = self._get_key(file_id, file_format) + return self.download_file_by_key(key) + + def download_file_by_key(self, key: str) -> bytes: + """Download file content from GCS using an explicit key path.""" + self._ensure_initialized() + if not self.is_enabled(): + error_msg = self._initialization_error or "GCS is not enabled or not configured" + raise StorageError(error_msg) + + _, NotFound, GoogleCloudError = _get_gcs_exception_types() + + try: + blob = self.bucket.blob(key) + return blob.download_as_bytes() + except NotFound: + raise StorageError(f"File not found in GCS: {key}") + except GoogleCloudError as e: + raise StorageError(f"Failed to download file from GCS: {str(e)}") + except Exception as e: + raise StorageError(f"Unexpected error downloading file from GCS: {str(e)}") + + def delete_file(self, file_id: uuid.UUID, file_format: str) -> bool: + """Delete file from GCS.""" + self._ensure_initialized() + if not self.is_enabled(): + error_msg = self._initialization_error or "GCS is not enabled or not configured" + raise StorageError(error_msg) + + _, NotFound, GoogleCloudError = _get_gcs_exception_types() + + try: + key = self._get_key(file_id, file_format) + return self.delete_file_by_key(key) + except StorageError: + raise + except GoogleCloudError as e: + raise StorageError(f"Failed to delete file from GCS: {str(e)}") + except Exception as e: + raise StorageError(f"Unexpected error deleting file from GCS: {str(e)}") + + def delete_file_by_key(self, key: str) -> bool: + """Delete file from GCS by key.""" + self._ensure_initialized() + if not self.is_enabled(): + error_msg = self._initialization_error or "GCS is not enabled or not configured" + raise StorageError(error_msg) + + _, NotFound, GoogleCloudError = _get_gcs_exception_types() + + try: + blob = self.bucket.blob(key) + if not blob.exists(): + return False + blob.delete() + return True + except GoogleCloudError as e: + raise StorageError(f"Failed to delete file from GCS: {str(e)}") + except Exception as e: + raise StorageError(f"Unexpected error deleting file from GCS: {str(e)}") + + def delete_keys(self, keys: List[str]) -> tuple[int, List[dict]]: + """Bulk-delete a list of GCS object keys. Returns (deleted_count, errors).""" + if not keys: + return 0, [] + + self._ensure_initialized() + if not self.is_enabled(): + error_msg = self._initialization_error or "GCS is not enabled or not configured" + raise StorageError(error_msg) + + _, NotFound, GoogleCloudError = _get_gcs_exception_types() + deduped = list({k for k in keys if k}) + deleted = 0 + errors: List[dict] = [] + + for key in deduped: + try: + blob = self.bucket.blob(key) + blob.delete() + deleted += 1 + except NotFound: + deleted += 1 + except GoogleCloudError as e: + errors.append({"Key": key, "Code": "DeleteError", "Message": str(e)}) + except Exception as e: + errors.append({"Key": key, "Code": "DeleteError", "Message": str(e)}) + + return deleted, errors + + def delete_keys_by_prefix(self, prefix: str) -> tuple[int, List[dict]]: + """List and bulk-delete every object whose key starts with ``prefix``.""" + if not prefix: + return 0, [] + + self._ensure_initialized() + if not self.is_enabled(): + error_msg = self._initialization_error or "GCS is not enabled or not configured" + raise StorageError(error_msg) + + _, NotFound, GoogleCloudError = _get_gcs_exception_types() + keys: List[str] = [] + try: + for blob in self.gcs_client.list_blobs(self.bucket_name, prefix=prefix): + if blob.name: + keys.append(blob.name) + except GoogleCloudError as e: + raise StorageError(f"Failed to list GCS objects under {prefix!r}: {e}") + + if not keys: + return 0, [] + + return self.delete_keys(keys) + + def file_exists(self, file_id: uuid.UUID, file_format: str) -> bool: + """Check if file exists in GCS.""" + self._ensure_initialized() + if not self.is_enabled(): + return False + + try: + key = self._get_key(file_id, file_format) + return self.bucket.blob(key).exists() + except Exception: + return False + + def list_audio_files( + self, + prefix: Optional[str] = None, + max_keys: int = 1000, + organization_id: Optional[str] = None, + ) -> List[dict]: + """List audio files in GCS bucket.""" + self._ensure_initialized() + if not self.is_enabled(): + error_msg = self._initialization_error or "GCS is not enabled or not configured" + raise StorageError(error_msg) + + _, NotFound, GoogleCloudError = _get_gcs_exception_types() + + try: + if organization_id: + search_prefix = f"{self.prefix}organizations/{organization_id}/audio/" + else: + search_prefix = prefix if prefix else self.prefix + + files = [] + for blob in self.gcs_client.list_blobs( + self.bucket_name, prefix=search_prefix, max_results=max_keys + ): + key = blob.name + if any( + key.lower().endswith(f".{fmt}") + for fmt in settings.ALLOWED_AUDIO_FORMATS + ): + updated = blob.updated or blob.time_created + files.append( + { + "key": key, + "size": blob.size or 0, + "last_modified": updated.isoformat() if updated else "", + "filename": Path(key).name, + } + ) + + return files + except GoogleCloudError as e: + raise StorageError(f"Failed to list files in GCS: {str(e)}") + except Exception as e: + raise StorageError(f"Unexpected error listing files in GCS: {str(e)}") + + def get_organization_root_prefix(self, organization_id: str) -> str: + """Get the root GCS prefix for a given organization.""" + return get_organization_root_prefix(settings.GCS_PREFIX, organization_id) + + def browse_folder( + self, + organization_id: str, + path: str = "", + max_keys: int = 1000, + ) -> dict: + """Browse a folder within an organization's GCS namespace.""" + self._ensure_initialized() + if not self.is_enabled(): + error_msg = self._initialization_error or "GCS is not enabled or not configured" + raise StorageError(error_msg) + + org_root = self.get_organization_root_prefix(organization_id) + full_prefix = f"{org_root}{path}" + if full_prefix and not full_prefix.endswith("/"): + full_prefix += "/" + + _, NotFound, GoogleCloudError = _get_gcs_exception_types() + + try: + iterator = self.gcs_client.list_blobs( + self.bucket_name, + prefix=full_prefix, + delimiter="/", + max_results=max_keys, + ) + + folders = [] + files = [] + for page in iterator.pages: + for prefix_name in page.prefixes: + relative = prefix_name[len(org_root):] + folder_name = relative.rstrip("/").rsplit("/", 1)[-1] + folders.append({"name": folder_name, "path": relative}) + + for blob in page: + key = blob.name + if key == full_prefix: + continue + updated = blob.updated or blob.time_created + files.append( + { + "key": key, + "filename": Path(key).name, + "size": blob.size or 0, + "last_modified": updated.isoformat() if updated else "", + } + ) + + return { + "folders": folders, + "files": files, + "current_path": path, + "organization_id": organization_id, + } + except GoogleCloudError as e: + raise StorageError(f"Failed to browse GCS folder: {str(e)}") + except Exception as e: + raise StorageError(f"Unexpected error browsing GCS folder: {str(e)}") + + def generate_presigned_url( + self, file_id: uuid.UUID, file_format: str, expiration: int = 3600 + ) -> str: + """Generate a signed URL for temporary file access.""" + self._ensure_initialized() + if not self.is_enabled(): + error_msg = self._initialization_error or "GCS is not enabled or not configured" + raise StorageError(error_msg) + + key = self._get_key(file_id, file_format) + return self.generate_presigned_url_by_key(key, expiration=expiration) + + def generate_presigned_url_by_key( + self, + key: str, + expiration: int = 3600, + *, + response_content_disposition: str | None = None, + ) -> str: + """Generate a signed URL for temporary file access by key.""" + self._ensure_initialized() + if not self.is_enabled(): + error_msg = self._initialization_error or "GCS is not enabled or not configured" + raise StorageError(error_msg) + + _, NotFound, GoogleCloudError = _get_gcs_exception_types() + + credentials = self._get_signing_credentials() + iam_params = None if credentials is not None else self._get_iam_signing_params() + if credentials is None and iam_params is None: + raise StorageError(_GCS_SIGNING_UNAVAILABLE_MSG) + + signed_url_kwargs: dict = { + "version": "v4", + "expiration": timedelta(seconds=expiration), + "method": "GET", + } + if response_content_disposition: + signed_url_kwargs["response_disposition"] = response_content_disposition + + try: + blob = self.bucket.blob(key) + if credentials is not None: + url = blob.generate_signed_url( + credentials=credentials, + **signed_url_kwargs, + ) + else: + sa_email, access_token = iam_params + url = blob.generate_signed_url( + service_account_email=sa_email, + access_token=access_token, + **signed_url_kwargs, + ) + return url + except GoogleCloudError as e: + raise StorageError(f"Failed to generate signed URL: {str(e)}") + except Exception as e: + raise StorageError(f"Unexpected error generating signed URL: {str(e)}") + + +# Singleton instance +gcs_service = GcsService() diff --git a/app/services/telephony/exotel_client.py b/app/services/telephony/exotel_client.py index 60268f5d..5371208a 100644 --- a/app/services/telephony/exotel_client.py +++ b/app/services/telephony/exotel_client.py @@ -102,21 +102,59 @@ def _penalize_if_fingerprinted(self, *, retry_after_seconds: Optional[int] = Non retry_after_seconds=retry_after_seconds, ) - def test_connection(self) -> bool: - """Make a trivial authenticated call to confirm credentials work.""" - url = f"{self._api_base}/v1/Accounts/{self._account_sid}/Calls?PageSize=1" + def _test_connection_on_base(self, api_base: str) -> Optional[Tuple[int, str]]: + """Return None on success, else (status_code, message).""" + url = ( + f"{api_base.rstrip('/')}/v1/Accounts/{self._account_sid}/Calls.json" + f"?PageSize=1" + ) try: with httpx.Client(timeout=self._timeout) as client: resp = client.get(url, auth=self._auth) - if resp.status_code in (401, 403): - raise ExotelAuthError( - f"Exotel auth failed (HTTP {resp.status_code}): {resp.text[:200]}" - ) - resp.raise_for_status() + except httpx.HTTPError as exc: + return (0, str(exc)) + + if resp.status_code in (401, 403): + return ( + resp.status_code, + f"Exotel auth failed (HTTP {resp.status_code}): {resp.text[:200]}", + ) + if resp.status_code == 404: + return ( + 404, + f"Exotel account not found (HTTP 404) at {api_base}: {resp.text[:200]}", + ) + if resp.status_code >= 400: + return ( + resp.status_code, + f"Exotel request failed (HTTP {resp.status_code}): {resp.text[:200]}", + ) + return None + + def test_connection(self) -> bool: + """Make a trivial authenticated call to confirm credentials work.""" + failure = self._test_connection_on_base(self._api_base) + if failure is None: return True - except (ExotelAuthError, httpx.HTTPError) as e: - logger.exception("Exotel test_connection failed") - raise ValueError(f"Failed to connect to Exotel: {str(e)}") + + status_code, message = failure + if status_code in (401, 403, 404) and self._api_base.rstrip("/") == DEFAULT_API_BASE: + india_base = "https://api.in.exotel.com" + if self._test_connection_on_base(india_base) is None: + raise ValueError( + "Exotel credentials are valid on api.in.exotel.com (India) but " + "this integration uses the default Singapore API host " + "(api.exotel.com). Set API Host to api.in.exotel.com in " + "Integrations and retry." + ) + + if status_code in (401, 403): + logger.warning("Exotel test_connection auth failed: {}", message) + raise ExotelAuthError( + f"{message} (API host: {self._api_base})" + ) + logger.warning("Exotel test_connection failed: {}", message) + raise ValueError(f"Failed to connect to Exotel at {self._api_base}: {message}") def list_incoming_phone_numbers(self) -> List[Dict[str, Any]]: """List incoming phone numbers (ExoPhones) on the account.""" diff --git a/app/workers/concurrency/eval_dispatch.py b/app/workers/concurrency/eval_dispatch.py index f2dd59d7..a86a1722 100644 --- a/app/workers/concurrency/eval_dispatch.py +++ b/app/workers/concurrency/eval_dispatch.py @@ -98,8 +98,18 @@ def _diarisation_in_flight(source_row: CallImportRow) -> bool: return bool((source_row.celery_task_id or "").strip()) -def _needs_import_for_eval(source_row: CallImportRow) -> bool: +def _needs_import_for_eval( + source_row: CallImportRow, + evaluation: CallImportEvaluation | None = None, +) -> bool: """True when the eval pipeline must fetch a recording before later stages.""" + if evaluation is not None and ( + (getattr(evaluation, "transcript_source", None) or "") + .strip() + .lower() + == "production" + ): + return False if source_row.status in ( CallImportRowStatus.COMPLETED, CallImportRowStatus.FAILED, @@ -156,8 +166,14 @@ def source_row_import_blocks_eval(source_row: CallImportRow) -> bool: def recover_eval_row_for_eval_chain(eval_row: CallImportEvaluationRow) -> None: """Undo a premature eval-row failure so the eval chain can continue.""" + from app.workers.tasks.evaluate_call_import_row_core import ( + EVAL_CANCELLED_BY_USER_ERROR, + ) + if eval_row.status != "failed": return + if (eval_row.error_message or "") == EVAL_CANCELLED_BY_USER_ERROR: + return eval_row.status = "pending" eval_row.error_message = None eval_row.finished_at = None @@ -252,6 +268,13 @@ def enqueue_eval_chain_transcribe_after_import( recover_eval_row_for_eval_chain(eval_row) + from app.workers.tasks.evaluate_call_import_row_core import ( + is_eval_row_user_cancelled, + ) + + if is_eval_row_user_cancelled(eval_row): + return False + with shard_row_write_context(db): source_row.diarised_transcript_status = "pending" source_row.diarised_transcript_error = None @@ -437,7 +460,7 @@ def _try_dispatch_single_row( if not (source_row.recording_s3_key or "").strip(): return EvalDispatchOutcome("skip") - if _needs_import_for_eval(source_row): + if _needs_import_for_eval(source_row, evaluation): from app.workers.concurrency.import_dispatch import ( _peek_authenticated_import_credit, ) @@ -481,6 +504,13 @@ def _enqueue_import(reserved_task_id: str): transcribe_overwrite=transcribe_overwrite, auto_transcribe=auto_transcribe, ): + from app.workers.tasks.evaluate_call_import_row_core import ( + is_eval_row_user_cancelled, + ) + + if is_eval_row_user_cancelled(eval_row): + return EvalDispatchOutcome("skip") + if _diarisation_in_flight(source_row): return EvalDispatchOutcome("skip") diff --git a/app/workers/concurrency/fair_dispatch.py b/app/workers/concurrency/fair_dispatch.py index 55e4bbc7..dc37cf19 100644 --- a/app/workers/concurrency/fair_dispatch.py +++ b/app/workers/concurrency/fair_dispatch.py @@ -17,6 +17,7 @@ CallImportEvaluation, CallImportEvaluationRow, CallImportRow, + Workspace, ) from app.workers.concurrency.eval_dispatch import ( DISPATCH_QUEUE, @@ -216,6 +217,7 @@ def _workspaces_with_pending_rows( CallImportEvaluationRow, CallImportEvaluationRow.evaluation_id == CallImportEvaluation.id, ) + .join(Workspace, Workspace.id == CallImportEvaluation.workspace_id) .filter( # Pending rows are authoritative — a run can stay ``partial`` # (or even ``completed``) while retry resets rows back to @@ -223,6 +225,7 @@ def _workspaces_with_pending_rows( CallImportEvaluation.status != "cancelled", CallImportEvaluationRow.status == "pending", CallImportEvaluationRow.celery_task_id.is_(None), + Workspace.is_active.is_(True), ) .distinct() .all() diff --git a/app/workers/concurrency/fair_import_dispatch.py b/app/workers/concurrency/fair_import_dispatch.py index 04d8f804..2ae70481 100644 --- a/app/workers/concurrency/fair_import_dispatch.py +++ b/app/workers/concurrency/fair_import_dispatch.py @@ -11,7 +11,7 @@ from app.config import settings from app.database import SessionLocal -from app.models.database import CallImport, CallImportRow +from app.models.database import CallImport, CallImportRow, Workspace from app.models.enums import CallImportRowStatus, CallImportStatus from app.workers.concurrency.eval_dispatch import IMPORTS_QUEUE from app.workers.concurrency.import_dispatch import _try_dispatch_single_import_row @@ -85,10 +85,12 @@ def _workspaces_with_pending_imports(db: Session) -> List[UUID]: rows = ( db.query(CallImport.workspace_id) .join(CallImportRow, CallImportRow.call_import_id == CallImport.id) + .join(Workspace, Workspace.id == CallImport.workspace_id) .filter( CallImport.status != CallImportStatus.DELETING, CallImportRow.status == CallImportRowStatus.PENDING, CallImportRow.celery_task_id.is_(None), + Workspace.is_active.is_(True), ) .distinct() .all() diff --git a/app/workers/config.py b/app/workers/config.py index dc1cb711..f4e44f36 100644 --- a/app/workers/config.py +++ b/app/workers/config.py @@ -135,6 +135,7 @@ "generate_evaluation_user_insights": {"queue": "evaluations"}, "generate_evaluation_metric_clusters": {"queue": "evaluations"}, "generate_evaluation_prompt_improvements": {"queue": "evaluations"}, + "evaluate_studio_run_item": {"queue": "evaluations"}, "generate_agent_flowchart": {"queue": "celery"}, "map_agent_flowchart_prompt_sections": {"queue": "celery"}, } diff --git a/app/workers/tasks/__init__.py b/app/workers/tasks/__init__.py index e4862d8e..e60a9ad2 100644 --- a/app/workers/tasks/__init__.py +++ b/app/workers/tasks/__init__.py @@ -21,6 +21,7 @@ from . import agent_flowchart_jobs from . import initiate_vobiz_outbound from . import finalize_telephony_recording +from . import evaluate_studio_run_item from . import call_import_bulk_ops from app.workers.concurrency import eval_dispatch from app.workers.concurrency import fair_dispatch @@ -52,6 +53,7 @@ "dispatch_evaluation_rows_task", "dispatch_fair_eval_rows_task", "dispatch_fair_diarization_rows_task", + "evaluate_studio_run_item_task", "dispatch_fair_import_rows_task", "bulk_diarize_call_import_task", "bulk_delete_call_import_rows_task", @@ -112,6 +114,9 @@ call_import_bulk_ops.materialize_call_import_rows_task ) delete_call_import_task = call_import_bulk_ops.delete_call_import_task +evaluate_studio_run_item_task = ( + evaluate_studio_run_item.evaluate_studio_run_item_task +) materialize_call_import_evaluation_task = ( call_import_bulk_ops.materialize_call_import_evaluation_task ) diff --git a/app/workers/tasks/evaluate_call_import_row_core.py b/app/workers/tasks/evaluate_call_import_row_core.py index 81f71199..98d9d8f2 100644 --- a/app/workers/tasks/evaluate_call_import_row_core.py +++ b/app/workers/tasks/evaluate_call_import_row_core.py @@ -6,7 +6,7 @@ from typing import Any, List, Optional from uuid import UUID -from sqlalchemy import case, func, update +from sqlalchemy import case, func, or_, update from sqlalchemy.orm import Session from app.models.database import ( @@ -51,6 +51,11 @@ def was_cancelled_externally(db, eval_row: CallImportEvaluationRow) -> bool: db.refresh(eval_row, attribute_names=["status", "error_message"]) except Exception: # noqa: BLE001 return False + return is_eval_row_user_cancelled(eval_row) + + +def is_eval_row_user_cancelled(eval_row: CallImportEvaluationRow) -> bool: + """True when the row was aborted by the operator.""" return ( (eval_row.status or "").lower() == "failed" and (eval_row.error_message or "") == EVAL_CANCELLED_BY_USER_ERROR @@ -324,6 +329,18 @@ def _apply_parent_status_from_counters( failed = int(evaluation.failed_rows or 0) in_progress = total - completed - failed + if (evaluation.status or "").strip().lower() == "cancelled": + if in_progress > 0: + return + evaluation.finished_at = now_utc() + if total == 0 or failed == 0: + evaluation.status = "completed" + elif completed == 0: + evaluation.status = "failed" + else: + evaluation.status = "partial" + return + if in_progress > 0: evaluation.status = "running" if not evaluation.started_at: @@ -489,6 +506,7 @@ def load_enabled_metrics( Metric.organization_id == evaluation.organization_id, Metric.id.in_(metric_ids), Metric.enabled.is_(True), + or_(Metric.lifecycle.is_(None), Metric.lifecycle == "active"), ) .all() ) diff --git a/app/workers/tasks/evaluate_studio_run_item.py b/app/workers/tasks/evaluate_studio_run_item.py new file mode 100644 index 00000000..96a56d53 --- /dev/null +++ b/app/workers/tasks/evaluate_studio_run_item.py @@ -0,0 +1,280 @@ +"""Celery task: evaluate one Metrics Studio run result.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any +from uuid import UUID + +from loguru import logger +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.database import SessionLocal +from app.models.database import ( + AIProvider, + MetricStudioRun, + MetricStudioRunResult, +) +from app.services.metric_studio.metric_selection import load_studio_run_metrics +from app.services.metric_studio.source_resolver import resolve_source +from app.workers.config import celery_app +from app.workers.tasks.evaluate_call_import_row_core import ( + build_parent_groups, + categorize_metrics, +) +from app.workers.tasks.helpers.audio_evaluation import ( + evaluate_audio_metrics, + handle_audio_evaluation_error, +) +from app.workers.tasks.helpers.llm_evaluation import ( + evaluate_with_llm, + handle_llm_evaluation_error, +) + + +def _now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def _rollup_run(db: Session, run: MetricStudioRun) -> None: + results = ( + db.query(MetricStudioRunResult) + .filter(MetricStudioRunResult.run_id == run.id) + .all() + ) + completed = sum(1 for r in results if r.status == "completed") + failed = sum(1 for r in results if r.status == "failed") + pending = sum(1 for r in results if r.status in {"pending", "running"}) + run.completed_items = completed + run.failed_items = failed + if pending: + run.status = "running" + elif failed and completed: + run.status = "partial" + run.finished_at = _now_utc() + elif failed: + run.status = "failed" + run.finished_at = _now_utc() + else: + run.status = "completed" + run.finished_at = _now_utc() + db.commit() + + +@celery_app.task( + bind=True, + name="evaluate_studio_run_item", + max_retries=0, +) +def evaluate_studio_run_item_task(self, result_row_id: str) -> dict[str, Any]: + db = SessionLocal() + try: + try: + row_uuid = UUID(result_row_id) + except ValueError: + return {"status": "error", "detail": "invalid result id"} + + result_row = ( + db.query(MetricStudioRunResult) + .filter(MetricStudioRunResult.id == row_uuid) + .first() + ) + if not result_row: + return {"status": "error", "detail": "result not found"} + + run = ( + db.query(MetricStudioRun) + .filter(MetricStudioRun.id == result_row.run_id) + .first() + ) + if not run: + return {"status": "error", "detail": "run not found"} + + result_row.status = "running" + result_row.started_at = result_row.started_at or _now_utc() + db.commit() + + sample = resolve_source( + db, + organization_id=run.organization_id, + workspace_id=run.workspace_id, + source_kind=result_row.source_kind, + source_ref=result_row.source_ref, + display_label=result_row.display_label, + ) + + transcript_source = (run.transcript_source or "diarised").lower() + if transcript_source == "production": + transcript = sample.transcript + else: + transcript = sample.diarised_transcript or sample.transcript + + metric_ids = [] + for item in run.selected_metric_ids or []: + try: + metric_ids.append(UUID(str(item))) + except (TypeError, ValueError): + continue + + metrics = load_studio_run_metrics(db, run.organization_id, metric_ids) + has_audio = bool(sample.audio_s3_key) + has_production = bool((sample.transcript or "").strip()) + has_diarised = bool((sample.diarised_transcript or "").strip()) + + transcript_metrics, audio_metrics, comparison_metrics, skipped = categorize_metrics( + metrics, + has_audio, + has_production_transcript=has_production, + has_diarised_transcript=has_diarised, + ) + llm_metrics = transcript_metrics + comparison_metrics + metric_scores: dict[str, Any] = dict(skipped) + + if not transcript and not has_audio: + result_row.status = "failed" + result_row.error_message = "No transcript or audio available for this source." + result_row.finished_at = _now_utc() + db.commit() + _rollup_run(db, run) + return {"status": "failed"} + + ai_providers = ( + db.query(AIProvider) + .filter( + AIProvider.organization_id == run.organization_id, + AIProvider.is_active.is_(True), + ) + .all() + ) + + if audio_metrics and sample.audio_s3_key: + try: + audio_scores = evaluate_audio_metrics( + audio_s3_key=sample.audio_s3_key, + audio_metrics=audio_metrics, + result_id=f"studio:{result_row.id}", + ) + metric_scores.update(audio_scores) + except Exception as audio_err: + logger.error( + f"[MetricStudio {result_row.id}] audio evaluation failed: {audio_err}", + exc_info=True, + ) + metric_scores.update( + handle_audio_evaluation_error(audio_metrics, audio_err) + ) + + if llm_metrics and transcript: + parents_by_id, children_by_parent, standalone = build_parent_groups( + db, llm_metrics + ) + result_id = f"studio:{result_row.id}" + + for parent_id, children in children_by_parent.items(): + parent = parents_by_id.get(parent_id) + if not parent or not children: + continue + try: + comparison_pair = None + if any(getattr(m, "compare_transcripts", False) for m in children): + comparison_pair = ( + sample.transcript or "", + sample.diarised_transcript or sample.transcript or "", + ) + scores, _ = evaluate_with_llm( + transcription=transcript, + llm_metrics=children, + ai_providers=ai_providers, + organization_id=run.organization_id, + result_id=result_id, + db=db, + parent_metric=parent, + comparison_pair=comparison_pair, + ) + metric_scores.update(scores) + except Exception as llm_err: + metric_scores.update( + handle_llm_evaluation_error(children, llm_err) + ) + + if standalone: + try: + comparison_standalone = [ + m + for m in standalone + if getattr(m, "compare_transcripts", False) + ] + transcript_standalone = [ + m for m in standalone if m not in comparison_standalone + ] + if transcript_standalone: + scores, _ = evaluate_with_llm( + transcription=transcript, + llm_metrics=transcript_standalone, + ai_providers=ai_providers, + organization_id=run.organization_id, + result_id=result_id, + db=db, + ) + metric_scores.update(scores) + for metric in comparison_standalone: + scores, _ = evaluate_with_llm( + transcription=transcript, + llm_metrics=[metric], + ai_providers=ai_providers, + organization_id=run.organization_id, + result_id=result_id, + db=db, + comparison_pair=( + sample.transcript or "", + sample.diarised_transcript or sample.transcript or "", + ), + ) + metric_scores.update(scores) + except Exception as llm_err: + metric_scores.update(handle_llm_evaluation_error(standalone, llm_err)) + + result_row.metric_scores = metric_scores + flag_modified(result_row, "metric_scores") + metadata = dict(result_row.source_metadata or sample.metadata or {}) + metadata.update(sample.metadata or {}) + if transcript: + metadata["evaluation_transcript"] = transcript + metadata["transcript_source_used"] = transcript_source + result_row.source_metadata = metadata + flag_modified(result_row, "source_metadata") + result_row.status = "completed" + result_row.error_message = None + result_row.finished_at = _now_utc() + db.commit() + _rollup_run(db, run) + return {"status": "completed", "scores": len(metric_scores)} + except Exception as exc: + logger.error( + f"[MetricStudio] evaluate_studio_run_item failed: {exc}", + exc_info=True, + ) + try: + result_row = ( + db.query(MetricStudioRunResult) + .filter(MetricStudioRunResult.id == UUID(result_row_id)) + .first() + ) + if result_row: + result_row.status = "failed" + result_row.error_message = str(exc) + result_row.finished_at = _now_utc() + db.commit() + run = ( + db.query(MetricStudioRun) + .filter(MetricStudioRun.id == result_row.run_id) + .first() + ) + if run: + _rollup_run(db, run) + except Exception: + db.rollback() + raise + finally: + db.close() diff --git a/app/workers/tasks/process_call_import_row.py b/app/workers/tasks/process_call_import_row.py index 2c6d482e..d3299b3d 100644 --- a/app/workers/tasks/process_call_import_row.py +++ b/app/workers/tasks/process_call_import_row.py @@ -31,6 +31,50 @@ _RETRYABLE_COUNTDOWN_SECONDS = 60 +_CREDENTIAL_AUTH_FAIL_ATTEMPT_THRESHOLD = 2 + + +def _is_credentialed_auth_rejection(exc: Exception) -> bool: + message = str(exc).lower() + return "rejected credentials" in message and ("401" in message or "403" in message) + + +def _terminal_transient_failure_message(self, row, exc: Exception) -> Optional[str]: + max_retries = getattr(self, "max_retries", None) + if max_retries is None: + max_retries = 3 + retries = getattr(getattr(self, "request", None), "retries", 0) + if _is_credentialed_auth_rejection(exc) and ( + row.attempts or 0 + ) >= _CREDENTIAL_AUTH_FAIL_ATTEMPT_THRESHOLD: + return f"Telephony credentials rejected: {exc}" + if retries >= max_retries: + return f"Recording fetch failed after {max_retries + 1} attempts: {exc}" + return None + + +def _mark_row_failed_for_fetch( + *, + row, + db, + row_id: str, + catalog_db, + row_db, + call_import, + message: str, + context: str, + reason: str = "non_retryable_provider_error", +): + from app.models.enums import CallImportRowStatus + + row.status = CallImportRowStatus.FAILED + row.error_message = message + if not _safe_commit(db, row_id=row_id, context=context): + return {"status": "skipped", "reason": "row_deleted"} + _rollup_parent_on_catalog(catalog_db, row_db, call_import) + if not _safe_commit(db, row_id=row_id, context=f"rollup_{context}"): + return {"status": "skipped", "reason": "row_deleted"} + return {"status": "failed", "reason": reason} def _retry_countdown_for_error(exc: Exception) -> int: @@ -45,9 +89,33 @@ def _retry_countdown_for_error(exc: Exception) -> int: return _RETRYABLE_COUNTDOWN_SECONDS -def _schedule_transient_retry(self, *, row, db, row_id: str, exc: Exception, context: str): +def _schedule_transient_retry( + self, + *, + row, + db, + row_id: str, + exc: Exception, + context: str, + catalog_db=None, + row_db=None, + call_import=None, +): from app.models.enums import CallImportRowStatus + terminal_message = _terminal_transient_failure_message(self, row, exc) + if terminal_message is not None and call_import is not None: + return _mark_row_failed_for_fetch( + row=row, + db=db, + row_id=row_id, + catalog_db=catalog_db, + row_db=row_db, + call_import=call_import, + message=terminal_message, + context=context, + ) + row.status = CallImportRowStatus.PENDING row.error_message = f"Transient: {exc}" if not _safe_commit(db, row_id=row_id, context=context): @@ -315,6 +383,46 @@ def process_call_import_row_task( original_csv_url = (row.recording_url or "").strip() or None + requires_recording_url = bool((call_import.provider or "").strip()) + if not original_csv_url: + if requires_recording_url: + provider_key = (call_import.provider or "").lower() + if provider_key == "exotel": + msg = ( + "Cannot fetch recording: Exotel import requires a " + "recording URL on each row." + ) + else: + msg = ( + "Cannot fetch recording: row has no recording URL." + ) + logger.warning("{} (row {})", msg, row_id) + row.status = CallImportRowStatus.FAILED + row.error_message = msg + if not _safe_commit(db, row_id=row_id, context="no_recording_source"): + return {"status": "skipped", "reason": "row_deleted"} + _rollup_parent_on_catalog(catalog_db, row_db, call_import) + if not _safe_commit( + db, row_id=row_id, context="rollup_no_recording_source" + ): + return {"status": "skipped", "reason": "row_deleted"} + return {"status": "failed", "reason": "no_recording_source"} + + row.status = CallImportRowStatus.COMPLETED + row.error_message = None + if not _safe_commit(db, row_id=row_id, context="mark_completed_no_recording"): + return {"status": "skipped", "reason": "row_deleted"} + _rollup_parent_on_catalog(catalog_db, row_db, call_import) + if not _safe_commit( + db, row_id=row_id, context="rollup_completed_no_recording" + ): + return {"status": "skipped", "reason": "row_deleted"} + return { + "status": "completed", + "row_id": row_id, + "s3_key": None, + } + # ------------------------------------------------------------------ # Direct-URL mode — download from the CSV-supplied URL only. # ------------------------------------------------------------------ @@ -324,21 +432,6 @@ def process_call_import_row_task( direct_failure: Optional[Exception] = None direct_was_transient = False - if not original_csv_url: - msg = ( - "Cannot fetch recording: direct URL import requires a " - "recording URL on each row." - ) - logger.warning("{} (row {})", msg, row_id) - row.status = CallImportRowStatus.FAILED - row.error_message = msg - if not _safe_commit(db, row_id=row_id, context="direct_url_no_source"): - return {"status": "skipped", "reason": "row_deleted"} - _rollup_parent_on_catalog(catalog_db, row_db, call_import) - if not _safe_commit(db, row_id=row_id, context="rollup_direct_url_no_source"): - return {"status": "skipped", "reason": "row_deleted"} - return {"status": "failed", "reason": "no_recording_source"} - try: fetched = download_public_recording(original_csv_url) audio_bytes, content_type = fetched @@ -367,6 +460,20 @@ def process_call_import_row_task( if audio_bytes is None: if direct_was_transient: + terminal_message = _terminal_transient_failure_message( + self, row, direct_failure + ) + if terminal_message is not None: + return _mark_row_failed_for_fetch( + row=row, + db=db, + row_id=row_id, + catalog_db=catalog_db, + row_db=row_db, + call_import=call_import, + message=terminal_message, + context="direct_url_transient_exhausted", + ) row.status = CallImportRowStatus.PENDING row.error_message = f"Transient: {direct_failure}" if not _safe_commit(db, row_id=row_id, context="direct_url_transient"): @@ -399,27 +506,6 @@ def process_call_import_row_task( download_failure: Optional[Exception] = None download_was_transient = False - if not original_csv_url: - provider_key = (call_import.provider or "").lower() - if provider_key == "exotel": - msg = ( - "Cannot fetch recording: Exotel import requires a " - "recording URL on each row." - ) - else: - msg = ( - "Cannot fetch recording: row has no recording URL." - ) - logger.warning("{} (row {})", msg, row_id) - row.status = CallImportRowStatus.FAILED - row.error_message = msg - if not _safe_commit(db, row_id=row_id, context="no_recording_source"): - return {"status": "skipped", "reason": "row_deleted"} - _rollup_parent_on_catalog(catalog_db, row_db, call_import) - if not _safe_commit(db, row_id=row_id, context="rollup_no_recording_source"): - return {"status": "skipped", "reason": "row_deleted"} - return {"status": "failed", "reason": "no_recording_source"} - if _use_credentialed_recording_download(call_import, client): throttle_exc = _consume_authenticated_import_credit(client) if throttle_exc is not None: @@ -430,6 +516,9 @@ def process_call_import_row_task( row_id=row_id, exc=throttle_exc, context="credential_throttled", + catalog_db=catalog_db, + row_db=row_db, + call_import=call_import, ) try: @@ -472,6 +561,9 @@ def process_call_import_row_task( row_id=row_id, exc=download_failure, context="transient_retry", + catalog_db=catalog_db, + row_db=row_db, + call_import=call_import, ) row.status = CallImportRowStatus.FAILED row.error_message = ( @@ -512,6 +604,19 @@ def process_call_import_row_task( s3_service.upload_file_by_key(audio_bytes, key, content_type=content_type) except Exception as exc: logger.exception("Failed to upload recording to S3 for row {}", row_id) + terminal_message = _terminal_transient_failure_message(self, row, exc) + if terminal_message is not None: + return _mark_row_failed_for_fetch( + row=row, + db=db, + row_id=row_id, + catalog_db=catalog_db, + row_db=row_db, + call_import=call_import, + message=f"S3 upload failed after retries: {exc}", + context="s3_upload_exhausted", + reason="s3_upload_failed", + ) row.error_message = f"S3 upload failed: {exc}" row.status = CallImportRowStatus.PENDING if not _safe_commit(db, row_id=row_id, context="s3_upload_retry"): @@ -610,7 +715,11 @@ def process_call_import_row_task( recover_eval_row_for_eval_chain, source_row_import_blocks_eval, ) + from app.workers.tasks.evaluate_call_import_row_core import ( + EVAL_CANCELLED_BY_USER_ERROR, + ) + user_cancelled = False try: cleanup_row_db, cleanup_catalog_db, eval_row, source_row, _ = ( locate_call_import_evaluation_row( @@ -621,7 +730,14 @@ def process_call_import_row_task( pass else: try: - if source_row_import_blocks_eval(source_row): + user_cancelled = ( + (eval_row.status or "").lower() == "failed" + and (eval_row.error_message or "") + == EVAL_CANCELLED_BY_USER_ERROR + ) + if user_cancelled: + pass + elif source_row_import_blocks_eval(source_row): if eval_row.status == "pending": _fail_eval_row_for_import( cleanup_row_db, eval_row, source_row @@ -648,7 +764,8 @@ def process_call_import_row_task( # Redispatch after local cleanup when import did not chain # transcription — the transcribe worker releases the slot and # schedules fair dispatch when chained transcribe was enqueued. - finish_eval_work_and_redispatch(slot_task_id) + if not user_cancelled: + finish_eval_work_and_redispatch(slot_task_id) else: from app.workers.concurrency.limits import slot_registered_for_task diff --git a/app/workers/tasks/process_evaluator_result.py b/app/workers/tasks/process_evaluator_result.py index d338f294..e19c061b 100644 --- a/app/workers/tasks/process_evaluator_result.py +++ b/app/workers/tasks/process_evaluator_result.py @@ -1,749 +1,751 @@ -"""Celery task: process evaluator result (transcribe and evaluate metrics).""" - -import time -import uuid as _uuid -from uuid import UUID - -from loguru import logger - -from app.database import SessionLocal -from app.models.database import ModelProvider - -from app.workers.config import celery_app -from app.workers.tasks.helpers.constants import ( - REMOVED_EVALUATION_METRIC_NAMES, - AUDIO_ONLY_METRIC_NAMES, -) -from app.workers.tasks.helpers.score_utils import provider_matches, get_metric_type_value -from app.workers.tasks.helpers.audio_evaluation import ( - evaluate_audio_metrics, - handle_audio_evaluation_error, -) -from app.workers.tasks.helpers.llm_evaluation import ( - evaluate_with_llm, - handle_llm_evaluation_error, -) -from app.services.evaluators.evaluator_result_call_data import slim_call_data_for_evaluator_result - - -def _commit_evaluator_result(db, result) -> None: - from app.services.live_entity_storage import sync_evaluator_result - - sync_evaluator_result(db, result) - db.commit() - - -def _make_json_serializable(obj): - """Recursively convert non-JSON-native values (e.g., NumPy types).""" - try: - import numpy as np - except Exception: - np = None - - if isinstance(obj, dict): - return {k: _make_json_serializable(v) for k, v in obj.items()} - if isinstance(obj, list): - return [_make_json_serializable(item) for item in obj] - if isinstance(obj, tuple): - return tuple(_make_json_serializable(item) for item in obj) - - if np is not None: - if isinstance(obj, np.integer): - return int(obj) - if isinstance(obj, np.floating): - return float(obj) - if isinstance(obj, np.ndarray): - return obj.tolist() - if isinstance(obj, np.bool_): - return bool(obj) - - return obj - - -def _load_related_entities(db, result): - """Load evaluator, agent, persona, scenario from database.""" - from app.models.database import Evaluator, Agent, Persona, Scenario - - evaluator = None - if result.evaluator_id: - evaluator = db.query(Evaluator).filter(Evaluator.id == result.evaluator_id).first() - if not evaluator: - logger.warning( - f"[EvaluatorResult {result.result_id}] Evaluator {result.evaluator_id} not found, " - "continuing without evaluator" - ) - - agent = None - if result.agent_id: - agent = db.query(Agent).filter(Agent.id == result.agent_id).first() - - persona = None - if result.persona_id: - persona = db.query(Persona).filter(Persona.id == result.persona_id).first() - - scenario = None - if result.scenario_id: - scenario = db.query(Scenario).filter(Scenario.id == result.scenario_id).first() - - return evaluator, agent, persona, scenario - - -def _transcribe_audio(result, ai_providers, db): - """Transcribe audio file and return transcript with timing info.""" - from app.services.ai.transcription_service import transcription_service - - stt_provider = ModelProvider.OPENAI - stt_model = "whisper-1" - - openai_provider = next( - (p for p in ai_providers if provider_matches(p.provider, ModelProvider.OPENAI)), - None, - ) - if not openai_provider: - logger.warning( - f"[EvaluatorResult {result.result_id}] No OpenAI provider found, using default whisper-1" - ) - - transcription_start_time = time.time() - transcription_result = transcription_service.transcribe( - audio_file_key=result.audio_s3_key, - stt_provider=stt_provider, - stt_model=stt_model, - organization_id=result.organization_id, - db=db, - language=None, - enable_speaker_diarization=True, - ) - transcription_time = time.time() - transcription_start_time - - return ( - transcription_result.get("transcript", ""), - transcription_result.get("speaker_segments", []), - transcription_time, - ) - - -def _generate_call_analysis(transcription, ai_providers, organization_id, result_id, db, agent=None, scenario=None): - """Generate call analysis (summary, sentiment, success) using LLM.""" - from app.services.ai.llm_service import llm_service - - llm_provider = ModelProvider.OPENAI - llm_model = "gpt-4o-mini" - - chosen_provider = next( - (p for p in ai_providers if provider_matches(p.provider, llm_provider)), - None, - ) - if not chosen_provider: - llm_provider = ModelProvider.GOOGLE - llm_model = "gemini-2.5-flash" - chosen_provider = next( - (p for p in ai_providers if provider_matches(p.provider, llm_provider)), - None, - ) - if not chosen_provider: - logger.warning(f"[EvaluatorResult {result_id}] No LLM provider available for call analysis") - return None - - agent_context = "" - if agent and agent.description: - agent_context = f"\n\nAgent Description:\n{agent.description}" - if scenario: - scenario_name = getattr(scenario, 'name', '') - scenario_desc = getattr(scenario, 'description', '') - if scenario_name or scenario_desc: - agent_context += f"\n\nScenario: {scenario_name}" - if scenario_desc: - agent_context += f"\nScenario Description: {scenario_desc}" - - messages = [ - { - "role": "system", - "content": ( - "You are a call analysis expert. Analyze the following conversation transcript " - "and provide a structured analysis. Respond ONLY with valid JSON, no markdown." - ), - }, - { - "role": "user", - "content": f"""Analyze this conversation transcript and provide: -1. A concise summary of the call (2-3 sentences) -2. The user/caller's overall sentiment (one of: Positive, Negative, Neutral, Mixed) -3. Whether the call was successful in achieving its objective (true/false) -{agent_context} - -Transcript: -{transcription} - -Respond in this exact JSON format: -{{"call_summary": "...", "user_sentiment": "...", "call_successful": true/false}}""", - }, - ] - - try: - llm_result = llm_service.generate_response( - messages=messages, - llm_provider=llm_provider, - llm_model=llm_model, - organization_id=organization_id, - db=db, - temperature=0.3, - max_tokens=500, - ) - import json - import re - text = llm_result.get("text", "") - json_match = re.search(r'\{[^{}]*\}', text, re.DOTALL) - if json_match: - analysis = json.loads(json_match.group()) - required_keys = {"call_summary", "user_sentiment", "call_successful"} - if required_keys.issubset(analysis.keys()): - logger.info(f"[EvaluatorResult {result_id}] Call analysis generated successfully") - return analysis - logger.warning(f"[EvaluatorResult {result_id}] Could not parse call analysis from LLM response") - return None - except Exception as e: - logger.error(f"[EvaluatorResult {result_id}] Call analysis failed: {e}", exc_info=True) - return None - - -def _categorize_metrics(enabled_metrics, has_audio): - """Split metrics into LLM-evaluable and audio-only categories.""" - llm_metrics = [] - audio_metrics = [] - skipped_scores = {} - - for m in enabled_metrics: - if m.name.lower() in AUDIO_ONLY_METRIC_NAMES: - if has_audio: - audio_metrics.append(m) - else: - skipped_scores[str(m.id)] = { - "value": None, - "type": get_metric_type_value(m), - "metric_name": m.name, - "skipped": "audio_required", - } - else: - llm_metrics.append(m) - - return llm_metrics, audio_metrics, skipped_scores - - -def _evaluate_llm_metrics_grouped( - *, - transcription: str, - llm_metrics: list, - ai_providers, - organization_id, - result_id: str, - db, - evaluator, - agent, - persona, - scenario, -) -> tuple[dict, float | None]: - """Evaluate LLM metrics, grouping categorization children by parent.""" - from app.workers.tasks.evaluate_call_import_row_core import build_parent_groups - - parents_by_id, children_by_parent, standalone_metrics = build_parent_groups( - db, llm_metrics - ) - metric_scores: dict = {} - evaluation_time: float | None = None - - def _run_bucket(bucket, parent_metric=None): - nonlocal evaluation_time - scores, eval_time = evaluate_with_llm( - transcription=transcription, - llm_metrics=bucket, - ai_providers=ai_providers, - organization_id=organization_id, - result_id=result_id, - db=db, - evaluator=evaluator, - agent=agent, - persona=persona, - scenario=scenario, - parent_metric=parent_metric, - ) - metric_scores.update(scores) - if eval_time is not None: - evaluation_time = eval_time - - if standalone_metrics: - _run_bucket(standalone_metrics, parent_metric=None) - - for parent_id, children in children_by_parent.items(): - parent_metric = parents_by_id.get(parent_id) - if not parent_metric: - logger.warning( - f"[EvaluatorResult {result_id}] Parent metric {parent_id} not found; " - "evaluating children as flat metrics" - ) - _run_bucket(children, parent_metric=None) - continue - _run_bucket(children, parent_metric=parent_metric) - - return metric_scores, evaluation_time - - -def _normalize_platform(platform: object) -> str: - """Normalize provider platform enum/string into lowercase string.""" - if not platform: - return "" - if hasattr(platform, "value"): - return str(platform.value).lower() - return str(platform).lower() - - -def _extract_audio_url(call_data: dict, platform: str) -> str | None: - """Extract provider-specific audio URL from call data.""" - recording_urls = call_data.get("recording_urls", {}) if isinstance(call_data, dict) else {} - provider_payload = call_data.get("provider_payload", {}) if isinstance(call_data, dict) else {} - artifact = call_data.get("artifact", {}) if isinstance(call_data, dict) else {} - recording = artifact.get("recording", {}) if isinstance(artifact, dict) else {} - mono_recording = recording.get("mono", {}) if isinstance(recording, dict) else {} - if platform == "elevenlabs": - return recording_urls.get("conversation_audio") - if platform == "retell": - return call_data.get("recording_url") - if platform == "vapi": - return ( - call_data.get("recordingUrl") - or call_data.get("stereoRecordingUrl") - or artifact.get("recordingUrl") - or artifact.get("stereoRecordingUrl") - or mono_recording.get("combinedUrl") - or recording_urls.get("combined_url") - or recording_urls.get("stereo_url") - or call_data.get("recordingUrl") - or provider_payload.get("recordingUrl") - or provider_payload.get("stereoRecordingUrl") - ) - if platform == "smallest": - return ( - call_data.get("recording_url") - or call_data.get("recordingUrl") - or recording_urls.get("combined_url") - or recording_urls.get("conversation_audio") - ) - return None - - -def _recover_missing_audio_for_result(result, db, refresh_call_data: bool = True) -> bool: - """ - Attempt to recover missing audio from provider, upload to S3, and persist key. - - Returns True when a new S3 key is successfully stored. - """ - import requests as _http - - from app.core.encryption import decrypt_api_key - from app.models.database import Agent, Integration - from app.services.storage.s3_service import s3_service - from app.services.voice_providers import get_voice_provider - - platform = _normalize_platform(result.provider_platform) - if platform not in {"retell", "vapi", "elevenlabs", "smallest"}: - return False - if not result.provider_call_id: - return False - - agent = db.query(Agent).filter(Agent.id == result.agent_id).first() if result.agent_id else None - integration = None - decrypted_key = None - if agent and agent.voice_ai_integration_id: - integration = db.query(Integration).filter( - Integration.id == agent.voice_ai_integration_id, - Integration.organization_id == result.organization_id, - ).first() - if integration: - try: - decrypted_key = decrypt_api_key(integration.api_key) - except Exception as decrypt_err: - logger.warning( - f"[EvaluatorResult {result.result_id}] Unable to decrypt integration key " - f"for audio recovery: {decrypt_err}" - ) - - call_data = result.call_data or {} - if refresh_call_data and decrypted_key: - try: - provider_class = get_voice_provider(platform) - provider_kwargs = {"api_key": decrypted_key} - if platform == "vapi" and integration and getattr(integration, "public_key", None): - provider_kwargs["public_key"] = integration.public_key - provider = provider_class(**provider_kwargs) - if hasattr(provider, "retrieve_call_metrics"): - refreshed = provider.retrieve_call_metrics(result.provider_call_id) - if isinstance(refreshed, dict) and refreshed: - call_data = refreshed - result.call_data = refreshed - db.commit() - except Exception as refresh_err: - logger.warning( - f"[EvaluatorResult {result.result_id}] Audio recovery could not refresh provider metrics: " - f"{refresh_err}" - ) - - audio_url = _extract_audio_url(call_data, platform) - if not audio_url: - logger.warning( - f"[EvaluatorResult {result.result_id}] Audio recovery failed: no provider recording URL available" - ) - return False - - headers = {"xi-api-key": decrypted_key} if platform == "elevenlabs" and decrypted_key else None - try: - response = _http.get(audio_url, headers=headers, timeout=120) - except Exception as download_err: - logger.warning( - f"[EvaluatorResult {result.result_id}] Audio recovery download failed: {download_err}" - ) - return False - - if response.status_code != 200 or not response.content: - logger.warning( - f"[EvaluatorResult {result.result_id}] Audio recovery download returned " - f"status={response.status_code}" - ) - return False - - content_type = response.headers.get("content-type", "audio/mpeg") - ext = "wav" if "wav" in content_type else "mp3" - org_id = str(result.organization_id) - s3_key = ( - f"audio/organizations/{org_id}/evaluations/" - f"{result.provider_call_id}/{_uuid.uuid4()}.{ext}" - ) - - try: - s3_service.upload_file_by_key(response.content, s3_key, content_type=content_type) - except Exception as upload_err: - logger.warning( - f"[EvaluatorResult {result.result_id}] Audio recovery upload failed: {upload_err}" - ) - return False - - result.audio_s3_key = s3_key - _commit_evaluator_result(db, result) - db.refresh(result) - logger.info( - f"[EvaluatorResult {result.result_id}] Recovered missing audio and stored at {s3_key}" - ) - return True - - -def _all_audio_scores_download_failed(audio_scores: dict[str, dict[str, object]]) -> bool: - """Check whether every audio metric failed due to missing/unreadable S3 object.""" - if not audio_scores: - return False - return all( - isinstance(score, dict) and score.get("error") == "audio_download_failed" - for score in audio_scores.values() - ) - - -def _playground_call_recording(db, result): - from app.models.database import CallRecording, CallRecordingSource - - return ( - db.query(CallRecording) - .filter( - CallRecording.evaluator_result_id == result.id, - CallRecording.source == CallRecordingSource.PLAYGROUND, - ) - .first() - ) - - -@celery_app.task(name="process_evaluator_result", bind=True, max_retries=3) -def process_evaluator_result_task(self, result_id: str): - """ - Celery task to process an evaluator result: transcribe audio and evaluate metrics. - - Workflow: - 1. QUEUED -> Job is created and queued - 2. TRANSCRIBING -> Audio is being transcribed - 3. EVALUATING -> Transcription is being evaluated against metrics - 4. COMPLETED -> All processing is complete - 5. FAILED -> An error occurred - """ - db = SessionLocal() - task_start_time = time.time() - - try: - from app.models.database import ( - EvaluatorResult, - EvaluatorResultStatus, - Metric, - AIProvider, - ) - - result_uuid = UUID(result_id) - result = db.query(EvaluatorResult).filter(EvaluatorResult.id == result_uuid).first() - - if not result: - logger.error(f"[EvaluatorResult {result_id}] Job not found in database") - return {"error": "Evaluator result not found"} - - logger.info(f"[EvaluatorResult {result.result_id}] Starting processing task") - - result.celery_task_id = self.request.id - db.commit() - - try: - if not result.audio_s3_key: - _recover_missing_audio_for_result(result, db, refresh_call_data=True) - - has_existing_transcript = bool(result.transcription) - if not result.audio_s3_key and not has_existing_transcript: - raise ValueError("No audio S3 key or existing transcript found") - - evaluator, agent, persona, scenario = _load_related_entities(db, result) - is_custom_evaluator = evaluator and ( - bool(evaluator.custom_prompt) - or evaluator.agent_id is None - ) - - if not is_custom_evaluator and not agent: - raise ValueError("Agent not found and no custom prompt available") - - ai_providers = db.query(AIProvider).filter( - AIProvider.organization_id == result.organization_id, - AIProvider.is_active == True, - ).all() - - # Step 1: Transcription - if has_existing_transcript: - transcription = result.transcription - speaker_segments = result.speaker_segments or [] - transcription_time = 0.0 - else: - result.status = EvaluatorResultStatus.TRANSCRIBING.value - db.commit() - - transcription, speaker_segments, transcription_time = _transcribe_audio( - result, ai_providers, db - ) - result.transcription = transcription - # Avoid duplicating transcript structure when provider call_data already carries it. - if not result.call_data: - result.speaker_segments = speaker_segments if speaker_segments else None - db.commit() - - # Step 2: Load and categorize metrics - # Include metrics that have "agent" in their enabled_surfaces so users can - # restrict metrics to specific surfaces from the metrics page. Legacy rows - # (created before the surfaces column existed, or via fixtures that don't - # set the field) keep the original behavior: an enabled=True metric with - # an empty enabled_surfaces list is treated as agent-enabled. - enabled_metrics = db.query(Metric).filter( - Metric.organization_id == result.organization_id, - Metric.enabled == True, - ).all() - enabled_metrics = [ - m for m in enabled_metrics - if (m.name or "").strip().lower() not in REMOVED_EVALUATION_METRIC_NAMES - and ( - "agent" in (m.enabled_surfaces or []) - or not (m.enabled_surfaces or []) # legacy/unset → default to agent - ) - ] - - # Explicit metric selection on the evaluator/suite. Categorization - # parents are stored as a single ID and expanded to child labels here. - from app.services.evaluators.evaluator_helpers import expand_metric_ids_for_evaluation - - selected_metric_ids = { - str(mid) for mid in (getattr(evaluator, "metric_ids", None) or []) - } - if selected_metric_ids: - expanded_ids = expand_metric_ids_for_evaluation( - db, - result.organization_id, - list(selected_metric_ids), - ) or set() - enabled_metrics = [ - m for m in enabled_metrics if str(m.id) in expanded_ids - ] - - has_audio = bool(result.audio_s3_key) - llm_metrics, audio_metrics, metric_scores = _categorize_metrics(enabled_metrics, has_audio) - selected_metric_count = len(llm_metrics) + len(audio_metrics) - - call_recording = _playground_call_recording(db, result) - if call_recording: - from app.services.billing.flexprice_service import ( - record_playground_call_evaluated, - ) - - evaluation_attempt_id = f"{result.id}:{self.request.id}" - record_playground_call_evaluated( - result.organization_id, - evaluation_attempt_id, - evaluator_result_id=result.id, - workspace_id=result.workspace_id, - call_short_id=call_recording.call_short_id, - metric_count=selected_metric_count, - ) - - evaluation_time = None - - # Step 3: Audio metrics evaluation - if audio_metrics and has_audio: - try: - audio_scores = evaluate_audio_metrics( - audio_s3_key=result.audio_s3_key, - audio_metrics=audio_metrics, - result_id=result.result_id, - ) - - if _all_audio_scores_download_failed(audio_scores): - logger.warning( - f"[EvaluatorResult {result.result_id}] Existing S3 audio unavailable; " - "attempting provider audio recovery" - ) - recovered = _recover_missing_audio_for_result(result, db, refresh_call_data=True) - if recovered and result.audio_s3_key: - audio_scores = evaluate_audio_metrics( - audio_s3_key=result.audio_s3_key, - audio_metrics=audio_metrics, - result_id=result.result_id, - ) - - metric_scores.update(audio_scores) - except Exception as audio_err: - logger.error( - f"[EvaluatorResult {result.result_id}] Audio analysis failed: {audio_err}", - exc_info=True, - ) - metric_scores.update(handle_audio_evaluation_error(audio_metrics, audio_err)) - - # Step 4: LLM metrics evaluation - if llm_metrics and transcription: - result.status = EvaluatorResultStatus.EVALUATING.value - db.commit() - - try: - llm_scores, evaluation_time = _evaluate_llm_metrics_grouped( - transcription=transcription, - llm_metrics=llm_metrics, - ai_providers=ai_providers, - organization_id=result.organization_id, - result_id=result.result_id, - db=db, - evaluator=evaluator, - agent=agent, - persona=persona, - scenario=scenario, - ) - metric_scores.update(llm_scores) - except Exception as llm_err: - error_msg = str(llm_err).replace("{", "{{").replace("}", "}}") - logger.error( - f"[EvaluatorResult {result.result_id}] ✗ LLM evaluation failed: {error_msg}", - exc_info=True, - ) - metric_scores.update(handle_llm_evaluation_error(llm_metrics, llm_err)) - else: - if not llm_metrics: - logger.warning( - f"[EvaluatorResult {result.result_id}] No LLM-evaluable metrics found " - "(audio-only metrics were skipped), skipping evaluation" - ) - if not transcription: - logger.warning( - f"[EvaluatorResult {result.result_id}] No transcription available, " - "skipping evaluation" - ) - - # Step 5: Call Analysis - if transcription and not (result.call_data and result.call_data.get("call_analysis")): - try: - call_analysis = _generate_call_analysis( - transcription=transcription, - ai_providers=ai_providers, - organization_id=result.organization_id, - result_id=result.result_id, - db=db, - agent=agent, - scenario=scenario, - ) - if call_analysis: - existing_call_data = dict(result.call_data) if isinstance(result.call_data, dict) else {} - existing_call_data["call_analysis"] = call_analysis - result.call_data = slim_call_data_for_evaluator_result(existing_call_data) - except Exception as analysis_err: - logger.warning( - f"[EvaluatorResult {result.result_id}] Call analysis failed (non-fatal): {analysis_err}" - ) - - # Step 6: Complete - from sqlalchemy.orm.attributes import flag_modified - - result.metric_scores = _make_json_serializable(metric_scores) - flag_modified(result, "metric_scores") - if isinstance(result.call_data, dict): - result.call_data = slim_call_data_for_evaluator_result(result.call_data) - if isinstance(result.call_data, (dict, list)): - result.call_data = _make_json_serializable(result.call_data) - flag_modified(result, "call_data") - result.status = EvaluatorResultStatus.COMPLETED.value - result.error_message = None - _commit_evaluator_result(db, result) - - from app.services.billing.flexprice_service import ( - record_playground_evaluation_completed, - ) - - call_recording = _playground_call_recording(db, result) - if call_recording: - record_playground_evaluation_completed( - result.organization_id, - f"{result.id}:{self.request.id}", - evaluator_result_id=result.id, - workspace_id=result.workspace_id, - call_short_id=call_recording.call_short_id, - duration_seconds=result.duration_seconds, - metric_count=len(metric_scores) or selected_metric_count, - ) - - total_time = time.time() - task_start_time - logger.info( - f"[EvaluatorResult {result.result_id}] Completed in {total_time:.2f}s, " - f"{len(metric_scores)} metrics evaluated" - ) - - return { - "result_id": result_id, - "status": "completed", - "transcription": transcription, - "metrics_evaluated": len(metric_scores), - "processing_time": total_time, - "transcription_time": transcription_time, - "evaluation_time": evaluation_time, - } - - except Exception as e: - db.rollback() - logger.error(f"[EvaluatorResult {result_id}] Processing failed: {e}", exc_info=True) - try: - failed_result = db.query(EvaluatorResult).filter(EvaluatorResult.id == result_uuid).first() - if failed_result: - failed_result.status = EvaluatorResultStatus.FAILED.value - failed_result.error_message = str(e) - db.commit() - except Exception as persist_err: - db.rollback() - logger.error( - f"[EvaluatorResult {result_id}] Failed to persist FAILED status: {persist_err}", - exc_info=True, - ) - raise - - except Exception as exc: - raise self.retry(exc=exc, countdown=60) - finally: - db.close() +"""Celery task: process evaluator result (transcribe and evaluate metrics).""" + +import time +import uuid as _uuid +from uuid import UUID + +from loguru import logger +from sqlalchemy import or_ + +from app.database import SessionLocal +from app.models.database import ModelProvider + +from app.workers.config import celery_app +from app.workers.tasks.helpers.constants import ( + REMOVED_EVALUATION_METRIC_NAMES, + AUDIO_ONLY_METRIC_NAMES, +) +from app.workers.tasks.helpers.score_utils import provider_matches, get_metric_type_value +from app.workers.tasks.helpers.audio_evaluation import ( + evaluate_audio_metrics, + handle_audio_evaluation_error, +) +from app.workers.tasks.helpers.llm_evaluation import ( + evaluate_with_llm, + handle_llm_evaluation_error, +) +from app.services.evaluators.evaluator_result_call_data import slim_call_data_for_evaluator_result + + +def _commit_evaluator_result(db, result) -> None: + from app.services.live_entity_storage import sync_evaluator_result + + sync_evaluator_result(db, result) + db.commit() + + +def _make_json_serializable(obj): + """Recursively convert non-JSON-native values (e.g., NumPy types).""" + try: + import numpy as np + except Exception: + np = None + + if isinstance(obj, dict): + return {k: _make_json_serializable(v) for k, v in obj.items()} + if isinstance(obj, list): + return [_make_json_serializable(item) for item in obj] + if isinstance(obj, tuple): + return tuple(_make_json_serializable(item) for item in obj) + + if np is not None: + if isinstance(obj, np.integer): + return int(obj) + if isinstance(obj, np.floating): + return float(obj) + if isinstance(obj, np.ndarray): + return obj.tolist() + if isinstance(obj, np.bool_): + return bool(obj) + + return obj + + +def _load_related_entities(db, result): + """Load evaluator, agent, persona, scenario from database.""" + from app.models.database import Evaluator, Agent, Persona, Scenario + + evaluator = None + if result.evaluator_id: + evaluator = db.query(Evaluator).filter(Evaluator.id == result.evaluator_id).first() + if not evaluator: + logger.warning( + f"[EvaluatorResult {result.result_id}] Evaluator {result.evaluator_id} not found, " + "continuing without evaluator" + ) + + agent = None + if result.agent_id: + agent = db.query(Agent).filter(Agent.id == result.agent_id).first() + + persona = None + if result.persona_id: + persona = db.query(Persona).filter(Persona.id == result.persona_id).first() + + scenario = None + if result.scenario_id: + scenario = db.query(Scenario).filter(Scenario.id == result.scenario_id).first() + + return evaluator, agent, persona, scenario + + +def _transcribe_audio(result, ai_providers, db): + """Transcribe audio file and return transcript with timing info.""" + from app.services.ai.transcription_service import transcription_service + + stt_provider = ModelProvider.OPENAI + stt_model = "whisper-1" + + openai_provider = next( + (p for p in ai_providers if provider_matches(p.provider, ModelProvider.OPENAI)), + None, + ) + if not openai_provider: + logger.warning( + f"[EvaluatorResult {result.result_id}] No OpenAI provider found, using default whisper-1" + ) + + transcription_start_time = time.time() + transcription_result = transcription_service.transcribe( + audio_file_key=result.audio_s3_key, + stt_provider=stt_provider, + stt_model=stt_model, + organization_id=result.organization_id, + db=db, + language=None, + enable_speaker_diarization=True, + ) + transcription_time = time.time() - transcription_start_time + + return ( + transcription_result.get("transcript", ""), + transcription_result.get("speaker_segments", []), + transcription_time, + ) + + +def _generate_call_analysis(transcription, ai_providers, organization_id, result_id, db, agent=None, scenario=None): + """Generate call analysis (summary, sentiment, success) using LLM.""" + from app.services.ai.llm_service import llm_service + + llm_provider = ModelProvider.OPENAI + llm_model = "gpt-4o-mini" + + chosen_provider = next( + (p for p in ai_providers if provider_matches(p.provider, llm_provider)), + None, + ) + if not chosen_provider: + llm_provider = ModelProvider.GOOGLE + llm_model = "gemini-2.5-flash" + chosen_provider = next( + (p for p in ai_providers if provider_matches(p.provider, llm_provider)), + None, + ) + if not chosen_provider: + logger.warning(f"[EvaluatorResult {result_id}] No LLM provider available for call analysis") + return None + + agent_context = "" + if agent and agent.description: + agent_context = f"\n\nAgent Description:\n{agent.description}" + if scenario: + scenario_name = getattr(scenario, 'name', '') + scenario_desc = getattr(scenario, 'description', '') + if scenario_name or scenario_desc: + agent_context += f"\n\nScenario: {scenario_name}" + if scenario_desc: + agent_context += f"\nScenario Description: {scenario_desc}" + + messages = [ + { + "role": "system", + "content": ( + "You are a call analysis expert. Analyze the following conversation transcript " + "and provide a structured analysis. Respond ONLY with valid JSON, no markdown." + ), + }, + { + "role": "user", + "content": f"""Analyze this conversation transcript and provide: +1. A concise summary of the call (2-3 sentences) +2. The user/caller's overall sentiment (one of: Positive, Negative, Neutral, Mixed) +3. Whether the call was successful in achieving its objective (true/false) +{agent_context} + +Transcript: +{transcription} + +Respond in this exact JSON format: +{{"call_summary": "...", "user_sentiment": "...", "call_successful": true/false}}""", + }, + ] + + try: + llm_result = llm_service.generate_response( + messages=messages, + llm_provider=llm_provider, + llm_model=llm_model, + organization_id=organization_id, + db=db, + temperature=0.3, + max_tokens=500, + ) + import json + import re + text = llm_result.get("text", "") + json_match = re.search(r'\{[^{}]*\}', text, re.DOTALL) + if json_match: + analysis = json.loads(json_match.group()) + required_keys = {"call_summary", "user_sentiment", "call_successful"} + if required_keys.issubset(analysis.keys()): + logger.info(f"[EvaluatorResult {result_id}] Call analysis generated successfully") + return analysis + logger.warning(f"[EvaluatorResult {result_id}] Could not parse call analysis from LLM response") + return None + except Exception as e: + logger.error(f"[EvaluatorResult {result_id}] Call analysis failed: {e}", exc_info=True) + return None + + +def _categorize_metrics(enabled_metrics, has_audio): + """Split metrics into LLM-evaluable and audio-only categories.""" + llm_metrics = [] + audio_metrics = [] + skipped_scores = {} + + for m in enabled_metrics: + if m.name.lower() in AUDIO_ONLY_METRIC_NAMES: + if has_audio: + audio_metrics.append(m) + else: + skipped_scores[str(m.id)] = { + "value": None, + "type": get_metric_type_value(m), + "metric_name": m.name, + "skipped": "audio_required", + } + else: + llm_metrics.append(m) + + return llm_metrics, audio_metrics, skipped_scores + + +def _evaluate_llm_metrics_grouped( + *, + transcription: str, + llm_metrics: list, + ai_providers, + organization_id, + result_id: str, + db, + evaluator, + agent, + persona, + scenario, +) -> tuple[dict, float | None]: + """Evaluate LLM metrics, grouping categorization children by parent.""" + from app.workers.tasks.evaluate_call_import_row_core import build_parent_groups + + parents_by_id, children_by_parent, standalone_metrics = build_parent_groups( + db, llm_metrics + ) + metric_scores: dict = {} + evaluation_time: float | None = None + + def _run_bucket(bucket, parent_metric=None): + nonlocal evaluation_time + scores, eval_time = evaluate_with_llm( + transcription=transcription, + llm_metrics=bucket, + ai_providers=ai_providers, + organization_id=organization_id, + result_id=result_id, + db=db, + evaluator=evaluator, + agent=agent, + persona=persona, + scenario=scenario, + parent_metric=parent_metric, + ) + metric_scores.update(scores) + if eval_time is not None: + evaluation_time = eval_time + + if standalone_metrics: + _run_bucket(standalone_metrics, parent_metric=None) + + for parent_id, children in children_by_parent.items(): + parent_metric = parents_by_id.get(parent_id) + if not parent_metric: + logger.warning( + f"[EvaluatorResult {result_id}] Parent metric {parent_id} not found; " + "evaluating children as flat metrics" + ) + _run_bucket(children, parent_metric=None) + continue + _run_bucket(children, parent_metric=parent_metric) + + return metric_scores, evaluation_time + + +def _normalize_platform(platform: object) -> str: + """Normalize provider platform enum/string into lowercase string.""" + if not platform: + return "" + if hasattr(platform, "value"): + return str(platform.value).lower() + return str(platform).lower() + + +def _extract_audio_url(call_data: dict, platform: str) -> str | None: + """Extract provider-specific audio URL from call data.""" + recording_urls = call_data.get("recording_urls", {}) if isinstance(call_data, dict) else {} + provider_payload = call_data.get("provider_payload", {}) if isinstance(call_data, dict) else {} + artifact = call_data.get("artifact", {}) if isinstance(call_data, dict) else {} + recording = artifact.get("recording", {}) if isinstance(artifact, dict) else {} + mono_recording = recording.get("mono", {}) if isinstance(recording, dict) else {} + if platform == "elevenlabs": + return recording_urls.get("conversation_audio") + if platform == "retell": + return call_data.get("recording_url") + if platform == "vapi": + return ( + call_data.get("recordingUrl") + or call_data.get("stereoRecordingUrl") + or artifact.get("recordingUrl") + or artifact.get("stereoRecordingUrl") + or mono_recording.get("combinedUrl") + or recording_urls.get("combined_url") + or recording_urls.get("stereo_url") + or call_data.get("recordingUrl") + or provider_payload.get("recordingUrl") + or provider_payload.get("stereoRecordingUrl") + ) + if platform == "smallest": + return ( + call_data.get("recording_url") + or call_data.get("recordingUrl") + or recording_urls.get("combined_url") + or recording_urls.get("conversation_audio") + ) + return None + + +def _recover_missing_audio_for_result(result, db, refresh_call_data: bool = True) -> bool: + """ + Attempt to recover missing audio from provider, upload to S3, and persist key. + + Returns True when a new S3 key is successfully stored. + """ + import requests as _http + + from app.core.encryption import decrypt_api_key + from app.models.database import Agent, Integration + from app.services.storage.s3_service import s3_service + from app.services.voice_providers import get_voice_provider + + platform = _normalize_platform(result.provider_platform) + if platform not in {"retell", "vapi", "elevenlabs", "smallest"}: + return False + if not result.provider_call_id: + return False + + agent = db.query(Agent).filter(Agent.id == result.agent_id).first() if result.agent_id else None + integration = None + decrypted_key = None + if agent and agent.voice_ai_integration_id: + integration = db.query(Integration).filter( + Integration.id == agent.voice_ai_integration_id, + Integration.organization_id == result.organization_id, + ).first() + if integration: + try: + decrypted_key = decrypt_api_key(integration.api_key) + except Exception as decrypt_err: + logger.warning( + f"[EvaluatorResult {result.result_id}] Unable to decrypt integration key " + f"for audio recovery: {decrypt_err}" + ) + + call_data = result.call_data or {} + if refresh_call_data and decrypted_key: + try: + provider_class = get_voice_provider(platform) + provider_kwargs = {"api_key": decrypted_key} + if platform == "vapi" and integration and getattr(integration, "public_key", None): + provider_kwargs["public_key"] = integration.public_key + provider = provider_class(**provider_kwargs) + if hasattr(provider, "retrieve_call_metrics"): + refreshed = provider.retrieve_call_metrics(result.provider_call_id) + if isinstance(refreshed, dict) and refreshed: + call_data = refreshed + result.call_data = refreshed + db.commit() + except Exception as refresh_err: + logger.warning( + f"[EvaluatorResult {result.result_id}] Audio recovery could not refresh provider metrics: " + f"{refresh_err}" + ) + + audio_url = _extract_audio_url(call_data, platform) + if not audio_url: + logger.warning( + f"[EvaluatorResult {result.result_id}] Audio recovery failed: no provider recording URL available" + ) + return False + + headers = {"xi-api-key": decrypted_key} if platform == "elevenlabs" and decrypted_key else None + try: + response = _http.get(audio_url, headers=headers, timeout=120) + except Exception as download_err: + logger.warning( + f"[EvaluatorResult {result.result_id}] Audio recovery download failed: {download_err}" + ) + return False + + if response.status_code != 200 or not response.content: + logger.warning( + f"[EvaluatorResult {result.result_id}] Audio recovery download returned " + f"status={response.status_code}" + ) + return False + + content_type = response.headers.get("content-type", "audio/mpeg") + ext = "wav" if "wav" in content_type else "mp3" + org_id = str(result.organization_id) + s3_key = ( + f"audio/organizations/{org_id}/evaluations/" + f"{result.provider_call_id}/{_uuid.uuid4()}.{ext}" + ) + + try: + s3_service.upload_file_by_key(response.content, s3_key, content_type=content_type) + except Exception as upload_err: + logger.warning( + f"[EvaluatorResult {result.result_id}] Audio recovery upload failed: {upload_err}" + ) + return False + + result.audio_s3_key = s3_key + _commit_evaluator_result(db, result) + db.refresh(result) + logger.info( + f"[EvaluatorResult {result.result_id}] Recovered missing audio and stored at {s3_key}" + ) + return True + + +def _all_audio_scores_download_failed(audio_scores: dict[str, dict[str, object]]) -> bool: + """Check whether every audio metric failed due to missing/unreadable S3 object.""" + if not audio_scores: + return False + return all( + isinstance(score, dict) and score.get("error") == "audio_download_failed" + for score in audio_scores.values() + ) + + +def _playground_call_recording(db, result): + from app.models.database import CallRecording, CallRecordingSource + + return ( + db.query(CallRecording) + .filter( + CallRecording.evaluator_result_id == result.id, + CallRecording.source == CallRecordingSource.PLAYGROUND, + ) + .first() + ) + + +@celery_app.task(name="process_evaluator_result", bind=True, max_retries=3) +def process_evaluator_result_task(self, result_id: str): + """ + Celery task to process an evaluator result: transcribe audio and evaluate metrics. + + Workflow: + 1. QUEUED -> Job is created and queued + 2. TRANSCRIBING -> Audio is being transcribed + 3. EVALUATING -> Transcription is being evaluated against metrics + 4. COMPLETED -> All processing is complete + 5. FAILED -> An error occurred + """ + db = SessionLocal() + task_start_time = time.time() + + try: + from app.models.database import ( + EvaluatorResult, + EvaluatorResultStatus, + Metric, + AIProvider, + ) + + result_uuid = UUID(result_id) + result = db.query(EvaluatorResult).filter(EvaluatorResult.id == result_uuid).first() + + if not result: + logger.error(f"[EvaluatorResult {result_id}] Job not found in database") + return {"error": "Evaluator result not found"} + + logger.info(f"[EvaluatorResult {result.result_id}] Starting processing task") + + result.celery_task_id = self.request.id + db.commit() + + try: + if not result.audio_s3_key: + _recover_missing_audio_for_result(result, db, refresh_call_data=True) + + has_existing_transcript = bool(result.transcription) + if not result.audio_s3_key and not has_existing_transcript: + raise ValueError("No audio S3 key or existing transcript found") + + evaluator, agent, persona, scenario = _load_related_entities(db, result) + is_custom_evaluator = evaluator and ( + bool(evaluator.custom_prompt) + or evaluator.agent_id is None + ) + + if not is_custom_evaluator and not agent: + raise ValueError("Agent not found and no custom prompt available") + + ai_providers = db.query(AIProvider).filter( + AIProvider.organization_id == result.organization_id, + AIProvider.is_active == True, + ).all() + + # Step 1: Transcription + if has_existing_transcript: + transcription = result.transcription + speaker_segments = result.speaker_segments or [] + transcription_time = 0.0 + else: + result.status = EvaluatorResultStatus.TRANSCRIBING.value + db.commit() + + transcription, speaker_segments, transcription_time = _transcribe_audio( + result, ai_providers, db + ) + result.transcription = transcription + # Avoid duplicating transcript structure when provider call_data already carries it. + if not result.call_data: + result.speaker_segments = speaker_segments if speaker_segments else None + db.commit() + + # Step 2: Load and categorize metrics + # Include metrics that have "agent" in their enabled_surfaces so users can + # restrict metrics to specific surfaces from the metrics page. Legacy rows + # (created before the surfaces column existed, or via fixtures that don't + # set the field) keep the original behavior: an enabled=True metric with + # an empty enabled_surfaces list is treated as agent-enabled. + enabled_metrics = db.query(Metric).filter( + Metric.organization_id == result.organization_id, + Metric.enabled == True, + or_(Metric.lifecycle.is_(None), Metric.lifecycle == "active"), + ).all() + enabled_metrics = [ + m for m in enabled_metrics + if (m.name or "").strip().lower() not in REMOVED_EVALUATION_METRIC_NAMES + and ( + "agent" in (m.enabled_surfaces or []) + or not (m.enabled_surfaces or []) # legacy/unset → default to agent + ) + ] + + # Explicit metric selection on the evaluator/suite. Categorization + # parents are stored as a single ID and expanded to child labels here. + from app.services.evaluators.evaluator_helpers import expand_metric_ids_for_evaluation + + selected_metric_ids = { + str(mid) for mid in (getattr(evaluator, "metric_ids", None) or []) + } + if selected_metric_ids: + expanded_ids = expand_metric_ids_for_evaluation( + db, + result.organization_id, + list(selected_metric_ids), + ) or set() + enabled_metrics = [ + m for m in enabled_metrics if str(m.id) in expanded_ids + ] + + has_audio = bool(result.audio_s3_key) + llm_metrics, audio_metrics, metric_scores = _categorize_metrics(enabled_metrics, has_audio) + selected_metric_count = len(llm_metrics) + len(audio_metrics) + + call_recording = _playground_call_recording(db, result) + if call_recording: + from app.services.billing.flexprice_service import ( + record_playground_call_evaluated, + ) + + evaluation_attempt_id = f"{result.id}:{self.request.id}" + record_playground_call_evaluated( + result.organization_id, + evaluation_attempt_id, + evaluator_result_id=result.id, + workspace_id=result.workspace_id, + call_short_id=call_recording.call_short_id, + metric_count=selected_metric_count, + ) + + evaluation_time = None + + # Step 3: Audio metrics evaluation + if audio_metrics and has_audio: + try: + audio_scores = evaluate_audio_metrics( + audio_s3_key=result.audio_s3_key, + audio_metrics=audio_metrics, + result_id=result.result_id, + ) + + if _all_audio_scores_download_failed(audio_scores): + logger.warning( + f"[EvaluatorResult {result.result_id}] Existing S3 audio unavailable; " + "attempting provider audio recovery" + ) + recovered = _recover_missing_audio_for_result(result, db, refresh_call_data=True) + if recovered and result.audio_s3_key: + audio_scores = evaluate_audio_metrics( + audio_s3_key=result.audio_s3_key, + audio_metrics=audio_metrics, + result_id=result.result_id, + ) + + metric_scores.update(audio_scores) + except Exception as audio_err: + logger.error( + f"[EvaluatorResult {result.result_id}] Audio analysis failed: {audio_err}", + exc_info=True, + ) + metric_scores.update(handle_audio_evaluation_error(audio_metrics, audio_err)) + + # Step 4: LLM metrics evaluation + if llm_metrics and transcription: + result.status = EvaluatorResultStatus.EVALUATING.value + db.commit() + + try: + llm_scores, evaluation_time = _evaluate_llm_metrics_grouped( + transcription=transcription, + llm_metrics=llm_metrics, + ai_providers=ai_providers, + organization_id=result.organization_id, + result_id=result.result_id, + db=db, + evaluator=evaluator, + agent=agent, + persona=persona, + scenario=scenario, + ) + metric_scores.update(llm_scores) + except Exception as llm_err: + error_msg = str(llm_err).replace("{", "{{").replace("}", "}}") + logger.error( + f"[EvaluatorResult {result.result_id}] ✗ LLM evaluation failed: {error_msg}", + exc_info=True, + ) + metric_scores.update(handle_llm_evaluation_error(llm_metrics, llm_err)) + else: + if not llm_metrics: + logger.warning( + f"[EvaluatorResult {result.result_id}] No LLM-evaluable metrics found " + "(audio-only metrics were skipped), skipping evaluation" + ) + if not transcription: + logger.warning( + f"[EvaluatorResult {result.result_id}] No transcription available, " + "skipping evaluation" + ) + + # Step 5: Call Analysis + if transcription and not (result.call_data and result.call_data.get("call_analysis")): + try: + call_analysis = _generate_call_analysis( + transcription=transcription, + ai_providers=ai_providers, + organization_id=result.organization_id, + result_id=result.result_id, + db=db, + agent=agent, + scenario=scenario, + ) + if call_analysis: + existing_call_data = dict(result.call_data) if isinstance(result.call_data, dict) else {} + existing_call_data["call_analysis"] = call_analysis + result.call_data = slim_call_data_for_evaluator_result(existing_call_data) + except Exception as analysis_err: + logger.warning( + f"[EvaluatorResult {result.result_id}] Call analysis failed (non-fatal): {analysis_err}" + ) + + # Step 6: Complete + from sqlalchemy.orm.attributes import flag_modified + + result.metric_scores = _make_json_serializable(metric_scores) + flag_modified(result, "metric_scores") + if isinstance(result.call_data, dict): + result.call_data = slim_call_data_for_evaluator_result(result.call_data) + if isinstance(result.call_data, (dict, list)): + result.call_data = _make_json_serializable(result.call_data) + flag_modified(result, "call_data") + result.status = EvaluatorResultStatus.COMPLETED.value + result.error_message = None + _commit_evaluator_result(db, result) + + from app.services.billing.flexprice_service import ( + record_playground_evaluation_completed, + ) + + call_recording = _playground_call_recording(db, result) + if call_recording: + record_playground_evaluation_completed( + result.organization_id, + f"{result.id}:{self.request.id}", + evaluator_result_id=result.id, + workspace_id=result.workspace_id, + call_short_id=call_recording.call_short_id, + duration_seconds=result.duration_seconds, + metric_count=len(metric_scores) or selected_metric_count, + ) + + total_time = time.time() - task_start_time + logger.info( + f"[EvaluatorResult {result.result_id}] Completed in {total_time:.2f}s, " + f"{len(metric_scores)} metrics evaluated" + ) + + return { + "result_id": result_id, + "status": "completed", + "transcription": transcription, + "metrics_evaluated": len(metric_scores), + "processing_time": total_time, + "transcription_time": transcription_time, + "evaluation_time": evaluation_time, + } + + except Exception as e: + db.rollback() + logger.error(f"[EvaluatorResult {result_id}] Processing failed: {e}", exc_info=True) + try: + failed_result = db.query(EvaluatorResult).filter(EvaluatorResult.id == result_uuid).first() + if failed_result: + failed_result.status = EvaluatorResultStatus.FAILED.value + failed_result.error_message = str(e) + db.commit() + except Exception as persist_err: + db.rollback() + logger.error( + f"[EvaluatorResult {result_id}] Failed to persist FAILED status: {persist_err}", + exc_info=True, + ) + raise + + except Exception as exc: + raise self.retry(exc=exc, countdown=60) + finally: + db.close() diff --git a/app/workers/tasks/transcribe_call_import_row.py b/app/workers/tasks/transcribe_call_import_row.py index c6e89198..68368b93 100644 --- a/app/workers/tasks/transcribe_call_import_row.py +++ b/app/workers/tasks/transcribe_call_import_row.py @@ -780,16 +780,28 @@ def transcribe_call_import_row_task( ) row_db = catalog_db = None + eval_row_for_chain = None try: if run_eval_row_id: try: _er_db, _cat, eval_row_for_chain, _, _ = ( locate_call_import_evaluation_row(UUID(run_eval_row_id)) ) + from app.workers.tasks.evaluate_call_import_row_core import ( + was_cancelled_externally, + ) + + if was_cancelled_externally(_er_db, eval_row_for_chain): + close_row_sessions(_er_db, _cat) + return { + "status": "skipped", + "reason": "eval_cancelled_by_user", + } evaluation_id_for_dispatch = str( eval_row_for_chain.evaluation_id ) close_row_sessions(_er_db, _cat) + eval_row_for_chain = None except LookupError: pass diff --git a/config.yml.example b/config.yml.example index 935737e6..91b43e2e 100644 --- a/config.yml.example +++ b/config.yml.example @@ -164,6 +164,9 @@ auth: refresh_token_ttl_days: 7 # Turn this off in Cloud SaaS to block self-serve signup. allow_signup: true + # When enabled, signup requires a valid reference code from the platform admin API. + gated_signup: + enabled: false # External OIDC (enterprise): bring your own IdP. Works with any # OIDC-compliant provider. See README.md > Authentication & Deployment diff --git a/docs-fumadocs/content/docs/products/meta.json b/docs-fumadocs/content/docs/products/meta.json index 7d9a5d39..557b361f 100644 --- a/docs-fumadocs/content/docs/products/meta.json +++ b/docs-fumadocs/content/docs/products/meta.json @@ -6,6 +6,7 @@ "scenarios", "evaluators", "metrics", + "metrics-studio", "playground", "voice-playground", "call-imports", diff --git a/docs-fumadocs/content/docs/products/metrics-studio.mdx b/docs-fumadocs/content/docs/products/metrics-studio.mdx new file mode 100644 index 00000000..3cb9128a --- /dev/null +++ b/docs-fumadocs/content/docs/products/metrics-studio.mdx @@ -0,0 +1,18 @@ +--- +id: metrics-studio +title: Metrics Studio +sidebar_position: 6 +--- + +# Metrics Studio + +Metrics Studio lets you experiment with metrics against ad-hoc call sources. + +Metrics Studio is a sub-tab under **Metrics → Studio**. Use it to: + +1. Select **enabled** metrics for the active workspace, or **draft** metrics you are iterating on +2. Pick call sources from call imports, playground/observability recordings, or simulated evaluator results +3. Run ad-hoc evaluations and inspect per-metric scores +4. Promote draft metrics to production when satisfied + +The **Active** tab lists only metrics that are enabled in your current workspace. Disabled metrics stay on the **Manage** tab until you turn them on. Draft metrics stay hidden from Call Import evaluations and the Manage tab until promoted. diff --git a/docs/diagrams/excalidraw/README.md b/docs/diagrams/excalidraw/README.md new file mode 100644 index 00000000..7fdc8944 --- /dev/null +++ b/docs/diagrams/excalidraw/README.md @@ -0,0 +1,59 @@ +# Call Import Architecture — Excalidraw Diagrams + +All call-import architecture diagrams live in **one file** for easy tracking and editing. + +## Single combined file + +| File | Description | +|------|-------------| +| [`call-import-architecture.excalidraw`](call-import-architecture.excalidraw) | All 6 diagrams stacked vertically with section headers (01–06) | + +### Sections inside the file + +| # | Section | Contents | +|---|---------|----------| +| 01 | Platform Architecture | K8s cluster: API, workers, Redis, catalog + 4 shards, external services | +| 02 | Postgres ↔ Redis ↔ Workers | 7-step coordination loop | +| 03 | Call Import Pipeline | End-to-end stages and import vs eval slot types | +| 04 | Database Sharding | Catalog vs data shards, routing formula, pool layout | +| 05 | Fair Dispatch & Inflight Limits | Workspace RR, limit hierarchy, dual 10k numbers | +| 06 | Scaling Roadmap | Now (no PgBouncer) vs future scaling phases | + +Scroll down in Excalidraw to move between sections. Each section has a blue header bar (01–06). + +## How to open & edit + +### VS Code / Cursor (recommended) + +1. Install the [Excalidraw extension](https://marketplace.visualstudio.com/items?itemName=pomdtr.excalidraw-editor) +2. Open `call-import-architecture.excalidraw` +3. Edit visually; save commits the JSON back to the repo + +### Excalidraw.com + +1. Go to [excalidraw.com](https://excalidraw.com) +2. **Open → Load from file** and select `call-import-architecture.excalidraw` + +### Regenerate from script + +After changing layout code: + +```bash +python3 scripts/generate_call_import_excalidraw_diagrams.py +``` + +Source: `scripts/excalidraw_builder.py` + `scripts/generate_call_import_excalidraw_diagrams.py` + +## Embed in Confluence + +1. Open the combined file in Excalidraw +2. Zoom to the section you need, or export the full canvas +3. **Export → PNG** or **SVG** +4. In Confluence page editor: insert image at the relevant section + +## Related docs + +- **Operator handbook (Confluence):** [Call Import Scaling — Operator Handbook (2 pages)](https://efficientai.atlassian.net/wiki/spaces/ETD/pages/59932681) +- **Full scaling guide (Confluence):** [Call Import Architecture & Scaling Guide](https://efficientai.atlassian.net/wiki/spaces/ETD/pages/59899905) +- PowerPoint: `docs/presentations/EfficientAI_Call_Import_Architecture_and_Scaling.pptx` +- Load test: `docs/operations/call-import-sharding-load-test.md` diff --git a/docs/diagrams/excalidraw/call-import-architecture.excalidraw b/docs/diagrams/excalidraw/call-import-architecture.excalidraw new file mode 100644 index 00000000..03accef6 --- /dev/null +++ b/docs/diagrams/excalidraw/call-import-architecture.excalidraw @@ -0,0 +1,4962 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "id": "TlMRZCf4MV", + "type": "rectangle", + "x": 60, + "y": 304, + "width": 1380, + "height": 492.75, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 394524304, + "version": 1, + "versionNonce": 1552703569, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "kE5BTqa9ha" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "kE5BTqa9ha", + "type": "text", + "x": 76, + "y": 312, + "width": 180, + "height": 24, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 581682065, + "version": 1, + "versionNonce": 419753360, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Kubernetes cluster", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": "TlMRZCf4MV", + "originalText": "Kubernetes cluster", + "lineHeight": 1.25 + }, + { + "id": "K299Not56h", + "type": "text", + "x": 80, + "y": 40, + "width": 1340, + "height": 74.4, + "angle": 0, + "strokeColor": "#1a365d", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 245822141, + "version": 1, + "versionNonce": 1999656150, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "EfficientAI Call Import \u2014 Architecture Diagrams", + "fontSize": 32, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "EfficientAI Call Import \u2014 Architecture Diagrams", + "lineHeight": 1.35 + }, + { + "id": "O3z1k8Ua1K", + "type": "text", + "x": 80, + "y": 96, + "width": 1340, + "height": 54.099999999999994, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 903944553, + "version": 1, + "versionNonce": 653330999, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "All 6 diagrams in one canvas \u2014 scroll down to navigate sections 01\u201306", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "All 6 diagrams in one canvas \u2014 scroll down to navigate sections 01\u201306", + "lineHeight": 1.35 + }, + { + "id": "W1kzekQWkM", + "type": "rectangle", + "x": 60, + "y": 180, + "width": 1380, + "height": 56, + "angle": 0, + "strokeColor": "#1a365d", + "backgroundColor": "#e7f5ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1706521925, + "version": 1, + "versionNonce": 949782449, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "c95Pn9S3Jr", + "type": "text", + "x": 80, + "y": 194, + "width": 1340, + "height": 62.8, + "angle": 0, + "strokeColor": "#1a365d", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1470258223, + "version": 1, + "versionNonce": 918605618, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "01 Platform Architecture", + "fontSize": 24, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "01 Platform Architecture", + "lineHeight": 1.35 + }, + { + "id": "JpIsPpMiyn", + "type": "text", + "x": 80, + "y": 268, + "width": 1340, + "height": 54.099999999999994, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1794758880, + "version": 1, + "versionNonce": 1182772249, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Kubernetes deployment with catalog DB, 4 data shards, Redis fair-share", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Kubernetes deployment with catalog DB, 4 data shards, Redis fair-share", + "lineHeight": 1.35 + }, + { + "id": "T8AqQgNsdM", + "type": "rectangle", + "x": 78.0, + "y": 312, + "width": 240, + "height": 110, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 153358603, + "version": 1, + "versionNonce": 1052280942, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "0u0Z5biNog" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "0u0Z5biNog", + "type": "text", + "x": 90.0, + "y": 324, + "width": 216, + "height": 86, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 44097903, + "version": 1, + "versionNonce": 342144768, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "API\n3 replicas", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "T8AqQgNsdM", + "originalText": "API\n3 replicas", + "lineHeight": 1.35 + }, + { + "id": "JTBYul53JL", + "type": "rectangle", + "x": 354.0, + "y": 312, + "width": 240, + "height": 120.8, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 589067158, + "version": 1, + "versionNonce": 1357773553, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "mouTEyEKqm" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "mouTEyEKqm", + "type": "text", + "x": 366.0, + "y": 324, + "width": 216, + "height": 96.8, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 315599458, + "version": 1, + "versionNonce": 835354134, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "worker-imports\n8\u201316 pods \u00d7 16 threads\nimports \u00b7 diarization \u00b7 eval", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "JTBYul53JL", + "originalText": "worker-imports\n8\u201316 pods \u00d7 16 threads\nimports \u00b7 diarization \u00b7 eval", + "lineHeight": 1.35 + }, + { + "id": "hUbsv5E3in", + "type": "rectangle", + "x": 630.0, + "y": 312, + "width": 240, + "height": 110, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 445390469, + "version": 1, + "versionNonce": 1014448259, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "6PnYVVsvgY" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "6PnYVVsvgY", + "type": "text", + "x": 642.0, + "y": 324, + "width": 216, + "height": 86, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1531409815, + "version": 1, + "versionNonce": 112548276, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "worker\naudio-metrics\nPraat / UTMOS / torch", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "hUbsv5E3in", + "originalText": "worker\naudio-metrics\nPraat / UTMOS / torch", + "lineHeight": 1.35 + }, + { + "id": "qhPZdWpvCf", + "type": "rectangle", + "x": 906.0, + "y": 312, + "width": 240, + "height": 110, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1787795300, + "version": 1, + "versionNonce": 1350533436, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "7cDUyacYs5" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "7cDUyacYs5", + "type": "text", + "x": 918.0, + "y": 324, + "width": 216, + "height": 86, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 285794430, + "version": 1, + "versionNonce": 471942517, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Redis\nCelery broker\ninflight counters", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "qhPZdWpvCf", + "originalText": "Redis\nCelery broker\ninflight counters", + "lineHeight": 1.35 + }, + { + "id": "s0Tp7x1Xzq", + "type": "rectangle", + "x": 1182.0, + "y": 312, + "width": 240, + "height": 110, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e7f5ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1211151990, + "version": 1, + "versionNonce": 1975681669, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "XbcJffDGW1" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "XbcJffDGW1", + "type": "text", + "x": 1194.0, + "y": 324, + "width": 216, + "height": 86, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1682050901, + "version": 1, + "versionNonce": 242346120, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Prometheus\n+ KEDA", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "s0Tp7x1Xzq", + "originalText": "Prometheus\n+ KEDA", + "lineHeight": 1.35 + }, + { + "id": "3xDlUsi6fL", + "type": "rectangle", + "x": 570.0, + "y": 496.8, + "width": 360, + "height": 101.95, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1345600089, + "version": 1, + "versionNonce": 1509759797, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "OzG7r0M93P" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "OzG7r0M93P", + "type": "text", + "x": 582.0, + "y": 508.8, + "width": 336, + "height": 77.95, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 952845058, + "version": 1, + "versionNonce": 1002962035, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Catalog Postgres \u2014 efficientai_catalog\nCallImport \u00b7 CallImportEvaluation \u00b7 metrics \u00b7 shard registry", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "3xDlUsi6fL", + "originalText": "Catalog Postgres \u2014 efficientai_catalog\nCallImport \u00b7 CallImportEvaluation \u00b7 metrics \u00b7 shard registry", + "lineHeight": 1.35 + }, + { + "id": "pfbpTsxo78", + "type": "rectangle", + "x": 130.0, + "y": 646.75, + "width": 280, + "height": 110, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1154575379, + "version": 1, + "versionNonce": 1639480808, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "yCpfJ6NjUV" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "yCpfJ6NjUV", + "type": "text", + "x": 142.0, + "y": 658.75, + "width": 256, + "height": 86, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1875570808, + "version": 1, + "versionNonce": 1978724294, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "data-shard-01\nCallImportRow\nEvalRow", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "pfbpTsxo78", + "originalText": "data-shard-01\nCallImportRow\nEvalRow", + "lineHeight": 1.35 + }, + { + "id": "kkRQd3haeY", + "type": "rectangle", + "x": 450.0, + "y": 646.75, + "width": 280, + "height": 110, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 324987909, + "version": 1, + "versionNonce": 1946894142, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "bJrGjFb1aq" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "bJrGjFb1aq", + "type": "text", + "x": 462.0, + "y": 658.75, + "width": 256, + "height": 86, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 330653832, + "version": 1, + "versionNonce": 1287996769, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "data-shard-02\nCallImportRow\nEvalRow", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "kkRQd3haeY", + "originalText": "data-shard-02\nCallImportRow\nEvalRow", + "lineHeight": 1.35 + }, + { + "id": "76oFY0jZIH", + "type": "rectangle", + "x": 770.0, + "y": 646.75, + "width": 280, + "height": 110, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1307918237, + "version": 1, + "versionNonce": 747343415, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "4iMmE3Ubpb" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "4iMmE3Ubpb", + "type": "text", + "x": 782.0, + "y": 658.75, + "width": 256, + "height": 86, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 208896712, + "version": 1, + "versionNonce": 1680264626, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "data-shard-03\nCallImportRow\nEvalRow", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "76oFY0jZIH", + "originalText": "data-shard-03\nCallImportRow\nEvalRow", + "lineHeight": 1.35 + }, + { + "id": "N6iqAcW3Vs", + "type": "rectangle", + "x": 1090.0, + "y": 646.75, + "width": 280, + "height": 110, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1654025096, + "version": 1, + "versionNonce": 1093757721, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "kA8tVUk5uD" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "kA8tVUk5uD", + "type": "text", + "x": 1102.0, + "y": 658.75, + "width": 256, + "height": 86, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1971241552, + "version": 1, + "versionNonce": 1554534650, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "data-shard-04\nCallImportRow\nEvalRow", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "N6iqAcW3Vs", + "originalText": "data-shard-04\nCallImportRow\nEvalRow", + "lineHeight": 1.35 + }, + { + "id": "mtSugtx1rP", + "type": "rectangle", + "x": 210.0, + "y": 812.75, + "width": 320, + "height": 90, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1347171710, + "version": 1, + "versionNonce": 1235870995, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "29rgs7nloR" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "29rgs7nloR", + "type": "text", + "x": 222.0, + "y": 824.75, + "width": 296, + "height": 66, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1572344985, + "version": 1, + "versionNonce": 260529661, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Blob storage\nS3 / GCS / Azure", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "mtSugtx1rP", + "originalText": "Blob storage\nS3 / GCS / Azure", + "lineHeight": 1.35 + }, + { + "id": "a6VpmRXSJc", + "type": "rectangle", + "x": 590.0, + "y": 812.75, + "width": 320, + "height": 90, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffd8a8", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 295947211, + "version": 1, + "versionNonce": 974114337, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "6iaIvjr7sD" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "6iaIvjr7sD", + "type": "text", + "x": 602.0, + "y": 824.75, + "width": 296, + "height": 66, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1422309345, + "version": 1, + "versionNonce": 1212912045, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "STT / LLM providers", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "a6VpmRXSJc", + "originalText": "STT / LLM providers", + "lineHeight": 1.35 + }, + { + "id": "cDlSpRpPoS", + "type": "rectangle", + "x": 970.0, + "y": 812.75, + "width": 320, + "height": 90, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffd8a8", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1762640468, + "version": 1, + "versionNonce": 1299167375, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "NC99x4rM4t" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "NC99x4rM4t", + "type": "text", + "x": 982.0, + "y": 824.75, + "width": 296, + "height": 66, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1394350838, + "version": 1, + "versionNonce": 295129998, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Telephony\nExotel / Plivo", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "cDlSpRpPoS", + "originalText": "Telephony\nExotel / Plivo", + "lineHeight": 1.35 + }, + { + "id": "v10dresgRj", + "type": "arrow", + "x": 474.0, + "y": 432.8, + "width": 0.0, + "height": 60.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 1472075123, + "version": 1, + "versionNonce": 1847662253, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 60.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "d8vQCntlEA", + "type": "text", + "x": 414.0, + "y": 450.8, + "width": 400, + "height": 49.75, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 175743387, + "version": 1, + "versionNonce": 1983802234, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "read/write rows", + "fontSize": 15, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "read/write rows", + "lineHeight": 1.35 + }, + { + "id": "rguYV22iLD", + "type": "arrow", + "x": 750.0, + "y": 598.75, + "width": 0.0, + "height": 44.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 872302731, + "version": 1, + "versionNonce": 1660422044, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 44.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "sNROJ8DI8P", + "type": "rectangle", + "x": 80, + "y": 950.75, + "width": 1340, + "height": 70, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e7f5ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1910547532, + "version": 1, + "versionNonce": 431903470, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "eccfFTuOHh" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "eccfFTuOHh", + "type": "text", + "x": 92, + "y": 962.75, + "width": 1316, + "height": 46, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 589553915, + "version": 1, + "versionNonce": 252193772, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Queue drain order on worker-imports: imports \u2192 diarization \u2192 eval-control \u2192 evaluations", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "sNROJ8DI8P", + "originalText": "Queue drain order on worker-imports: imports \u2192 diarization \u2192 eval-control \u2192 evaluations", + "lineHeight": 1.35 + }, + { + "id": "k8gcwwq8nv", + "type": "rectangle", + "x": 60, + "y": 1160.75, + "width": 1380, + "height": 56, + "angle": 0, + "strokeColor": "#1a365d", + "backgroundColor": "#e7f5ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 757040782, + "version": 1, + "versionNonce": 1812404225, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "ZwvKjYIGIu", + "type": "text", + "x": 80, + "y": 1174.75, + "width": 1340, + "height": 62.8, + "angle": 0, + "strokeColor": "#1a365d", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1744837721, + "version": 1, + "versionNonce": 932605375, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "02 Postgres \u2194 Redis \u2194 Workers", + "fontSize": 24, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "02 Postgres \u2194 Redis \u2194 Workers", + "lineHeight": 1.35 + }, + { + "id": "RoSMtaZCSY", + "type": "text", + "x": 80, + "y": 1248.75, + "width": 1340, + "height": 54.099999999999994, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1212506177, + "version": 1, + "versionNonce": 1152055985, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Pending backlog lives in Postgres. Redis decides who may run.", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Pending backlog lives in Postgres. Redis decides who may run.", + "lineHeight": 1.35 + }, + { + "id": "Hx9Xo3lwz5", + "type": "rectangle", + "x": 470.0, + "y": 1284.75, + "width": 560, + "height": 92, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1249442482, + "version": 1, + "versionNonce": 1776901981, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "m739KBK5uS" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "m739KBK5uS", + "type": "text", + "x": 482.0, + "y": 1296.75, + "width": 536, + "height": 68, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 15883358, + "version": 1, + "versionNonce": 1600587086, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "1 \u00b7 Write headers + rows\nPostgres catalog + data shards", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Hx9Xo3lwz5", + "originalText": "1 \u00b7 Write headers + rows\nPostgres catalog + data shards", + "lineHeight": 1.35 + }, + { + "id": "tfjZ2EK8pn", + "type": "arrow", + "x": 750.0, + "y": 1380.75, + "width": 0.0, + "height": 56.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 903257034, + "version": 1, + "versionNonce": 1556669628, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 56.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "I0n30bBPbl", + "type": "rectangle", + "x": 470.0, + "y": 1440.75, + "width": 560, + "height": 92, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1759532754, + "version": 1, + "versionNonce": 326559592, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "PvYH7Swcig" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "PvYH7Swcig", + "type": "text", + "x": 482.0, + "y": 1452.75, + "width": 536, + "height": 68, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1317955263, + "version": 1, + "versionNonce": 288787229, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "2 \u00b7 Fair dispatcher reads pending rows\nPostgres scatter-gather (NOT Celery queue depth)", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "I0n30bBPbl", + "originalText": "2 \u00b7 Fair dispatcher reads pending rows\nPostgres scatter-gather (NOT Celery queue depth)", + "lineHeight": 1.35 + }, + { + "id": "trfF7xdJDA", + "type": "arrow", + "x": 750.0, + "y": 1536.75, + "width": 0.0, + "height": 56.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 743155969, + "version": 1, + "versionNonce": 205667959, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 56.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "nbdyPZPRV4", + "type": "rectangle", + "x": 470.0, + "y": 1596.75, + "width": 560, + "height": 92, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 333232116, + "version": 1, + "versionNonce": 1336147590, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "odGzuJN7HC" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "odGzuJN7HC", + "type": "text", + "x": 482.0, + "y": 1608.75, + "width": 536, + "height": 68, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 390437842, + "version": 1, + "versionNonce": 1177291157, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "3 \u00b7 acquire_eval_slot() / acquire_import_slot()\nRedis Lua script increments inflight counters", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "nbdyPZPRV4", + "originalText": "3 \u00b7 acquire_eval_slot() / acquire_import_slot()\nRedis Lua script increments inflight counters", + "lineHeight": 1.35 + }, + { + "id": "r1kCYdT0Bq", + "type": "arrow", + "x": 750.0, + "y": 1692.75, + "width": 0.0, + "height": 56.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 426786593, + "version": 1, + "versionNonce": 1642379714, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 56.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "NWkKMwyAlM", + "type": "rectangle", + "x": 470.0, + "y": 1752.75, + "width": 560, + "height": 92, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1946320108, + "version": 1, + "versionNonce": 1235724836, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "dkN3Q3fgro" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "dkN3Q3fgro", + "type": "text", + "x": 482.0, + "y": 1764.75, + "width": 536, + "height": 68, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 879371329, + "version": 1, + "versionNonce": 1719692900, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "4 \u00b7 Enqueue Celery task + store celery_task_id\nPostgres shard row updated", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "NWkKMwyAlM", + "originalText": "4 \u00b7 Enqueue Celery task + store celery_task_id\nPostgres shard row updated", + "lineHeight": 1.35 + }, + { + "id": "j3ZRHGAgdT", + "type": "arrow", + "x": 750.0, + "y": 1848.75, + "width": 0.0, + "height": 56.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 830417180, + "version": 1, + "versionNonce": 278527957, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 56.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "tsfB2OXb9H", + "type": "rectangle", + "x": 470.0, + "y": 1908.75, + "width": 560, + "height": 92, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1859953935, + "version": 1, + "versionNonce": 944087617, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "lnvG6Z4gxS" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "lnvG6Z4gxS", + "type": "text", + "x": 482.0, + "y": 1920.75, + "width": 536, + "height": 68, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 881437238, + "version": 1, + "versionNonce": 413621878, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "5 \u00b7 Worker executes task\nPostgres reads/writes + STT / LLM / telephony APIs", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "tsfB2OXb9H", + "originalText": "5 \u00b7 Worker executes task\nPostgres reads/writes + STT / LLM / telephony APIs", + "lineHeight": 1.35 + }, + { + "id": "CptfYuMLPr", + "type": "arrow", + "x": 750.0, + "y": 2004.75, + "width": 0.0, + "height": 56.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 1925207887, + "version": 1, + "versionNonce": 448690694, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 56.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "stLmxT12qu", + "type": "rectangle", + "x": 470.0, + "y": 2064.75, + "width": 560, + "height": 92, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 139631613, + "version": 1, + "versionNonce": 1811206735, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "3MOqhNechl" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "3MOqhNechl", + "type": "text", + "x": 482.0, + "y": 2076.75, + "width": 536, + "height": 68, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 339145536, + "version": 1, + "versionNonce": 1522724006, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "6 \u00b7 release_*_slot()\nRedis decrements inflight counters", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "stLmxT12qu", + "originalText": "6 \u00b7 release_*_slot()\nRedis decrements inflight counters", + "lineHeight": 1.35 + }, + { + "id": "U7LFVpyI5c", + "type": "arrow", + "x": 750.0, + "y": 2160.75, + "width": 0.0, + "height": 56.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 1451487363, + "version": 1, + "versionNonce": 218159213, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 56.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "NKsmGO7pKv", + "type": "rectangle", + "x": 470.0, + "y": 2220.75, + "width": 560, + "height": 92, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 577812966, + "version": 1, + "versionNonce": 704882453, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "ebaAEwPiK5" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "ebaAEwPiK5", + "type": "text", + "x": 482.0, + "y": 2232.75, + "width": 536, + "height": 68, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1151138854, + "version": 1, + "versionNonce": 1443967415, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "7 \u00b7 schedule_fair_dispatch()\nNext workspace round-robin turn", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "NKsmGO7pKv", + "originalText": "7 \u00b7 schedule_fair_dispatch()\nNext workspace round-robin turn", + "lineHeight": 1.35 + }, + { + "id": "PaudEhgnaE", + "type": "rectangle", + "x": 80, + "y": 2360.75, + "width": 1340, + "height": 74.4, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff3bf", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 324864503, + "version": 1, + "versionNonce": 1376294333, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "s8z0mZsOU7" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "s8z0mZsOU7", + "type": "text", + "x": 92, + "y": 2372.75, + "width": 1316, + "height": 50.400000000000006, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 851026162, + "version": 1, + "versionNonce": 901418407, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Key insight: Celery queue can look empty while thousands of rows remain pending in Postgres.\nScale on org-wide pending rows + inflight saturation \u2014 not queue depth alone.", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "PaudEhgnaE", + "originalText": "Key insight: Celery queue can look empty while thousands of rows remain pending in Postgres.\nScale on org-wide pending rows + inflight saturation \u2014 not queue depth alone.", + "lineHeight": 1.35 + }, + { + "id": "L1gUZBWc3s", + "type": "rectangle", + "x": 60, + "y": 2575.15, + "width": 1380, + "height": 56, + "angle": 0, + "strokeColor": "#1a365d", + "backgroundColor": "#e7f5ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1303862275, + "version": 1, + "versionNonce": 1953801125, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "9UdmztIugR", + "type": "text", + "x": 80, + "y": 2589.15, + "width": 1340, + "height": 62.8, + "angle": 0, + "strokeColor": "#1a365d", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1367222602, + "version": 1, + "versionNonce": 340872301, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "03 Call Import Pipeline", + "fontSize": 24, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "03 Call Import Pipeline", + "lineHeight": 1.35 + }, + { + "id": "qcadyT7Jfz", + "type": "text", + "x": 80, + "y": 2663.15, + "width": 1340, + "height": 54.099999999999994, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 706033874, + "version": 1, + "versionNonce": 1368311, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Two slot types: import:* (bulk CSV) vs eval:* (transcribe + scoring chain)", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Two slot types: import:* (bulk CSV) vs eval:* (transcribe + scoring chain)", + "lineHeight": 1.35 + }, + { + "id": "L4aTWa44HO", + "type": "rectangle", + "x": 470.0, + "y": 2699.15, + "width": 560, + "height": 92, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 16513670, + "version": 1, + "versionNonce": 1842759528, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "N7VahxxBDm" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "N7VahxxBDm", + "type": "text", + "x": 482.0, + "y": 2711.15, + "width": 536, + "height": 68, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 817398689, + "version": 1, + "versionNonce": 425304888, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Upload CSV\nAPI creates CallImport header", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "L4aTWa44HO", + "originalText": "Upload CSV\nAPI creates CallImport header", + "lineHeight": 1.35 + }, + { + "id": "3UkzphAsvd", + "type": "arrow", + "x": 750.0, + "y": 2795.15, + "width": 0.0, + "height": 48.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 1287589957, + "version": 1, + "versionNonce": 1561521807, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 48.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "QMzBqEZlIu", + "type": "rectangle", + "x": 470.0, + "y": 2847.15, + "width": 560, + "height": 92, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1532123169, + "version": 1, + "versionNonce": 1389395176, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "s2mH110dfN" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "s2mH110dfN", + "type": "text", + "x": 482.0, + "y": 2859.15, + "width": 536, + "height": 68, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1435163832, + "version": 1, + "versionNonce": 220550237, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Materialize rows\nBulk insert onto data shards", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "QMzBqEZlIu", + "originalText": "Materialize rows\nBulk insert onto data shards", + "lineHeight": 1.35 + }, + { + "id": "sIGJ9Auu9N", + "type": "arrow", + "x": 750.0, + "y": 2943.15, + "width": 0.0, + "height": 48.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 1918631579, + "version": 1, + "versionNonce": 1501058246, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 48.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "BOEUL5C7qn", + "type": "rectangle", + "x": 470.0, + "y": 2995.15, + "width": 560, + "height": 92, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 168254117, + "version": 1, + "versionNonce": 1865726650, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "AhY72TQ9eL" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "AhY72TQ9eL", + "type": "text", + "x": 482.0, + "y": 3007.15, + "width": 536, + "height": 68, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 70875553, + "version": 1, + "versionNonce": 386681103, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Bulk import (optional)\nprocess_call_import_row \u2014 uses import:* slots", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "BOEUL5C7qn", + "originalText": "Bulk import (optional)\nprocess_call_import_row \u2014 uses import:* slots", + "lineHeight": 1.35 + }, + { + "id": "hvuhgQYJ8A", + "type": "arrow", + "x": 750.0, + "y": 3091.15, + "width": 0.0, + "height": 48.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 1686169790, + "version": 1, + "versionNonce": 700745299, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 48.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "0bFw4HXHhN", + "type": "rectangle", + "x": 470.0, + "y": 3143.15, + "width": 560, + "height": 92, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1755608488, + "version": 1, + "versionNonce": 66776272, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "ZFv7E38zxF" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "ZFv7E38zxF", + "type": "text", + "x": 482.0, + "y": 3155.15, + "width": 536, + "height": 68, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1346601392, + "version": 1, + "versionNonce": 829540641, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Create evaluation\nCallImportEvaluation header on catalog", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "0bFw4HXHhN", + "originalText": "Create evaluation\nCallImportEvaluation header on catalog", + "lineHeight": 1.35 + }, + { + "id": "LaT0Mvvbi9", + "type": "arrow", + "x": 750.0, + "y": 3239.15, + "width": 0.0, + "height": 48.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 1098221273, + "version": 1, + "versionNonce": 838404175, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 48.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "r76KEt91GK", + "type": "rectangle", + "x": 470.0, + "y": 3291.15, + "width": 560, + "height": 92, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1733359906, + "version": 1, + "versionNonce": 1721260671, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "TAdi9JHlNn" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "TAdi9JHlNn", + "type": "text", + "x": 482.0, + "y": 3303.15, + "width": 536, + "height": 68, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1745726422, + "version": 1, + "versionNonce": 1267900309, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Fair eval dispatch\nUses eval:* slots from here onward", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "r76KEt91GK", + "originalText": "Fair eval dispatch\nUses eval:* slots from here onward", + "lineHeight": 1.35 + }, + { + "id": "mHEitva9ZY", + "type": "arrow", + "x": 750.0, + "y": 3387.15, + "width": 0.0, + "height": 48.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 79354953, + "version": 1, + "versionNonce": 1672625286, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 48.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "UCYVpqOz5K", + "type": "rectangle", + "x": 470.0, + "y": 3439.15, + "width": 560, + "height": 92, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1037143454, + "version": 1, + "versionNonce": 1787396441, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "aDYCZSg5g1" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "aDYCZSg5g1", + "type": "text", + "x": 482.0, + "y": 3451.15, + "width": 536, + "height": 68, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 283931029, + "version": 1, + "versionNonce": 1370066697, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Transcribe / diarize\nSTT + LLM diarisation queue", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "UCYVpqOz5K", + "originalText": "Transcribe / diarize\nSTT + LLM diarisation queue", + "lineHeight": 1.35 + }, + { + "id": "8rrsJ4g4So", + "type": "arrow", + "x": 750.0, + "y": 3535.15, + "width": 0.0, + "height": 48.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 1604545233, + "version": 1, + "versionNonce": 897238211, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 48.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "WRxjSGARvB", + "type": "rectangle", + "x": 470.0, + "y": 3587.15, + "width": 560, + "height": 92, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 133931527, + "version": 1, + "versionNonce": 112731326, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "p5WAlm3qa6" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "p5WAlm3qa6", + "type": "text", + "x": 482.0, + "y": 3599.15, + "width": 536, + "height": 68, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 327392428, + "version": 1, + "versionNonce": 1489453663, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "LLM + audio metrics\nFinal metric scores on shard row", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "WRxjSGARvB", + "originalText": "LLM + audio metrics\nFinal metric scores on shard row", + "lineHeight": 1.35 + }, + { + "id": "15syKbdg9X", + "type": "rectangle", + "x": 80, + "y": 3727.15, + "width": 1340, + "height": 74.4, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff3bf", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 63382381, + "version": 1, + "versionNonce": 261855256, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "FP1ZDIa9gc" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "FP1ZDIa9gc", + "type": "text", + "x": 92, + "y": 3739.15, + "width": 1316, + "height": 50.400000000000006, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1822873391, + "version": 1, + "versionNonce": 1210963697, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Eval-chain recording fetch uses eval:* slots (NOT import:*).\nOne eval slot is held from dispatch until the row finishes scoring.", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "15syKbdg9X", + "originalText": "Eval-chain recording fetch uses eval:* slots (NOT import:*).\nOne eval slot is held from dispatch until the row finishes scoring.", + "lineHeight": 1.35 + }, + { + "id": "ikzeqYPXwn", + "type": "rectangle", + "x": 60, + "y": 3941.55, + "width": 1380, + "height": 56, + "angle": 0, + "strokeColor": "#1a365d", + "backgroundColor": "#e7f5ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 495783960, + "version": 1, + "versionNonce": 248983815, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "897sSJjQd8", + "type": "text", + "x": 80, + "y": 3955.55, + "width": 1340, + "height": 62.8, + "angle": 0, + "strokeColor": "#1a365d", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1155228534, + "version": 1, + "versionNonce": 1448293122, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "04 Database Sharding", + "fontSize": 24, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "04 Database Sharding", + "lineHeight": 1.35 + }, + { + "id": "T0CEoBTkwj", + "type": "text", + "x": 80, + "y": 4029.55, + "width": 1340, + "height": 54.099999999999994, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 630762, + "version": 1, + "versionNonce": 1673147856, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Row routing spreads 10k imports across shards (~2.5k rows each)", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Row routing spreads 10k imports across shards (~2.5k rows each)", + "lineHeight": 1.35 + }, + { + "id": "68X7xjplRi", + "type": "rectangle", + "x": 540.0, + "y": 4065.55, + "width": 420, + "height": 110, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1684818790, + "version": 1, + "versionNonce": 1619194194, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "ey9Q9tZux7" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "ey9Q9tZux7", + "type": "text", + "x": 552.0, + "y": 4077.55, + "width": 396, + "height": 86, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1409257200, + "version": 1, + "versionNonce": 37415610, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Catalog DB\nCallImport \u00b7 CallImportEvaluation \u00b7 metrics \u00b7 call_import_shard_slices registry", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "68X7xjplRi", + "originalText": "Catalog DB\nCallImport \u00b7 CallImportEvaluation \u00b7 metrics \u00b7 call_import_shard_slices registry", + "lineHeight": 1.35 + }, + { + "id": "PRsunKg5WT", + "type": "rectangle", + "x": 124.0, + "y": 4231.55, + "width": 280, + "height": 80, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 61148606, + "version": 1, + "versionNonce": 958194951, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "bVAIuitkpz" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "bVAIuitkpz", + "type": "text", + "x": 136.0, + "y": 4243.55, + "width": 256, + "height": 56, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 574475616, + "version": 1, + "versionNonce": 1485109789, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "data-shard-01", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "PRsunKg5WT", + "originalText": "data-shard-01", + "lineHeight": 1.35 + }, + { + "id": "5pAOzYtmHM", + "type": "rectangle", + "x": 448.0, + "y": 4231.55, + "width": 280, + "height": 80, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 772697781, + "version": 1, + "versionNonce": 1496519256, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "XlO0GgRbzl" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "XlO0GgRbzl", + "type": "text", + "x": 460.0, + "y": 4243.55, + "width": 256, + "height": 56, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 53401825, + "version": 1, + "versionNonce": 1319710303, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "data-shard-02", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "5pAOzYtmHM", + "originalText": "data-shard-02", + "lineHeight": 1.35 + }, + { + "id": "sQK1YEjnJN", + "type": "rectangle", + "x": 772.0, + "y": 4231.55, + "width": 280, + "height": 80, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1752828089, + "version": 1, + "versionNonce": 1830029821, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "29j1cU3alh" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "29j1cU3alh", + "type": "text", + "x": 784.0, + "y": 4243.55, + "width": 256, + "height": 56, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1504936975, + "version": 1, + "versionNonce": 1173366283, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "data-shard-03", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "sQK1YEjnJN", + "originalText": "data-shard-03", + "lineHeight": 1.35 + }, + { + "id": "CN53jhKJWC", + "type": "rectangle", + "x": 1096.0, + "y": 4231.55, + "width": 280, + "height": 80, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1218166551, + "version": 1, + "versionNonce": 1815055062, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "2A1JBz4GnY" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "2A1JBz4GnY", + "type": "text", + "x": 1108.0, + "y": 4243.55, + "width": 256, + "height": 56, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 121990602, + "version": 1, + "versionNonce": 1463803155, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "data-shard-04", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "CN53jhKJWC", + "originalText": "data-shard-04", + "lineHeight": 1.35 + }, + { + "id": "ApgNNJZVB6", + "type": "arrow", + "x": 750.0, + "y": 4175.55, + "width": 0.0, + "height": 52.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 888052752, + "version": 1, + "versionNonce": 6854781, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 52.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "BP9CpCJxuT", + "type": "arrow", + "x": 750.0, + "y": 4175.55, + "width": 0.0, + "height": 52.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 442176439, + "version": 1, + "versionNonce": 1627521382, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 52.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "o27oxLgvtw", + "type": "arrow", + "x": 750.0, + "y": 4175.55, + "width": 0.0, + "height": 52.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 723419179, + "version": 1, + "versionNonce": 1711050185, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 52.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "NVWZueOgLB", + "type": "arrow", + "x": 750.0, + "y": 4175.55, + "width": 0.0, + "height": 52.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 1842146388, + "version": 1, + "versionNonce": 818495744, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 52.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "3QyJb0oIQT", + "type": "rectangle", + "x": 80, + "y": 4359.55, + "width": 1340, + "height": 101.95, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e7f5ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1199924623, + "version": 1, + "versionNonce": 1965002828, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "z2cGn8OwlE" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "z2cGn8OwlE", + "type": "text", + "x": 92, + "y": 4371.55, + "width": 1316, + "height": 77.95, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 288088742, + "version": 1, + "versionNonce": 1124275845, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Routing formula:\n slice_id = row_index // 500\n shard_id = SHA256(call_import_id : slice_id) mod 4", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "3QyJb0oIQT", + "originalText": "Routing formula:\n slice_id = row_index // 500\n shard_id = SHA256(call_import_id : slice_id) mod 4", + "lineHeight": 1.35 + }, + { + "id": "X7OeKCFmcl", + "type": "rectangle", + "x": 80, + "y": 4501.5, + "width": 1340, + "height": 74.4, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff3bf", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1250496653, + "version": 1, + "versionNonce": 1435213247, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "hNAeBHtePI" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "hNAeBHtePI", + "type": "text", + "x": 92, + "y": 4513.5, + "width": 1316, + "height": 50.400000000000006, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 854823744, + "version": 1, + "versionNonce": 548655884, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Each pod holds SQLAlchemy pools: 1 catalog + 4 shards.\nRule: concurrency per pod \u2264 catalog pool max (pool_size + max_overflow).", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "X7OeKCFmcl", + "originalText": "Each pod holds SQLAlchemy pools: 1 catalog + 4 shards.\nRule: concurrency per pod \u2264 catalog pool max (pool_size + max_overflow).", + "lineHeight": 1.35 + }, + { + "id": "zdkuaIH8Ph", + "type": "rectangle", + "x": 60, + "y": 4715.9, + "width": 1380, + "height": 56, + "angle": 0, + "strokeColor": "#1a365d", + "backgroundColor": "#e7f5ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1527941879, + "version": 1, + "versionNonce": 277010156, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "ZhtuDp1R84", + "type": "text", + "x": 80, + "y": 4729.9, + "width": 1340, + "height": 62.8, + "angle": 0, + "strokeColor": "#1a365d", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 879139622, + "version": 1, + "versionNonce": 1549760945, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "05 Fair Dispatch & Inflight Limits", + "fontSize": 24, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "05 Fair Dispatch & Inflight Limits", + "lineHeight": 1.35 + }, + { + "id": "vDB5TEqkQi", + "type": "text", + "x": 80, + "y": 4803.9, + "width": 1340, + "height": 54.099999999999994, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1009367673, + "version": 1, + "versionNonce": 451355123, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Dual 10k across two workspaces \u2014 equal fair share", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Dual 10k across two workspaces \u2014 equal fair share", + "lineHeight": 1.35 + }, + { + "id": "ZY1L6dDUT7", + "type": "rectangle", + "x": 330.0, + "y": 4839.9, + "width": 360, + "height": 100, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1894234912, + "version": 1, + "versionNonce": 1556910175, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "70nbZh5lUj" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "70nbZh5lUj", + "type": "text", + "x": 342.0, + "y": 4851.9, + "width": 336, + "height": 76, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 592015219, + "version": 1, + "versionNonce": 1926980675, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Workspace A\n10,000 rows pending", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "ZY1L6dDUT7", + "originalText": "Workspace A\n10,000 rows pending", + "lineHeight": 1.35 + }, + { + "id": "kvUZlTN4K3", + "type": "rectangle", + "x": 810.0, + "y": 4839.9, + "width": 360, + "height": 100, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 927847288, + "version": 1, + "versionNonce": 1938195071, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "vkm2v2ruMY" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "vkm2v2ruMY", + "type": "text", + "x": 822.0, + "y": 4851.9, + "width": 336, + "height": 76, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 560200721, + "version": 1, + "versionNonce": 1106346467, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Workspace B\n10,000 rows pending", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "kvUZlTN4K3", + "originalText": "Workspace B\n10,000 rows pending", + "lineHeight": 1.35 + }, + { + "id": "dp5xahHjCb", + "type": "rectangle", + "x": 450.0, + "y": 4995.9, + "width": 600, + "height": 120.8, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 150862157, + "version": 1, + "versionNonce": 63185105, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "xJaJZ33nSk" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "xJaJZ33nSk", + "type": "text", + "x": 462.0, + "y": 5007.9, + "width": 576, + "height": 96.8, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1629400200, + "version": 1, + "versionNonce": 1835883440, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Global round-robin dispatcher\n\u2022 max_workspace_turns = 999 on eval create (fill capacity)\n\u2022 max_workspace_turns = 1 after each row completes (fair refill)\n\u2022 batch_size should match eval_workspace_inflight_limit", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "dp5xahHjCb", + "originalText": "Global round-robin dispatcher\n\u2022 max_workspace_turns = 999 on eval create (fill capacity)\n\u2022 max_workspace_turns = 1 after each row completes (fair refill)\n\u2022 batch_size should match eval_workspace_inflight_limit", + "lineHeight": 1.35 + }, + { + "id": "bT6iMFP6CJ", + "type": "rectangle", + "x": 290.0, + "y": 5164.7, + "width": 420, + "height": 130, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 761158896, + "version": 1, + "versionNonce": 1999879171, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "rT1zP2ABgz" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "rT1zP2ABgz", + "type": "text", + "x": 302.0, + "y": 5176.7, + "width": 396, + "height": 106, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1065088490, + "version": 1, + "versionNonce": 98840210, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Eval slot checks\n(all must pass)\nworkspace \u2192 org \u2192 global \u2192 job", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "bT6iMFP6CJ", + "originalText": "Eval slot checks\n(all must pass)\nworkspace \u2192 org \u2192 global \u2192 job", + "lineHeight": 1.35 + }, + { + "id": "0SV3a5zxoz", + "type": "rectangle", + "x": 790.0, + "y": 5164.7, + "width": 420, + "height": 130, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff3bf", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1151190447, + "version": 1, + "versionNonce": 512039572, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "PXsDcW9zF2" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "PXsDcW9zF2", + "type": "text", + "x": 802.0, + "y": 5176.7, + "width": 396, + "height": 106, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1485867097, + "version": 1, + "versionNonce": 1835397109, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Effective parallelism\nmin(job, workspace,\nglobal, threads)", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "0SV3a5zxoz", + "originalText": "Effective parallelism\nmin(job, workspace,\nglobal, threads)", + "lineHeight": 1.35 + }, + { + "id": "j2v7Qnxmeb", + "type": "arrow", + "x": 690.0, + "y": 4889.9, + "width": -248.0, + "height": 0.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 353377791, + "version": 1, + "versionNonce": 682884335, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -248.0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "GBWpOkNsVg", + "type": "arrow", + "x": 810.0, + "y": 4889.9, + "width": 248.0, + "height": 0.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 700578811, + "version": 1, + "versionNonce": 737363216, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 248.0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "kQ5no0lYD2", + "type": "arrow", + "x": 750.0, + "y": 4939.9, + "width": 0.0, + "height": 52.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 260592044, + "version": 1, + "versionNonce": 1390584913, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 52.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "xK30v1IxcZ", + "type": "rectangle", + "x": 80, + "y": 5342.7, + "width": 1340, + "height": 97.6, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e7f5ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 886442950, + "version": 1, + "versionNonce": 612211236, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "SlqKg3qJza" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "SlqKg3qJza", + "type": "text", + "x": 92, + "y": 5354.7, + "width": 1316, + "height": 73.6, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 814750487, + "version": 1, + "versionNonce": 1177493610, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Recommended (2 workspaces, 16\u00d716 pods):\n eval_global = 256 \u00b7 eval_workspace = 128 each \u00b7 eval_job = 128\n import_global = 96 \u00b7 import_workspace = 48 each \u00b7 import_batch = 48", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "xK30v1IxcZ", + "originalText": "Recommended (2 workspaces, 16\u00d716 pods):\n eval_global = 256 \u00b7 eval_workspace = 128 each \u00b7 eval_job = 128\n import_global = 96 \u00b7 import_workspace = 48 each \u00b7 import_batch = 48", + "lineHeight": 1.35 + }, + { + "id": "JOU7gdnsAY", + "type": "rectangle", + "x": 60, + "y": 5580.3, + "width": 1380, + "height": 56, + "angle": 0, + "strokeColor": "#1a365d", + "backgroundColor": "#e7f5ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 949802229, + "version": 1, + "versionNonce": 1564457959, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "6jZNWPicgv", + "type": "text", + "x": 80, + "y": 5594.3, + "width": 1340, + "height": 62.8, + "angle": 0, + "strokeColor": "#1a365d", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1685326867, + "version": 1, + "versionNonce": 1064933184, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "06 Scaling Roadmap (KEDA + PgBouncer)", + "fontSize": 24, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "06 Scaling Roadmap (KEDA + PgBouncer)", + "lineHeight": 1.35 + }, + { + "id": "ULTdp11guI", + "type": "text", + "x": 80, + "y": 5668.3, + "width": 1340, + "height": 54.099999999999994, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 469377266, + "version": 1, + "versionNonce": 1518780927, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Phase 1\u20133 now (no PgBouncer) \u2192 Phase 4\u20135 with PgBouncer", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Phase 1\u20133 now (no PgBouncer) \u2192 Phase 4\u20135 with PgBouncer", + "lineHeight": 1.35 + }, + { + "id": "6laIcdoTDY", + "type": "rectangle", + "x": 150.0, + "y": 5704.3, + "width": 560, + "height": 280, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 175140147, + "version": 1, + "versionNonce": 1709614658, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "EzdxDeqSUT" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "EzdxDeqSUT", + "type": "text", + "x": 162.0, + "y": 5716.3, + "width": 536, + "height": 256, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1844295765, + "version": 1, + "versionNonce": 1214745328, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Now \u2014 Phases 1\u20133\n\n\u2022 8\u201316 pods \u00d7 16 concurrency\n\u2022 eval_global 256 \u00b7 job 128\n\u2022 import 96 / workspace 48 (keep)\n\u2022 KEDA on org-wide pending rows\n\u2022 No PgBouncer\n\n~256 parallel rows at max scale\nDual 10k \u2248 32 min", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "6laIcdoTDY", + "originalText": "Now \u2014 Phases 1\u20133\n\n\u2022 8\u201316 pods \u00d7 16 concurrency\n\u2022 eval_global 256 \u00b7 job 128\n\u2022 import 96 / workspace 48 (keep)\n\u2022 KEDA on org-wide pending rows\n\u2022 No PgBouncer\n\n~256 parallel rows at max scale\nDual 10k \u2248 32 min", + "lineHeight": 1.35 + }, + { + "id": "QycvVexkbv", + "type": "rectangle", + "x": 790.0, + "y": 5704.3, + "width": 560, + "height": 280, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1615767400, + "version": 1, + "versionNonce": 654061525, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "W5pP9l24lt" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "W5pP9l24lt", + "type": "text", + "x": 802.0, + "y": 5716.3, + "width": 536, + "height": 256, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1164602807, + "version": 1, + "versionNonce": 671757191, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "Future \u2014 Phases 4\u20135\n\n\u2022 16\u201320 pods \u00d7 28 concurrency\n\u2022 eval_global 560 \u00b7 workspace 280\n\u2022 PgBouncer per catalog + shard\n\u2022 transaction pool mode\n\u2022 Small app pools (5\u20138)\n\nDual 10k in ~30 min target", + "fontSize": 17, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "QycvVexkbv", + "originalText": "Future \u2014 Phases 4\u20135\n\n\u2022 16\u201320 pods \u00d7 28 concurrency\n\u2022 eval_global 560 \u00b7 workspace 280\n\u2022 PgBouncer per catalog + shard\n\u2022 transaction pool mode\n\u2022 Small app pools (5\u20138)\n\nDual 10k in ~30 min target", + "lineHeight": 1.35 + }, + { + "id": "iqG75iKkSi", + "type": "arrow", + "x": 714.0, + "y": 5844.3, + "width": 72.0, + "height": 0.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "seed": 1353476583, + "version": 1, + "versionNonce": 1561609223, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 72.0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "rXdP6v2nxz", + "type": "text", + "x": 714.0, + "y": 5826.3, + "width": 400, + "height": 49.75, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1164526210, + "version": 1, + "versionNonce": 266061570, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "PgBouncer", + "fontSize": 15, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "PgBouncer", + "lineHeight": 1.35 + }, + { + "id": "hOhSrI4f69", + "type": "rectangle", + "x": 80, + "y": 6032.3, + "width": 1340, + "height": 74.4, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff3bf", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1812030557, + "version": 1, + "versionNonce": 1402787976, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "EwapwQ8waU" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "EwapwQ8waU", + "type": "text", + "x": 92, + "y": 6044.3, + "width": 1316, + "height": 50.400000000000006, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1326600754, + "version": 1, + "versionNonce": 1134993590, + "isDeleted": false, + "boundElements": null, + "updated": 1, + "link": null, + "locked": false, + "text": "PgBouncer multiplexes many client connections into fewer server connections to RDS.\nRequired before scaling to 20 pods \u00d7 28 threads without pool_timeout errors.", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "hOhSrI4f69", + "originalText": "PgBouncer multiplexes many client connections into fewer server connections to RDS.\nRequired before scaling to 20 pods \u00d7 28 threads without pool_timeout errors.", + "lineHeight": 1.35 + } + ], + "appState": { + "gridSize": 20, + "viewBackgroundColor": "#ffffff", + "scrollX": 48, + "scrollY": 48, + "zoom": { + "value": 0.75 + } + }, + "files": {} +} \ No newline at end of file diff --git a/docs/operations/call-import-sharding-load-test.md b/docs/operations/call-import-sharding-load-test.md index d279c638..e30a0a6c 100644 --- a/docs/operations/call-import-sharding-load-test.md +++ b/docs/operations/call-import-sharding-load-test.md @@ -3,6 +3,8 @@ Scenarios A–D from the internal Confluence runbook should be executed on AWS staging (1 catalog + 5–6 row RDS instances) after enabling `database.sharding.enabled`. +**Operator quick reference:** [Call Import Scaling — Operator Handbook (Confluence)](https://efficientai.atlassian.net/wiki/spaces/ETD/pages/59932681). + ## Exit criteria - Row shard CPU ≤ 75% under scenario D (25k-row eval) @@ -18,4 +20,8 @@ Scenarios A–D from the internal Confluence runbook should be executed on AWS s | C | Concurrent workspaces (fair dispatch) | | D | 25k rows, max concurrency | +## Manual audio upload chunking + +Large manual uploads should be sent in chunks of about 25 files per request (the UI does this automatically). The API rejects more than 100 files per request on `/audio-upload` and `/audio-append`. + Record results in the customer sign-off doc after GCP production sizing is confirmed. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 2d745a02..3d5a7112 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -7,6 +7,8 @@ import Layout from './components/Layout' import Login from './pages/auth/Login' import LoginCallback from './pages/auth/LoginCallback' import SelectOrganization from './pages/auth/SelectOrganization' +import PlatformLogin from './pages/platform/PlatformLogin' +import PlatformAdmin from './pages/platform/PlatformAdmin' // Dashboard import Dashboard from './pages/dashboard/Dashboard' @@ -24,8 +26,10 @@ import Personas from './pages/personas/Personas' import Scenarios from './pages/scenarios/Scenarios' // Metrics -import Metrics from './pages/metrics/Metrics' +import MetricsLayout from './pages/metrics/MetricsLayout' import MetricsManagement from './pages/metrics/MetricsManagement' +import MetricsStudio from './pages/metrics/MetricsStudio' +import MetricsStudioRunDetail from './pages/metrics/MetricsStudioRunDetail' // Playground - Agent import AgentPlayground from './pages/playground/agent/AgentPlayground' @@ -133,6 +137,8 @@ function App() { } /> + } /> + } /> } /> } /> {/* Public blind test form - intentionally outside PrivateRoute and EnterpriseGate. @@ -158,14 +164,18 @@ function App() { } /> } /> } /> - } /> + } /> } /> } /> } /> } /> } /> } /> - } /> + }> + } /> + } /> + } /> + } /> } /> } /> diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index c621a485..f8a6b3f9 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -1,683 +1,692 @@ -import { Outlet, Link, useLocation } from 'react-router-dom' -import { useAuthStore } from '../store/authStore' -import { useAgentStore, type Agent } from '../store/agentStore' -import { useLicenseStore } from '../store/licenseStore' -import { useQuery } from '@tanstack/react-query' -import { apiClient } from '../lib/api' -import { Profile } from '../types/api' -import { - LayoutDashboard, - FileCheck, - LogOut, - Menu, - Phone, - Users, - FileText, - Shield, - Play, - BarChart3, - Plug, - ChevronRight, - ChevronDown, - ChevronLeft, - Database, - Settings, - Mic, - Bot, - Activity, - Bell, - History, - Key, - Clock, - Volume2, - Gamepad2, - Lock, - ScrollText, - Github, - Sparkles, - Upload, -} from 'lucide-react' -import { useState, useEffect } from 'react' -import Logo from './Logo' -import WorkspaceSwitcher from './WorkspaceSwitcher' -import WalkthroughRail from './walkthrough/WalkthroughRail' - -interface NavItem { - name: string - href: string - icon: React.ComponentType<{ className?: string }> - enterpriseFeature?: string -} - -interface NavSection { - title: string - items: NavItem[] - icon: React.ComponentType<{ className?: string }> -} - -const navigationSections: NavSection[] = [ - { - title: 'Simulations', - icon: Play, - items: [ - { name: 'Agents', href: '/agents', icon: Bot }, - { name: 'Personas', href: '/personas', icon: Users }, - { name: 'Scenarios', href: '/scenarios', icon: FileText }, - ], - }, - { - title: 'Playground', - icon: Gamepad2, - items: [ - { name: 'Agent Playground', href: '/playground', icon: Play }, - { name: 'Voice Playground', href: '/voice-playground', icon: Volume2, enterpriseFeature: 'voice_playground' }, - ], - }, - { - title: 'Evaluations', - icon: FileCheck, - items: [ - { name: 'Evaluators', href: '/evaluate-test-agents', icon: Mic }, - { name: 'Evaluation Results', href: '/results', icon: BarChart3 }, - { name: 'Judge Alignment', href: '/judge-alignment', icon: Sparkles }, - { name: 'Call Imports', href: '/call-imports', icon: Upload, enterpriseFeature: 'call_imports' }, - ], - }, - { - title: 'Prompts', - icon: ScrollText, - items: [ - { name: 'Partials', href: '/prompt-partials', icon: FileText }, - { name: 'Optimization', href: '/prompt-optimization', icon: Sparkles, enterpriseFeature: 'gepa_optimization' }, - ], - }, - { - title: 'Observability', - icon: BarChart3, - items: [ - { name: 'Overview', href: '/observability', icon: Activity }, - { name: 'Calls', href: '/observability/calls', icon: Phone }, - ], - }, - { - title: 'Alerting', - icon: Bell, - items: [ - { name: 'Alerts', href: '/alerts', icon: Bell }, - { name: 'Alert History', href: '/alerts/history', icon: History }, - ], - }, - { - title: 'Configurations', - icon: Settings, - items: [ - { name: 'Cloud Storage', href: '/data-sources', icon: Database }, - { name: 'VoiceBundle', href: '/voicebundles', icon: Mic }, - { name: 'Integrations', href: '/integrations', icon: Plug }, - { name: 'Telephony Numbers', href: '/telephony-numbers', icon: Phone }, - { name: 'API Keys', href: '/settings', icon: Key }, - { name: 'Cron Jobs', href: '/cron-jobs', icon: Clock }, - ], - }, -] - -const otherNavigation: NavItem[] = [ - { name: 'Dashboard', href: '/', icon: LayoutDashboard }, - { name: 'Metrics', href: '/metrics-management', icon: BarChart3 }, -] - -const bottomNavigation = [ - { name: 'IAM', href: '/iam', icon: Shield }, -] - -const SIDEBAR_COLLAPSED_KEY = 'sidebarCollapsed' - -function readSidebarCollapsedPreference(): boolean { - if (typeof window === 'undefined') return false - return localStorage.getItem(SIDEBAR_COLLAPSED_KEY) === 'true' -} - -function getFlattenedNavItems(): NavItem[] { - const items: NavItem[] = [...otherNavigation] - for (const section of navigationSections) { - items.push(...section.items) - } - items.push(...bottomNavigation) - return items -} - -export default function Layout() { - const location = useLocation() - const { logout } = useAuthStore() - const { selectedAgent, setSelectedAgent, loadPreferences, isInitialized } = useAgentStore() - const { fetchLicense } = useLicenseStore() - const [sidebarOpen, setSidebarOpen] = useState(false) - const [desktopSidebarCollapsed, setDesktopSidebarCollapsed] = useState(readSidebarCollapsedPreference) - const [showAgentDropdown, setShowAgentDropdown] = useState(false) - - const toggleDesktopSidebar = () => { - setDesktopSidebarCollapsed((prev) => { - const next = !prev - localStorage.setItem(SIDEBAR_COLLAPSED_KEY, String(next)) - return next - }) - } - - // Fetch agents - const { data: agents = [], isSuccess: agentsLoaded } = useQuery({ - queryKey: ['agents'], - queryFn: () => apiClient.listAgents(), - }) - - // Load user preferences and license info on mount - useEffect(() => { - loadPreferences() - fetchLicense() - }, [loadPreferences, fetchLicense]) - - // Auto-select first agent if none is selected (after both preferences and agents are loaded) - useEffect(() => { - if (agentsLoaded && isInitialized && !selectedAgent && agents.length > 0) { - setSelectedAgent(agents[0]) - } - }, [agents, agentsLoaded, isInitialized, selectedAgent, setSelectedAgent]) - - // Clear selection if selected agent no longer exists (only after both have loaded) - useEffect(() => { - // Only run cleanup after agents query has completed AND preferences are initialized - if (!agentsLoaded || !isInitialized) return - - if (selectedAgent && !agents.find((a: Agent) => a.id === selectedAgent.id)) { - if (agents.length > 0) { - setSelectedAgent(agents[0]) - } else { - setSelectedAgent(null) - } - } - }, [agents, agentsLoaded, isInitialized, selectedAgent, setSelectedAgent]) - - return ( -
- {/* Mobile sidebar */} -
-
setSidebarOpen(false)} - /> -
- -
-
- - {/* Desktop sidebar */} -
-
- -
-
- - {/* Main content */} -
- {/* Top bar */} -
- -
- {/* Agent Selector */} -
- - - {/* Dropdown */} - {showAgentDropdown && ( - <> -
setShowAgentDropdown(false)} - /> -
- {agents.length === 0 ? ( -
-

No agents available

- setShowAgentDropdown(false)} - className="inline-block text-sm text-primary-600 hover:text-primary-700 font-medium" - > - Create your first agent → - -
- ) : ( - <> - {agents.map((agent: Agent) => ( - - ))} -
- setShowAgentDropdown(false)} - className="text-sm text-primary-600 hover:text-primary-700 font-medium" - > - Manage Agents → - -
- - )} -
- - )} -
- - -
-
- - {/* Page content */} -
-
- -
-
- -
-
- ) -} - -function ProfileAvatar() { - const location = useLocation() - const { data: profile } = useQuery({ - queryKey: ['profile'], - queryFn: () => apiClient.getProfile(), - }) - - const getInitials = () => { - if (profile?.first_name && profile?.last_name) { - return `${profile.first_name.charAt(0).toUpperCase()}${profile.last_name.charAt(0).toUpperCase()}` - } else if (profile?.first_name) { - return profile.first_name.charAt(0).toUpperCase() - } else if (profile?.name) { - const nameParts = profile.name.trim().split(/\s+/) - if (nameParts.length >= 2) { - return `${nameParts[0].charAt(0).toUpperCase()}${nameParts[nameParts.length - 1].charAt(0).toUpperCase()}` - } - return nameParts[0].charAt(0).toUpperCase() - } else if (profile?.email) { - return profile.email.charAt(0).toUpperCase() - } - return 'U' - } - - const initials = getInitials() - - return ( -
- -
- {initials} -
- -
- ) -} - -function SidebarContent({ - onLogout, - location, - collapsed = false, - onToggleCollapse, -}: { - onLogout: () => void - location: ReturnType - collapsed?: boolean - onToggleCollapse?: () => void -}) { - const { isFeatureEnabled } = useLicenseStore() - const [expandedSections, setExpandedSections] = useState>( - new Set(['Simulations', 'Playground', 'Evaluations', 'Prompts', 'Observability', 'Alerting', 'Configurations']) - ) - - const toggleSection = (title: string) => { - const newExpanded = new Set(expandedSections) - if (newExpanded.has(title)) { - newExpanded.delete(title) - } else { - newExpanded.add(title) - } - setExpandedSections(newExpanded) - } - - const isSectionActive = (section: NavSection) => { - return section.items.some(item => location.pathname === item.href) - } - - if (collapsed) { - return ( -
-
- - {onToggleCollapse && ( - - )} -
-
- -
-
- -
-
- ) - } - - return ( -
-
- - {onToggleCollapse && ( - - )} -
- {/* Workspace switcher - sits above the Dashboard nav so the - active workspace context is always visible in the sidebar - (and not buried in the top header). The small label above - the switcher makes it explicit that the dropdown scopes - the whole left-nav to a workspace, not just an org. */} -
-
- Workspace -
- -
-
- - -
-
- -
-
- ) -} - -function SidebarIconLink({ - item, - isActive, - isGated, -}: { - item: NavItem - isActive: boolean - isGated: boolean -}) { - return ( - - - {isGated && ( - - )} - - ) -} - +import { Outlet, Link, useLocation } from 'react-router-dom' +import { useAuthStore } from '../store/authStore' +import { useAgentStore, type Agent } from '../store/agentStore' +import { useLicenseStore } from '../store/licenseStore' +import { useQuery } from '@tanstack/react-query' +import { apiClient } from '../lib/api' +import { Profile } from '../types/api' +import { + LayoutDashboard, + FileCheck, + LogOut, + Menu, + Phone, + Users, + FileText, + Shield, + Play, + BarChart3, + Plug, + ChevronRight, + ChevronDown, + ChevronLeft, + Database, + Settings, + Mic, + Bot, + Activity, + Bell, + History, + Key, + Clock, + Volume2, + Gamepad2, + Lock, + ScrollText, + Github, + Sparkles, + Upload, +} from 'lucide-react' +import { useState, useEffect } from 'react' +import Logo from './Logo' +import WorkspaceSwitcher from './WorkspaceSwitcher' +import WalkthroughRail from './walkthrough/WalkthroughRail' + +interface NavItem { + name: string + href: string + icon: React.ComponentType<{ className?: string }> + enterpriseFeature?: string +} + +interface NavSection { + title: string + items: NavItem[] + icon: React.ComponentType<{ className?: string }> +} + +const navigationSections: NavSection[] = [ + { + title: 'Simulations', + icon: Play, + items: [ + { name: 'Agents', href: '/agents', icon: Bot }, + { name: 'Personas', href: '/personas', icon: Users }, + { name: 'Scenarios', href: '/scenarios', icon: FileText }, + ], + }, + { + title: 'Playground', + icon: Gamepad2, + items: [ + { name: 'Agent Playground', href: '/playground', icon: Play }, + { name: 'Voice Playground', href: '/voice-playground', icon: Volume2, enterpriseFeature: 'voice_playground' }, + ], + }, + { + title: 'Evaluations', + icon: FileCheck, + items: [ + { name: 'Evaluators', href: '/evaluate-test-agents', icon: Mic }, + { name: 'Evaluation Results', href: '/results', icon: BarChart3 }, + { name: 'Judge Alignment', href: '/judge-alignment', icon: Sparkles }, + { name: 'Call Imports', href: '/call-imports', icon: Upload, enterpriseFeature: 'call_imports' }, + ], + }, + { + title: 'Prompts', + icon: ScrollText, + items: [ + { name: 'Partials', href: '/prompt-partials', icon: FileText }, + { name: 'Optimization', href: '/prompt-optimization', icon: Sparkles, enterpriseFeature: 'gepa_optimization' }, + ], + }, + { + title: 'Observability', + icon: BarChart3, + items: [ + { name: 'Overview', href: '/observability', icon: Activity }, + { name: 'Calls', href: '/observability/calls', icon: Phone }, + ], + }, + { + title: 'Alerting', + icon: Bell, + items: [ + { name: 'Alerts', href: '/alerts', icon: Bell }, + { name: 'Alert History', href: '/alerts/history', icon: History }, + ], + }, + { + title: 'Configurations', + icon: Settings, + items: [ + { name: 'Cloud Storage', href: '/data-sources', icon: Database }, + { name: 'VoiceBundle', href: '/voicebundles', icon: Mic }, + { name: 'Integrations', href: '/integrations', icon: Plug }, + { name: 'Telephony Numbers', href: '/telephony-numbers', icon: Phone }, + { name: 'API Keys', href: '/settings', icon: Key }, + { name: 'Cron Jobs', href: '/cron-jobs', icon: Clock }, + ], + }, +] + +const otherNavigation: NavItem[] = [ + { name: 'Dashboard', href: '/', icon: LayoutDashboard }, + { name: 'Metrics', href: '/metrics-management', icon: BarChart3 }, +] + +const bottomNavigation = [ + { name: 'IAM', href: '/iam', icon: Shield }, +] + +const SIDEBAR_COLLAPSED_KEY = 'sidebarCollapsed' + +function readSidebarCollapsedPreference(): boolean { + if (typeof window === 'undefined') return false + return localStorage.getItem(SIDEBAR_COLLAPSED_KEY) === 'true' +} + +function getFlattenedNavItems(): NavItem[] { + const items: NavItem[] = [...otherNavigation] + for (const section of navigationSections) { + items.push(...section.items) + } + items.push(...bottomNavigation) + return items +} + +function isNavItemActive(href: string, pathname: string): boolean { + if (href === '/metrics-management') { + return ( + pathname === href || pathname.startsWith('/metrics-management/') + ) + } + return pathname === href +} + +export default function Layout() { + const location = useLocation() + const { logout } = useAuthStore() + const { selectedAgent, setSelectedAgent, loadPreferences, isInitialized } = useAgentStore() + const { fetchLicense } = useLicenseStore() + const [sidebarOpen, setSidebarOpen] = useState(false) + const [desktopSidebarCollapsed, setDesktopSidebarCollapsed] = useState(readSidebarCollapsedPreference) + const [showAgentDropdown, setShowAgentDropdown] = useState(false) + + const toggleDesktopSidebar = () => { + setDesktopSidebarCollapsed((prev) => { + const next = !prev + localStorage.setItem(SIDEBAR_COLLAPSED_KEY, String(next)) + return next + }) + } + + // Fetch agents + const { data: agents = [], isSuccess: agentsLoaded } = useQuery({ + queryKey: ['agents'], + queryFn: () => apiClient.listAgents(), + }) + + // Load user preferences and license info on mount + useEffect(() => { + loadPreferences() + fetchLicense() + }, [loadPreferences, fetchLicense]) + + // Auto-select first agent if none is selected (after both preferences and agents are loaded) + useEffect(() => { + if (agentsLoaded && isInitialized && !selectedAgent && agents.length > 0) { + setSelectedAgent(agents[0]) + } + }, [agents, agentsLoaded, isInitialized, selectedAgent, setSelectedAgent]) + + // Clear selection if selected agent no longer exists (only after both have loaded) + useEffect(() => { + // Only run cleanup after agents query has completed AND preferences are initialized + if (!agentsLoaded || !isInitialized) return + + if (selectedAgent && !agents.find((a: Agent) => a.id === selectedAgent.id)) { + if (agents.length > 0) { + setSelectedAgent(agents[0]) + } else { + setSelectedAgent(null) + } + } + }, [agents, agentsLoaded, isInitialized, selectedAgent, setSelectedAgent]) + + return ( +
+ {/* Mobile sidebar */} +
+
setSidebarOpen(false)} + /> +
+ +
+
+ + {/* Desktop sidebar */} +
+
+ +
+
+ + {/* Main content */} +
+ {/* Top bar */} +
+ +
+ {/* Agent Selector */} +
+ + + {/* Dropdown */} + {showAgentDropdown && ( + <> +
setShowAgentDropdown(false)} + /> +
+ {agents.length === 0 ? ( +
+

No agents available

+ setShowAgentDropdown(false)} + className="inline-block text-sm text-primary-600 hover:text-primary-700 font-medium" + > + Create your first agent → + +
+ ) : ( + <> + {agents.map((agent: Agent) => ( + + ))} +
+ setShowAgentDropdown(false)} + className="text-sm text-primary-600 hover:text-primary-700 font-medium" + > + Manage Agents → + +
+ + )} +
+ + )} +
+ + +
+
+ + {/* Page content */} +
+
+ +
+
+ +
+
+ ) +} + +function ProfileAvatar() { + const location = useLocation() + const { data: profile } = useQuery({ + queryKey: ['profile'], + queryFn: () => apiClient.getProfile(), + }) + + const getInitials = () => { + if (profile?.first_name && profile?.last_name) { + return `${profile.first_name.charAt(0).toUpperCase()}${profile.last_name.charAt(0).toUpperCase()}` + } else if (profile?.first_name) { + return profile.first_name.charAt(0).toUpperCase() + } else if (profile?.name) { + const nameParts = profile.name.trim().split(/\s+/) + if (nameParts.length >= 2) { + return `${nameParts[0].charAt(0).toUpperCase()}${nameParts[nameParts.length - 1].charAt(0).toUpperCase()}` + } + return nameParts[0].charAt(0).toUpperCase() + } else if (profile?.email) { + return profile.email.charAt(0).toUpperCase() + } + return 'U' + } + + const initials = getInitials() + + return ( +
+ +
+ {initials} +
+ +
+ ) +} + +function SidebarContent({ + onLogout, + location, + collapsed = false, + onToggleCollapse, +}: { + onLogout: () => void + location: ReturnType + collapsed?: boolean + onToggleCollapse?: () => void +}) { + const { isFeatureEnabled } = useLicenseStore() + const [expandedSections, setExpandedSections] = useState>( + new Set(['Simulations', 'Playground', 'Evaluations', 'Prompts', 'Observability', 'Alerting', 'Configurations']) + ) + + const toggleSection = (title: string) => { + const newExpanded = new Set(expandedSections) + if (newExpanded.has(title)) { + newExpanded.delete(title) + } else { + newExpanded.add(title) + } + setExpandedSections(newExpanded) + } + + const isSectionActive = (section: NavSection) => { + return section.items.some(item => location.pathname === item.href) + } + + if (collapsed) { + return ( +
+
+ + {onToggleCollapse && ( + + )} +
+
+ +
+
+ +
+
+ ) + } + + return ( +
+
+ + {onToggleCollapse && ( + + )} +
+ {/* Workspace switcher - sits above the Dashboard nav so the + active workspace context is always visible in the sidebar + (and not buried in the top header). The small label above + the switcher makes it explicit that the dropdown scopes + the whole left-nav to a workspace, not just an org. */} +
+
+ Workspace +
+ +
+
+ + +
+
+ +
+
+ ) +} + +function SidebarIconLink({ + item, + isActive, + isGated, +}: { + item: NavItem + isActive: boolean + isGated: boolean +}) { + return ( + + + {isGated && ( + + )} + + ) +} + diff --git a/frontend/src/components/WorkspaceSwitcher.tsx b/frontend/src/components/WorkspaceSwitcher.tsx index 571fb3f1..7ca29b41 100644 --- a/frontend/src/components/WorkspaceSwitcher.tsx +++ b/frontend/src/components/WorkspaceSwitcher.tsx @@ -45,11 +45,13 @@ export default function WorkspaceSwitcher() { useEffect(() => { if (!workspaces.length) return const stored = activeId - const isValid = stored && workspaces.some((w) => w.id === stored) + const activeWorkspaces = workspaces.filter((w) => w.is_active) + const selectable = activeWorkspaces.length > 0 ? activeWorkspaces : workspaces + const isValid = stored && selectable.some((w) => w.id === stored) const fallback = - (isValid ? workspaces.find((w) => w.id === stored) : null) ?? - workspaces.find((w) => w.is_default) ?? - workspaces[0] + (isValid ? selectable.find((w) => w.id === stored) : null) ?? + selectable.find((w) => w.is_default) ?? + selectable[0] if (!fallback) return @@ -58,7 +60,7 @@ export default function WorkspaceSwitcher() { return } - const current = workspaces.find((w) => w.id === stored) + const current = selectable.find((w) => w.id === stored) if (!current) return const nextCaps = current.capabilities ?? [] @@ -78,6 +80,9 @@ export default function WorkspaceSwitcher() { ) const handleSelect = async (workspace: Workspace) => { + if (!workspace.is_active) { + return + } if (workspace.id === activeId) { setOpen(false) return @@ -152,14 +157,18 @@ export default function WorkspaceSwitcher() { )} {workspaces.map((ws) => { const isCurrent = ws.id === activeId + const isInactive = !ws.is_active return ( diff --git a/frontend/src/components/callImports/AuditMetaChips.tsx b/frontend/src/components/callImports/AuditMetaChips.tsx index fbe0d8c6..4b74ae6a 100644 --- a/frontend/src/components/callImports/AuditMetaChips.tsx +++ b/frontend/src/components/callImports/AuditMetaChips.tsx @@ -1,142 +1,142 @@ -import type { ReactNode } from 'react' - -/** Inline audit metadata for call imports and evaluations. */ - -export function formatMetaDateTime(iso: string | null | undefined): string { - if (!iso) return '—' - const parsed = new Date(iso) - if (Number.isNaN(parsed.getTime())) return '—' - return parsed.toLocaleString() -} - -type AuditMetaInlineItemProps = { - label: string - value: string | null | undefined - /** `text-xs` for dense rows (evaluation list); default `text-sm` */ - dense?: boolean - className?: string -} - -export function AuditMetaChip({ - label, - value, - dense, - className = '', -}: AuditMetaInlineItemProps) { - const display = value?.trim() || '—' - const sizeClass = dense ? 'text-xs' : 'text-sm' - return ( - - {label}: - {display} - - ) -} - -type AuditMetaRowProps = { - className?: string - dense?: boolean - /** Join parent flex row (chips become siblings of status/provider). */ - inline?: boolean - children: ReactNode -} - -function AuditMetaRow({ className = '', dense, inline, children }: AuditMetaRowProps) { - const textClass = dense ? 'text-xs' : 'text-sm' - if (inline) { - return
{children}
- } - return ( -
- {children} -
- ) -} - -type CallImportAuditMetaProps = { - createdAt: string | null | undefined - updatedAt: string | null | undefined - createdByEmail?: string | null - lastUpdatedByEmail?: string | null - className?: string - dense?: boolean - inline?: boolean -} - -/** Created / updated timestamps + actor emails for a call-import batch. */ -export function CallImportAuditMeta({ - createdAt, - updatedAt, - createdByEmail, - lastUpdatedByEmail, - className, - dense, - inline, -}: CallImportAuditMetaProps) { - return ( - - - - - - - ) -} - -type EvaluationAuditMetaProps = { - createdAt?: string | null | undefined - updatedAt?: string | null | undefined - startedAt?: string | null - finishedAt?: string | null - runByEmail?: string | null - createdByEmail?: string | null - lastUpdatedByEmail?: string | null - formatDate?: (iso: string | null | undefined) => string - className?: string - dense?: boolean - inline?: boolean - showRunTimes?: boolean - showTimestamps?: boolean -} - -/** Evaluation run metadata (detail header or list card). */ -export function EvaluationAuditMeta({ - createdAt, - updatedAt, - startedAt, - finishedAt, - runByEmail, - createdByEmail, - lastUpdatedByEmail, - formatDate = formatMetaDateTime, - className, - dense, - inline, - showRunTimes = true, - showTimestamps = false, -}: EvaluationAuditMetaProps) { - const runner = runByEmail ?? createdByEmail - return ( - - - - {showTimestamps && createdAt ? ( - - ) : null} - {showTimestamps && updatedAt ? ( - - ) : null} - {showRunTimes && startedAt ? ( - - ) : null} - {showRunTimes && finishedAt ? ( - - ) : null} - - ) -} +import type { ReactNode } from 'react' + +/** Inline audit metadata for call imports and evaluations. */ + +export function formatMetaDateTime(iso: string | null | undefined): string { + if (!iso) return '—' + const parsed = new Date(iso) + if (Number.isNaN(parsed.getTime())) return '—' + return parsed.toLocaleString() +} + +type AuditMetaInlineItemProps = { + label: string + value: string | null | undefined + /** `text-xs` for dense rows (evaluation list); default `text-sm` */ + dense?: boolean + className?: string +} + +export function AuditMetaChip({ + label, + value, + dense, + className = '', +}: AuditMetaInlineItemProps) { + const display = value?.trim() || '—' + const sizeClass = dense ? 'text-xs' : 'text-sm' + return ( + + {label}: + {display} + + ) +} + +type AuditMetaRowProps = { + className?: string + dense?: boolean + /** Join parent flex row (chips become siblings of status/provider). */ + inline?: boolean + children: ReactNode +} + +function AuditMetaRow({ className = '', dense, inline, children }: AuditMetaRowProps) { + const textClass = dense ? 'text-xs' : 'text-sm' + if (inline) { + return
{children}
+ } + return ( +
+ {children} +
+ ) +} + +type CallImportAuditMetaProps = { + createdAt: string | null | undefined + updatedAt: string | null | undefined + createdByEmail?: string | null + lastUpdatedByEmail?: string | null + className?: string + dense?: boolean + inline?: boolean +} + +/** Created / updated timestamps + actor emails for a call-import batch. */ +export function CallImportAuditMeta({ + createdAt, + updatedAt, + createdByEmail, + lastUpdatedByEmail, + className, + dense, + inline, +}: CallImportAuditMetaProps) { + return ( + + + + + + + ) +} + +type EvaluationAuditMetaProps = { + createdAt?: string | null | undefined + updatedAt?: string | null | undefined + startedAt?: string | null + finishedAt?: string | null + runByEmail?: string | null + createdByEmail?: string | null + lastUpdatedByEmail?: string | null + formatDate?: (iso: string | null | undefined) => string + className?: string + dense?: boolean + inline?: boolean + showRunTimes?: boolean + showTimestamps?: boolean +} + +/** Evaluation run metadata (detail header or list card). */ +export function EvaluationAuditMeta({ + createdAt, + updatedAt, + startedAt, + finishedAt, + runByEmail, + createdByEmail, + lastUpdatedByEmail, + formatDate = formatMetaDateTime, + className, + dense, + inline, + showRunTimes = true, + showTimestamps = false, +}: EvaluationAuditMetaProps) { + const runner = runByEmail ?? createdByEmail + return ( + + + + {showTimestamps && createdAt ? ( + + ) : null} + {showTimestamps && updatedAt ? ( + + ) : null} + {showRunTimes && startedAt ? ( + + ) : null} + {showRunTimes && finishedAt ? ( + + ) : null} + + ) +} diff --git a/frontend/src/components/iam/WorkspaceMembersSection.tsx b/frontend/src/components/iam/WorkspaceMembersSection.tsx index 3a1edbc9..f08fbb86 100644 --- a/frontend/src/components/iam/WorkspaceMembersSection.tsx +++ b/frontend/src/components/iam/WorkspaceMembersSection.tsx @@ -1,9 +1,9 @@ import { useEffect, useMemo, useState } from 'react' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { ChevronDown, FolderKanban, Trash2, UserPlus, Users } from 'lucide-react' +import { ChevronDown, FolderKanban, Settings, Trash2, UserPlus, Users, X } from 'lucide-react' import { apiClient } from '../../lib/api' import { getApiErrorMessage } from '../../lib/apiErrors' -import { useCanWrite, useIsReader } from '../../hooks/useRole' +import { useCanWrite, useIsAdmin, useIsReader } from '../../hooks/useRole' import { useToast } from '../../hooks/useToast' import { useAuthStore } from '../../store/authStore' import { useWorkspaceStore } from '../../store/workspaceStore' @@ -18,6 +18,7 @@ function workspaceCaps(caps: string[] | undefined) { return { canViewMembers: list.includes('workspace.members.view'), canManageMembers: list.includes('workspace.members.manage'), + canManageSettings: list.includes('workspace.settings'), } } @@ -34,13 +35,18 @@ export default function WorkspaceMembersSection() { const queryClient = useQueryClient() const { showToast, ToastContainer } = useToast() const canWrite = useCanWrite() + const isAdmin = useIsAdmin() const isReader = useIsReader() const currentUserId = useAuthStore((s) => s.user?.id ?? null) const activeWorkspaceId = useWorkspaceStore((s) => s.activeWorkspaceId) + const switchWorkspace = useWorkspaceStore((s) => s.switchWorkspace) const [selectedWorkspaceId, setSelectedWorkspaceId] = useState(null) const [showAdd, setShowAdd] = useState(false) const [selectedUserId, setSelectedUserId] = useState('') const [selectedRoleId, setSelectedRoleId] = useState('') + const [editingName, setEditingName] = useState(false) + const [nameDraft, setNameDraft] = useState('') + const [showDeactivateModal, setShowDeactivateModal] = useState(false) const { data: workspaces = [], isLoading: workspacesLoading } = useQuery({ queryKey: ['workspaces'], @@ -63,10 +69,56 @@ export default function WorkspaceMembersSection() { [workspaces, selectedWorkspaceId], ) - const { canViewMembers, canManageMembers: wsCanManageMembers } = workspaceCaps( - selectedWorkspace?.capabilities, - ) + const { canViewMembers, canManageMembers: wsCanManageMembers, canManageSettings } = + workspaceCaps(selectedWorkspace?.capabilities) const canManageMembers = wsCanManageMembers && canWrite + const canEditWorkspaceName = canManageSettings && canWrite && selectedWorkspace?.is_active + + useEffect(() => { + if (!selectedWorkspace) return + setEditingName(false) + setNameDraft(selectedWorkspace.name) + }, [selectedWorkspace?.id, selectedWorkspace?.name]) + + const renameMutation = useMutation({ + mutationFn: (name: string) => + apiClient.updateWorkspace(selectedWorkspaceId!, { name }), + onSuccess: (updated) => { + queryClient.invalidateQueries({ queryKey: ['workspaces'] }) + if (activeWorkspaceId === updated.id) { + switchWorkspace(updated.id, updated.capabilities ?? []) + } + setEditingName(false) + showToast('Workspace name updated', 'success') + }, + onError: (error: unknown) => { + showToast(getApiErrorMessage(error, 'Failed to update workspace name'), 'error') + }, + }) + + const activeStatusMutation = useMutation({ + mutationFn: (is_active: boolean) => + apiClient.updateWorkspace(selectedWorkspaceId!, { is_active }), + onSuccess: (updated) => { + queryClient.invalidateQueries({ queryKey: ['workspaces'] }) + if (!updated.is_active && activeWorkspaceId === updated.id) { + const fallback = + workspaces.find((w) => w.is_default && w.is_active) ?? + workspaces.find((w) => w.is_active && w.id !== updated.id) + if (fallback) { + switchWorkspace(fallback.id, fallback.capabilities ?? []) + } + } + setShowDeactivateModal(false) + showToast( + updated.is_active ? 'Workspace reactivated' : 'Workspace deactivated', + 'success', + ) + }, + onError: (error: unknown) => { + showToast(getApiErrorMessage(error, 'Failed to update workspace status'), 'error') + }, + }) const { data: members = [], @@ -234,6 +286,7 @@ export default function WorkspaceMembersSection() { ))} @@ -262,6 +315,126 @@ export default function WorkspaceMembersSection() { )}
+ {selectedWorkspace && ( +
+

+ + Workspace Settings +

+ +
+ + {editingName && canEditWorkspaceName ? ( +
+ setNameDraft(e.target.value)} + maxLength={255} + className="flex-1 px-3 py-2 border border-gray-300 rounded-lg text-sm focus:outline-none focus:ring-2 focus:ring-primary-500" + /> + + +
+ ) : ( +
+ + {selectedWorkspace.name} + + {canEditWorkspaceName && ( + + )} +
+ )} +

+ Slug stays fixed for links:{' '} + {selectedWorkspace.slug} +

+
+ + {isAdmin && ( +
+
+
+

+ Status +

+ + {selectedWorkspace.is_active ? 'Active' : 'Inactive'} + + {!selectedWorkspace.is_active && ( +

+ Inactive workspaces are fully locked for all non-org-admin + users. Reactivate to restore access. +

+ )} +
+ {selectedWorkspace.is_active ? ( + + ) : ( + + )} +
+
+ )} +
+ )} + {!selectedWorkspaceId ? (
{workspacesLoading ? 'Loading workspaces…' : 'Select a workspace.'} @@ -400,6 +573,54 @@ export default function WorkspaceMembersSection() { )}
+ + {showDeactivateModal && selectedWorkspace && ( +
setShowDeactivateModal(false)} + > +
e.stopPropagation()} + > +
+

Deactivate Workspace

+ +
+
+

+ Deactivate {selectedWorkspace.name}? + All non-org-admin users will lose access immediately, including read access. + Background processing for this workspace will stop. +

+
+ + +
+
+
+
+ )}
) } diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 1ac0029d..aaa95382 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1,4 +1,10 @@ import axios, { AxiosInstance } from 'axios' +import { + getApiErrorDetail, + isOrganizationAccessDenied, + organizationAccessDeniedMessage, + redirectToLoginWithMessage, +} from './authSession' import type { GenerateScenariosFromPromptParams, GenerateTestPromptParams, @@ -137,6 +143,7 @@ export interface AuthProviderConfig { export interface AuthConfigResponse { providers: AuthProviderConfig[] tier: 'oss' | 'enterprise' + gated_signup?: boolean } export interface AuthUserSummary { @@ -170,6 +177,58 @@ export interface LoginOrgSelectionResponse { organizations: LoginOrgOption[] } +export interface PlatformAdminSummary { + id: string + email: string +} + +export interface PlatformTokenResponse { + access_token: string + token_type: string + expires_in: number + admin: PlatformAdminSummary +} + +export interface PlatformOrganizationItem { + id: string + name: string + is_active: boolean + member_count: number + created_at?: string | null + disabled_at?: string | null +} + +export interface PlatformOrganizationListResponse { + items: PlatformOrganizationItem[] + total: number + offset: number + limit: number +} + +export interface PlatformOrganizationStats { + total: number + active: number + disabled: number +} + +export interface PlatformOrgUser { + id: string + email: string + role: string + is_active: boolean +} + +export interface PlatformSignupCode { + id: string + label?: string | null + max_uses?: number | null + use_count: number + expires_at?: string | null + is_active: boolean + created_at?: string | null + code?: string | null +} + export type LoginResponse = TokenResponse | LoginOrgSelectionResponse export function isLoginOrgSelectionResponse( @@ -514,17 +573,28 @@ class ApiClient { } const originalRequest = error.config + const requestUrl = String(originalRequest?.url || '') + const isAuthEndpoint = + requestUrl.includes('/auth/login') || + requestUrl.includes('/auth/signup') || + requestUrl.includes('/auth/refresh') || + requestUrl.includes('/auth/config') + + const detail = getApiErrorDetail(error) + if ( + response?.status === 403 && + isOrganizationAccessDenied(detail) && + !isAuthEndpoint + ) { + redirectToLoginWithMessage(organizationAccessDeniedMessage(detail)) + return Promise.reject(error) + } + if ( response?.status === 401 && originalRequest && !originalRequest._retry ) { - const url = String(originalRequest.url || '') - const isAuthEndpoint = - url.includes('/auth/login') || - url.includes('/auth/signup') || - url.includes('/auth/refresh') - if (!isAuthEndpoint) { originalRequest._retry = true const newToken = await this.tryRefreshAccessToken() @@ -566,7 +636,18 @@ class ApiClient { } return access_token as string }) - .catch(() => null) + .catch((err) => { + const refreshDetail = getApiErrorDetail(err) + if ( + err?.response?.status === 403 && + isOrganizationAccessDenied(refreshDetail) + ) { + redirectToLoginWithMessage( + organizationAccessDeniedMessage(refreshDetail), + ) + } + return null + }) .finally(() => { refreshPromise = null }) @@ -615,7 +696,7 @@ class ApiClient { async updateWorkspace( workspaceId: string, - payload: { name: string }, + payload: { name?: string; is_active?: boolean }, ): Promise { const response = await this.client.patch( `/api/v1/workspaces/${workspaceId}`, @@ -713,11 +794,121 @@ class ApiClient { organization_name?: string first_name?: string last_name?: string + reference_code?: string }): Promise { const response = await this.client.post('/api/v1/auth/signup', data) return response.data } + private platformHeaders() { + const token = localStorage.getItem('platformAccessToken') + return token ? { Authorization: `Bearer ${token}` } : {} + } + + async platformLogin(email: string, password: string): Promise { + const response = await axios.post( + `${API_BASE_URL}/api/v1/platform/auth/login`, + { email, password }, + { headers: { 'Content-Type': 'application/json' } }, + ) + return response.data + } + + async getPlatformOrganizationStats(): Promise { + const response = await axios.get(`${API_BASE_URL}/api/v1/platform/organizations/stats`, { + headers: this.platformHeaders(), + }) + return response.data + } + + async listPlatformOrganizations(params?: { + offset?: number + limit?: number + search?: string + is_active?: boolean + }): Promise { + const response = await axios.get(`${API_BASE_URL}/api/v1/platform/organizations`, { + headers: this.platformHeaders(), + params, + }) + return response.data + } + + async updatePlatformOrganization( + orgId: string, + data: { is_active: boolean }, + ): Promise { + const response = await axios.patch( + `${API_BASE_URL}/api/v1/platform/organizations/${orgId}`, + data, + { headers: { ...this.platformHeaders(), 'Content-Type': 'application/json' } }, + ) + return response.data + } + + async listPlatformOrganizationUsers( + orgId: string, + params?: { role?: string }, + ): Promise { + const response = await axios.get( + `${API_BASE_URL}/api/v1/platform/organizations/${orgId}/users`, + { headers: this.platformHeaders(), params }, + ) + return response.data + } + + async platformResetUserPassword( + orgId: string, + userId: string, + newPassword: string, + ): Promise<{ user_id: string; email: string; message: string }> { + const response = await axios.post( + `${API_BASE_URL}/api/v1/platform/organizations/${orgId}/users/${userId}/reset-password`, + { new_password: newPassword }, + { headers: { ...this.platformHeaders(), 'Content-Type': 'application/json' } }, + ) + return response.data + } + + async listPlatformSignupCodes(): Promise { + const response = await axios.get(`${API_BASE_URL}/api/v1/platform/signup-codes`, { + headers: this.platformHeaders(), + }) + return response.data + } + + async createPlatformSignupCode(data: { + code: string + label?: string + max_uses?: number + expires_at?: string + }): Promise { + const response = await axios.post(`${API_BASE_URL}/api/v1/platform/signup-codes`, data, { + headers: { ...this.platformHeaders(), 'Content-Type': 'application/json' }, + }) + return response.data + } + + async updatePlatformSignupCode( + codeId: string, + data: { is_active?: boolean; max_uses?: number; label?: string }, + ): Promise { + const response = await axios.patch( + `${API_BASE_URL}/api/v1/platform/signup-codes/${codeId}`, + data, + { headers: { ...this.platformHeaders(), 'Content-Type': 'application/json' } }, + ) + return response.data + } + + async deactivatePlatformSignupCode(codeId: string): Promise { + const response = await axios.delete( + `${API_BASE_URL}/api/v1/platform/signup-codes/${codeId}`, + { headers: this.platformHeaders() }, + ) + return response.data + } + async loginWithPassword( email: string, password: string, @@ -1763,6 +1954,7 @@ class ApiClient { options: { dataset: string tagIds?: string[] + batchName?: string }, ): Promise { const formData = new FormData() @@ -1770,6 +1962,9 @@ class ApiClient { formData.append('files', file) } formData.append('dataset', options.dataset) + if (options.batchName?.trim()) { + formData.append('batch_name', options.batchName.trim()) + } if (options.tagIds && options.tagIds.length > 0) { for (const tagId of options.tagIds) { formData.append('tag_ids', tagId) @@ -1781,6 +1976,118 @@ class ApiClient { return response.data } + async appendCallImportAudio( + importId: string, + files: File[], + ): Promise { + const formData = new FormData() + for (const file of files) { + formData.append('files', file) + } + const response = await this.client.post( + `/api/v1/call-imports/${importId}/audio-append`, + formData, + { + headers: { 'Content-Type': 'multipart/form-data' }, + }, + ) + return response.data + } + + async uploadCallImportAudioChunked( + files: File[], + options: { + dataset: string + tagIds?: string[] + batchName?: string + onProgress?: (progress: CallImportAudioUploadProgress) => void + }, + ): Promise { + if (files.length === 0) { + throw new Error('At least one audio file is required.') + } + + const chunks: File[][] = [] + for (let i = 0; i < files.length; i += AUDIO_UPLOAD_CHUNK_SIZE) { + chunks.push(files.slice(i, i + AUDIO_UPLOAD_CHUNK_SIZE)) + } + + let result: CallImportUploadResponse | null = null + for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex += 1) { + const chunk = chunks[chunkIndex] + options.onProgress?.({ + chunkIndex: chunkIndex + 1, + totalChunks: chunks.length, + uploadedFiles: chunkIndex * AUDIO_UPLOAD_CHUNK_SIZE, + totalFiles: files.length, + }) + + if (chunkIndex === 0) { + result = await this.uploadCallImportAudio(chunk, { + dataset: options.dataset, + tagIds: options.tagIds, + batchName: options.batchName, + }) + } else { + result = await this.appendCallImportAudio(result!.id, chunk) + } + + options.onProgress?.({ + chunkIndex: chunkIndex + 1, + totalChunks: chunks.length, + uploadedFiles: Math.min( + (chunkIndex + 1) * AUDIO_UPLOAD_CHUNK_SIZE, + files.length, + ), + totalFiles: files.length, + }) + } + + return result! + } + + async uploadCallImportAudioAppendChunked( + importId: string, + files: File[], + options?: { + onProgress?: (progress: CallImportAudioUploadProgress) => void + }, + ): Promise { + if (files.length === 0) { + throw new Error('At least one audio file is required.') + } + + const chunks: File[][] = [] + for (let i = 0; i < files.length; i += AUDIO_UPLOAD_CHUNK_SIZE) { + chunks.push(files.slice(i, i + AUDIO_UPLOAD_CHUNK_SIZE)) + } + + let result: CallImportUploadResponse | null = null + for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex += 1) { + const chunk = chunks[chunkIndex] + options?.onProgress?.({ + chunkIndex: chunkIndex + 1, + totalChunks: chunks.length, + uploadedFiles: chunkIndex * AUDIO_UPLOAD_CHUNK_SIZE, + totalFiles: files.length, + }) + + result = await this.appendCallImportAudio(importId, chunk) + + options?.onProgress?.({ + chunkIndex: chunkIndex + 1, + totalChunks: chunks.length, + uploadedFiles: Math.min( + (chunkIndex + 1) * AUDIO_UPLOAD_CHUNK_SIZE, + files.length, + ), + totalFiles: files.length, + }) + } + + return result! + } + /** * MAP stage of the staged call-import flow. * @@ -1939,6 +2246,7 @@ class ApiClient { async updateCallImport( id: string, payload: { + original_filename?: string | null dataset?: string | null tag_ids?: string[] /** @@ -3632,16 +3940,129 @@ class ApiClient { return response.data } - async listMetrics(surface?: string, includeChildren: boolean = true): Promise { + async listMetrics( + surface?: string, + includeChildren: boolean = true, + options?: { includeDrafts?: boolean; draftsOnly?: boolean; enabledOnly?: boolean }, + ): Promise { const response = await this.client.get('/api/v1/metrics', { params: { ...(surface ? { surface } : {}), include_children: includeChildren, + ...(options?.includeDrafts ? { include_drafts: true } : {}), + ...(options?.draftsOnly ? { drafts_only: true } : {}), + ...(options?.enabledOnly ? { enabled_only: true } : {}), }, }) return response.data } + async createMetricDraft(data: { + name: string + description?: string + metric_type: string + trigger: string + metric_origin?: string + studio_notes?: string + parent_metric_id?: string + selection_mode?: string + custom_data_type?: string + custom_config?: Record + capture_rationale?: boolean + compare_transcripts?: boolean + scope?: 'workspace' | 'organization' + supported_surfaces?: string[] + enabled_surfaces?: string[] + tags?: string[] + enabled?: boolean + }): Promise { + const response = await this.client.post('/api/v1/metrics/drafts', data) + return response.data + } + + async createMetricDraftWithChildren(data: { + name: string + description?: string | null + selection_mode: 'single_choice' | 'multi_label' + allow_discovery?: boolean + capture_rationale?: boolean + supported_surfaces: string[] + enabled_surfaces: string[] + children: Array<{ + name: string + description?: string | null + example?: string | null + capture_rationale?: boolean + enabled?: boolean + }> + scope?: 'workspace' | 'organization' + studio_notes?: string + }): Promise { + const response = await this.client.post('/api/v1/metrics/drafts/with-children', data) + return response.data + } + + async promoteMetric(metricId: string): Promise<{ metric: any; promoted_at: string }> { + const response = await this.client.post(`/api/v1/metrics/${metricId}/promote`) + return response.data + } + + async createMetricStudioRun(data: { + name?: string + metric_ids: string[] + sources: Array<{ + source_kind: 'call_import_row' | 'call_recording' | 'evaluator_result' + source_ref: string + display_label?: string + }> + transcript_source?: 'production' | 'diarised' + llm_provider?: string + llm_model?: string + llm_credential_id?: string + llm_config?: LLMGenerationConfig | null + metric_llm_overrides?: Record + }): Promise { + const response = await this.client.post('/api/v1/metric-studio/runs', data) + return response.data + } + + async listMetricStudioRuns(skip = 0, limit = 50): Promise<{ items: any[]; total: number }> { + const response = await this.client.get('/api/v1/metric-studio/runs', { + params: { skip, limit }, + }) + return response.data + } + + async getMetricStudioRun(runId: string): Promise { + const response = await this.client.get(`/api/v1/metric-studio/runs/${runId}`) + return response.data + } + + async listMetricStudioRunResults( + runId: string, + skip = 0, + limit = 100, + ): Promise<{ items: any[]; total: number }> { + const response = await this.client.get(`/api/v1/metric-studio/runs/${runId}/results`, { + params: { skip, limit }, + }) + return response.data + } + + async retryMetricStudioRun( + runId: string, + resultIds?: string[], + ): Promise { + const response = await this.client.post(`/api/v1/metric-studio/runs/${runId}/retry`, { + result_ids: resultIds, + }) + return response.data + } + + async deleteMetricStudioRun(runId: string): Promise { + await this.client.delete(`/api/v1/metric-studio/runs/${runId}`) + } + // Evaluator Results endpoints async listEvaluatorResults( paramsOrEvaluatorId?: ListEvaluatorResultsParams | string, @@ -4827,6 +5248,15 @@ export interface JudgeOptimizeResponse { test_sample_count: number } +export const AUDIO_UPLOAD_CHUNK_SIZE = 25 + +export interface CallImportAudioUploadProgress { + chunkIndex: number + totalChunks: number + uploadedFiles: number + totalFiles: number +} + // Factory function to create ApiClient instance // This ensures TypeScript correctly infers the type as ApiClient, not AxiosInstance function createApiClient(): ApiClient { diff --git a/frontend/src/lib/authSession.ts b/frontend/src/lib/authSession.ts new file mode 100644 index 00000000..4a1e673c --- /dev/null +++ b/frontend/src/lib/authSession.ts @@ -0,0 +1,45 @@ +export const AUTH_REDIRECT_MESSAGE_KEY = 'authRedirectMessage' + +export function getApiErrorDetail(error: unknown): string | undefined { + const detail = (error as { response?: { data?: { detail?: unknown } } })?.response?.data + ?.detail + return typeof detail === 'string' ? detail : undefined +} + +export function isOrganizationAccessDenied(detail?: string): boolean { + if (!detail) return false + const normalized = detail.toLowerCase() + return ( + normalized.includes('organization disabled') || + normalized.includes('not a member of any active organization') + ) +} + +export function clearAuthSession(): void { + localStorage.removeItem('apiKey') + localStorage.removeItem('accessToken') + localStorage.removeItem('refreshToken') + localStorage.removeItem('authUser') + localStorage.removeItem('activeWorkspaceId') +} + +export function redirectToLoginWithMessage(message: string): void { + sessionStorage.setItem(AUTH_REDIRECT_MESSAGE_KEY, message) + clearAuthSession() + window.location.href = '/login' +} + +export function consumeAuthRedirectMessage(): string | null { + const message = sessionStorage.getItem(AUTH_REDIRECT_MESSAGE_KEY) + if (message) { + sessionStorage.removeItem(AUTH_REDIRECT_MESSAGE_KEY) + } + return message +} + +export function organizationAccessDeniedMessage(detail?: string): string { + if (detail?.toLowerCase().includes('organization disabled')) { + return "Your organization's access has been disabled. Contact your administrator." + } + return detail || "Your organization's access has been disabled. Contact your administrator." +} diff --git a/frontend/src/pages/auth/Login.tsx b/frontend/src/pages/auth/Login.tsx index 9edf0838..f395437f 100644 --- a/frontend/src/pages/auth/Login.tsx +++ b/frontend/src/pages/auth/Login.tsx @@ -5,9 +5,10 @@ import { apiClient, isLoginOrgSelectionResponse } from '../../lib/api' import type { AuthConfigResponse, AuthProviderConfig, LoginOrgOption } from '../../lib/api' import { buildAuthorizeUrl } from '../../lib/oidc' import { PASSWORD_POLICY_HINT, validatePasswordPolicy } from '../../lib/passwordPolicy' +import { consumeAuthRedirectMessage } from '../../lib/authSession' import { AlertCircle, Building2, Eye, EyeOff, Loader2 } from 'lucide-react' import Logo from '../../components/Logo' -import { Card, CardBody, Button, Divider, Chip, Tabs, Tab } from '@heroui/react' +import { Card, CardBody, Button, Divider, Tabs, Tab } from '@heroui/react' /** * Provider-aware sign-in screen. @@ -24,6 +25,18 @@ import { Card, CardBody, Button, Divider, Chip, Tabs, Tab } from '@heroui/react' type Mode = 'password' | 'signup' | 'sso' type LoginStep = 'credentials' | 'org-select' +function LoginFormError({ message }: { message: string }) { + return ( +
+ +

{message}

+
+ ) +} + export default function Login() { const navigate = useNavigate() const { setSession } = useAuthStore() @@ -38,6 +51,7 @@ export default function Login() { const [orgName, setOrgName] = useState('') const [firstName, setFirstName] = useState('') const [lastName, setLastName] = useState('') + const [referenceCode, setReferenceCode] = useState('') const [error, setError] = useState('') const [isLoading, setIsLoading] = useState(false) const [loginStep, setLoginStep] = useState('credentials') @@ -45,6 +59,13 @@ export default function Login() { const [selectingOrgId, setSelectingOrgId] = useState(null) const [showPassword, setShowPassword] = useState(false) + useEffect(() => { + const redirectMessage = consumeAuthRedirectMessage() + if (redirectMessage) { + setError(redirectMessage) + } + }, []) + useEffect(() => { let active = true apiClient @@ -129,6 +150,7 @@ export default function Login() { organization_name: orgName || undefined, first_name: firstName || undefined, last_name: lastName || undefined, + reference_code: authConfig?.gated_signup ? referenceCode : undefined, }) setSession(res.access_token, res.user, res.refresh_token) navigate('/') @@ -253,14 +275,10 @@ export default function Login() { {showPassword ? : }
- {error && ( - } className="w-full max-w-full h-auto py-2"> - {error} - - )} + {error && } )} @@ -292,11 +310,6 @@ export default function Login() { ) })}
- {error && ( - } className="w-full max-w-full h-auto py-2"> - {error} - - )} + {error && } )} @@ -341,14 +355,20 @@ export default function Login() { setOrgName(e.target.value)} className="w-full px-4 py-3 bg-gray-50 border-2 border-gray-200 rounded-xl focus:outline-none focus:border-[#ca8a04] focus:bg-white" /> - {error && ( - } className="w-full max-w-full h-auto py-2"> - {error} - + {authConfig?.gated_signup && ( + setReferenceCode(e.target.value)} + required + className="w-full px-4 py-3 bg-gray-50 border-2 border-gray-200 rounded-xl focus:outline-none focus:border-[#ca8a04] focus:bg-white" + /> )} + {error && }

By signing up you become the admin of a new organization. You can invite teammates later.

@@ -360,11 +380,7 @@ export default function Login() { - {error && ( - } className="w-full max-w-full h-auto py-2"> - {error} - - )} + {error && }

You'll be redirected to your identity provider to complete sign-in.

diff --git a/frontend/src/pages/callImports/CallImportDetail.tsx b/frontend/src/pages/callImports/CallImportDetail.tsx index 4b262eaf..45b8bcfa 100644 --- a/frontend/src/pages/callImports/CallImportDetail.tsx +++ b/frontend/src/pages/callImports/CallImportDetail.tsx @@ -34,6 +34,7 @@ import { Search, Square, Trash2, + Upload, Volume2, X, XCircle, @@ -72,6 +73,7 @@ import { } from '../../lib/llmModelOptions' import CallImportProgressBar from './components/CallImportProgressBar' import RetryFailedImportModal from './components/RetryFailedImportModal' +import UploadAudioModal from './components/UploadAudioModal' import InsightsMetricCard, { INSIGHTS_PALETTE, } from './components/InsightsMetricCard' @@ -131,6 +133,13 @@ function isNonRetryableError(message: string | null | undefined): boolean { return /(401|403|404|forbidden|unauthor|not found|exceeds|too large|invalid content)/i.test(message) } +function isImportCredentialError(message: string | null | undefined): boolean { + if (!message) return false + return /(rejected credentials|telephony credentials rejected|auth failed|credentials could not be verified)/i.test( + message, + ) +} + // --- Insights tab helpers --------------------------------------------- // // KPI cards in the Insights header use a tinted gradient background + @@ -408,15 +417,12 @@ export default function CallImportDetail() { const audioRef = useRef(null) const [showDeleteImport, setShowDeleteImport] = useState(false) + const [showAppendAudioModal, setShowAppendAudioModal] = useState(false) const [showRetryFailedImportConfirm, setShowRetryFailedImportConfirm] = useState(false) const [retryFailedImportError, setRetryFailedImportError] = useState( null, ) - const [showForceFailDiarisationConfirm, setShowForceFailDiarisationConfirm] = - useState(false) - const [forceFailDiarisationError, setForceFailDiarisationError] = - useState(null) const [pendingDeleteRow, setPendingDeleteRow] = useState(null) const [deleteError, setDeleteError] = useState(null) const [showRunEval, setShowRunEval] = useState(false) @@ -609,6 +615,7 @@ export default function CallImportDetail() { }) const [editingMeta, setEditingMeta] = useState(false) + const [draftImportName, setDraftImportName] = useState('') const [draftDataset, setDraftDataset] = useState('') const [draftTagIds, setDraftTagIds] = useState([]) @@ -624,8 +631,11 @@ export default function CallImportDetail() { }) const updateMetaMutation = useMutation({ - mutationFn: (payload: { dataset?: string | null; tag_ids?: string[] }) => - apiClient.updateCallImport(id!, payload), + mutationFn: (payload: { + original_filename?: string | null + dataset?: string | null + tag_ids?: string[] + }) => apiClient.updateCallImport(id!, payload), onSuccess: () => { queryClient.invalidateQueries({ queryKey: ['call-import', activeWorkspaceId, id] }) queryClient.invalidateQueries({ queryKey: ['call-imports'] }) @@ -681,27 +691,6 @@ export default function CallImportDetail() { }, }) - const forceFailDiarisationMutation = useMutation({ - mutationFn: () => apiClient.cancelCallImportDiarisation(id!, null), - onSuccess: (result) => { - queryClient.invalidateQueries({ queryKey: ['call-import', activeWorkspaceId, id] }) - setShowForceFailDiarisationConfirm(false) - const parts = [`Force-failed ${result.cancelled} diarisation row${result.cancelled === 1 ? '' : 's'}`] - if (result.skipped > 0) { - parts.push(`skipped ${result.skipped}`) - } - setForceFailDiarisationError(null) - setBulkActionResult(parts.join(' · ')) - }, - onError: (err: any) => { - setForceFailDiarisationError( - err?.response?.data?.detail || - err?.message || - 'Failed to force-fail in-flight diarisation rows.', - ) - }, - }) - const deleteRowMutation = useMutation({ mutationFn: ({ importId, rowId }: { importId: string; rowId: string }) => apiClient.deleteCallImportRow(importId, rowId), @@ -1432,32 +1421,6 @@ export default function CallImportDetail() { }) } - // Toggle one child while keeping the parent reference in sync: if a - // child is the only remaining selected member of its group we drop - // the parent id; if every child is selected we add the parent id so - // the run modal can show a single chip for the whole group. - const toggleChildMetric = (parent: any, childId: string) => { - const childIds: string[] = Array.isArray(parent.children) - ? parent.children.filter((c: any) => c.enabled).map((c: any) => c.id) - : [] - setSelectedMetricIds((prev) => { - const set = new Set(prev) - if (set.has(childId)) { - set.delete(childId) - } else { - set.add(childId) - } - const allSelected = - childIds.length > 0 && childIds.every((cid) => set.has(cid)) - if (allSelected) { - set.add(parent.id) - } else { - set.delete(parent.id) - } - return Array.from(set) - }) - } - const openTranscribeModal = (rows: CallImportRow[]) => { setTranscribeTargetRows(rows) setTranscribeError(null) @@ -1528,6 +1491,14 @@ export default function CallImportDetail() { } const rows = data.rows ?? [] + const pendingCredentialErrorCount = rows.filter( + (row) => + row.status === 'pending' && + !!row.error_message && + isImportCredentialError(row.error_message), + ).length + const showCredentialImportWarning = + data.status === 'processing' && pendingCredentialErrorCount > 0 // The staged-flow batch isn't ready to render rows / evaluations // until the user finishes the MAP + IMPORT steps. We swap out the @@ -1536,6 +1507,9 @@ export default function CallImportDetail() { const needsMapping = data.status === 'uploaded' || data.status === 'mapped' const showWorkflowTabs = !needsMapping const isDeleting = data.status === 'deleting' + const isManualAudioImport = (data.source_format || '').toLowerCase() === 'audio' + const canAppendManualAudio = + isManualAudioImport && data.status === 'completed' && !isDeleting const canRunEvaluation = data.status === 'mapped' || showWorkflowTabs // Selection state — ``selectedRowIds`` can now span pages (we no @@ -1614,6 +1588,17 @@ export default function CallImportDetail() { Run Evaluation )} + {canAppendManualAudio && ( + + )} - )} ) : (
+
+ + setDraftImportName(e.target.value)} + placeholder="Leave blank to clear" + className="w-full max-w-sm px-3 py-2 text-sm border border-gray-300 rounded-lg focus:ring-2 focus:ring-primary-500 focus:border-transparent" + /> +
)} + {showCredentialImportWarning && ( +
+
+ +
+

+ {pendingCredentialErrorCount} row + {pendingCredentialErrorCount === 1 ? '' : 's'} failed to fetch + recordings due to telephony credential errors. Update credentials in + Integrations, then use Retry failed to try again. +

+
+
+
+ )} + {(data.source_row_skips?.length ?? 0) > 0 && data.total_rows > 0 && (
@@ -2593,6 +2597,15 @@ export default function CallImportDetail() {
)} + {row.status === 'pending' && row.error_message && ( +
+ +
+ Retry pending: {row.error_message} +
+
+ )} + {/* Surface diarisation-specific failures even when the row's recording download itself succeeded. Previously these only showed inside the expanded @@ -4066,6 +4079,22 @@ export default function CallImportDetail() { (() => { const enabledMetrics = metrics.filter((m: any) => m.enabled) const disabledMetrics = metrics.filter((m: any) => !m.enabled) + const allSelectableIds = enabledMetrics.flatMap((metric: any) => { + const children: any[] = Array.isArray(metric.children) + ? metric.children.filter((c: any) => c.enabled) + : [] + const isParent = + !!metric.selection_mode && children.length > 0 + if (isParent) { + return [metric.id, ...children.map((c: any) => c.id)] + } + return [metric.id] + }) + const allMetricsSelected = + allSelectableIds.length > 0 && + allSelectableIds.every((metricId) => + selectedMetricIds.includes(metricId), + ) // Group the user's selection into "override targets" — one // entry per parent categorization metric (regardless of how // many of its labels are picked) and one entry per standalone @@ -4237,17 +4266,36 @@ export default function CallImportDetail() { *

- {selectedMetricIds.length === 0 ? ( - - - Pick at least one - - ) : ( - - {selectedMetricIds.length} selected - - )} +
+ {enabledMetrics.length > 0 && ( + + )} + {selectedMetricIds.length === 0 ? ( + + + Pick at least one + + ) : ( + + {selectedMetricIds.length} selected + + )} +
+
{enabledMetrics.map((metric: any) => { const children: any[] = Array.isArray(metric.children) ? metric.children.filter((c: any) => c.enabled) @@ -4258,21 +4306,17 @@ export default function CallImportDetail() { return ( ) @@ -4281,75 +4325,42 @@ export default function CallImportDetail() { const selectedChildCount = childIds.filter((cid) => selectedMetricIds.includes(cid), ).length - const allSelected = + const allChildrenSelected = selectedChildCount === childIds.length && childIds.length > 0 - const someSelected = - selectedChildCount > 0 && !allSelected + const someChildrenSelected = + selectedChildCount > 0 && !allChildrenSelected return ( -
- -
- {children.map((child: any) => ( - - ))} -
-
+ {someChildrenSelected ? ( + + {selectedChildCount}/{childIds.length} labels + + ) : null} + + ) })} +
{disabledMetrics.length > 0 && ( @@ -4550,9 +4561,9 @@ export default function CallImportDetail() { Transcript for evaluation

- Choose whether to score the CSV production - transcript or diarize first, then score the - diarized output. + {isManualAudioImport + ? 'Recordings are already uploaded to this batch. Diarization uses the stored audio files — no CSV recording URL column is needed.' + : 'Choose whether to score the CSV production transcript or diarize first, then score the diarized output.'}

{evalTranscriptSource === 'production' ? (

- Skips diarization and scores the transcript - imported from your CSV. Recordings are still - fetched from Exotel/Plivo when a recording URL - is mapped (needed for audio metrics). + Skips diarization and recording download, and + scores the transcript imported from your CSV + directly. Audio metrics require a recording + already stored on the row.

) : null}
@@ -5091,33 +5102,18 @@ export default function CallImportDetail() { }} /> - { - const lines = [ - 'Any row currently pending/running in diarisation will be marked failed immediately.', - 'You can then re-diarise those failed rows from the same page.', - forceFailDiarisationError ? `Error: ${forceFailDiarisationError}` : '', - ] - return lines.filter(Boolean).join('\n\n') - })()} - confirmLabel={`Force-fail ${diarisationInFlightCount} row${ - diarisationInFlightCount === 1 ? '' : 's' - }`} - cancelLabel="Cancel" - variant="danger" - isLoading={forceFailDiarisationMutation.isPending} - onConfirm={() => { - if (forceFailDiarisationMutation.isPending) return - forceFailDiarisationMutation.mutate() - }} - onCancel={() => { - if (forceFailDiarisationMutation.isPending) return - setShowForceFailDiarisationConfirm(false) - setForceFailDiarisationError(null) + setShowAppendAudioModal(false)} + appendToImportId={id} + appendToImportName={data.original_filename || undefined} + onAppendSuccess={(response) => { + showToast( + response.message || + `Added recordings (${response.total_rows} total in batch).`, + 'success', + ) + refetch() }} /> diff --git a/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx b/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx index 35c4f703..55b70c81 100644 --- a/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx +++ b/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx @@ -433,7 +433,6 @@ export default function CallImportEvaluationDetail() { const [pendingDeleteRow, setPendingDeleteRow] = useState(null) const [deleteEvalOpen, setDeleteEvalOpen] = useState(false) - const [forceFailPendingOpen, setForceFailPendingOpen] = useState(false) const [downloadMenuOpen, setDownloadMenuOpen] = useState(false) const downloadMenuRef = useRef(null) const [pdfReportMenuOpen, setPdfReportMenuOpen] = useState(false) @@ -1068,15 +1067,16 @@ export default function CallImportEvaluationDetail() { // persists them onto the run so the next retry defaults to them. const retryAllFailedMutation = useMutation({ mutationFn: () => { - // Only forward LLM overrides when the user actually picked - // BOTH provider and model — backend 400s on half-configured - // input, and "did the user touch the picker" is fuzzy anyway - // because it gets seeded from the run's saved values. + // Forward LLM overrides only when the picker resolves to a + // complete selection (catalog model, gateway credential, etc.). const llmChanged = retryLLM.provider !== (evaluation?.llm_provider ?? null) || retryLLM.model !== (evaluation?.llm_model ?? null) || (retryLLM.credential_id ?? null) !== - (evaluation?.llm_credential_id ?? null) + (evaluation?.llm_credential_id ?? null) || + JSON.stringify(retryLLM.llm_config ?? null) !== + JSON.stringify(evaluation?.llm_config ?? null) + const llmComplete = isLLMSelectionComplete(retryLLM, aiProviders) const sttChanged = retrySTT.provider !== (evaluation?.stt_provider ?? null) || retrySTT.model !== (evaluation?.stt_model ?? null) || @@ -1089,6 +1089,10 @@ export default function CallImportEvaluationDetail() { (evaluation?.diarisation_llm_model ?? null) || (retryDiariserLLM.credential_id ?? null) !== (evaluation?.diarisation_llm_credential_id ?? null) + const diariserComplete = isLLMSelectionComplete( + retryDiariserLLM, + aiProviders, + ) const transcribeModeChanged = retryTranscribeMode !== (evaluation?.transcribe_mode ?? 'stt_llm') const diarisationPromptChanged = @@ -1107,17 +1111,16 @@ export default function CallImportEvaluationDetail() { return apiClient.retryCallImportEvaluation(id!, evalId!, { llmProvider: - llmChanged && retryLLM.provider && retryLLM.model - ? retryLLM.provider - : undefined, + llmChanged && llmComplete ? retryLLM.provider ?? undefined : undefined, llmModel: - llmChanged && retryLLM.provider && retryLLM.model - ? retryLLM.model + llmChanged && llmComplete + ? resolveLLMModelForSubmit(retryLLM, aiProviders) ?? + retryLLM.model ?? + undefined : undefined, llmCredentialId: - llmChanged && retryLLM.provider && retryLLM.model - ? retryLLM.credential_id ?? null - : undefined, + llmChanged && llmComplete ? retryLLM.credential_id ?? null : undefined, + llmConfig: llmChanged ? retryLLM.llm_config ?? null : undefined, sttProvider: sttChanged && retrySTT.provider && retrySTT.model ? retrySTT.provider @@ -1134,16 +1137,17 @@ export default function CallImportEvaluationDetail() { ? retryTranscribeMode : undefined, diarizationLlmProvider: - diariserChanged && retryDiariserLLM.provider && retryDiariserLLM.model - ? retryDiariserLLM.provider + diariserChanged && diariserComplete + ? retryDiariserLLM.provider ?? undefined : undefined, diarizationLlmModel: - diariserChanged && retryDiariserLLM.provider && retryDiariserLLM.model + diariserChanged && diariserComplete ? resolveLLMModelForSubmit(retryDiariserLLM, aiProviders) ?? - retryDiariserLLM.model + retryDiariserLLM.model ?? + undefined : undefined, diarizationLlmCredentialId: - diariserChanged && retryDiariserLLM.provider && retryDiariserLLM.model + diariserChanged && diariserComplete ? retryDiariserLLM.credential_id ?? null : undefined, diarizationPrompt: diarisationPromptChanged @@ -1252,21 +1256,20 @@ export default function CallImportEvaluationDetail() { (evaluation?.llm_credential_id ?? null) || JSON.stringify(rerunLLM.llm_config ?? null) !== JSON.stringify(evaluation?.llm_config ?? null) + const llmComplete = isLLMSelectionComplete(rerunLLM, aiProviders) return apiClient.retryCallImportEvaluation(id!, evalId!, { metricIds, includeCompleted: true, llmProvider: - llmChanged && rerunLLM.provider && rerunLLM.model - ? rerunLLM.provider - : undefined, + llmChanged && llmComplete ? rerunLLM.provider ?? undefined : undefined, llmModel: - llmChanged && rerunLLM.model && rerunLLM.provider - ? rerunLLM.model + llmChanged && llmComplete + ? resolveLLMModelForSubmit(rerunLLM, aiProviders) ?? + rerunLLM.model ?? + undefined : undefined, llmCredentialId: - llmChanged && rerunLLM.provider && rerunLLM.model - ? rerunLLM.credential_id ?? null - : undefined, + llmChanged && llmComplete ? rerunLLM.credential_id ?? null : undefined, llmConfig: llmChanged ? rerunLLM.llm_config ?? null : undefined, }) }, @@ -1360,29 +1363,6 @@ export default function CallImportEvaluationDetail() { }, }) - const forceFailPendingMutation = useMutation({ - mutationFn: () => apiClient.forceFailCallImportEvaluationPending(id!, evalId!), - onMutate: () => { - setCancelError(null) - }, - onSuccess: (data: CallImportEvaluationBulkActionResponse) => { - if (data.target_count > 0) { - setEvaluationBulkOperationOptimistic('force_fail_pending') - } - setForceFailPendingOpen(false) - invalidateEvaluationQueries() - }, - onError: (err: any) => { - setCancelError( - err?.response?.status === 409 - ? err?.response?.data?.detail || bulkOperationConflictMessage - : err?.response?.data?.detail || - err?.message || - 'Failed to force-fail pending rows.', - ) - }, - }) - // Row-level cancel: same shape as the run-level mutation but // scoped to a single row so the operator can stop one wedged row // without aborting siblings that are progressing fine. @@ -2357,12 +2337,6 @@ export default function CallImportEvaluationDetail() { : `Evaluation ${evaluation.id.slice(0, 8)}` const bulkOperation = evaluation.bulk_operation ?? null const bulkOperationActive = bulkOperation !== null - const pendingRowCount = Math.max( - 0, - (evaluation.total_rows ?? 0) - - (evaluation.completed_rows ?? 0) - - (evaluation.failed_rows ?? 0), - ) const getMetricLlmLabel = (metricId: string): string => { const override = evaluation.metric_llm_overrides?.[metricId] const overrideProvider = override?.provider?.trim() @@ -2418,36 +2392,6 @@ export default function CallImportEvaluationDetail() { Abort run )} - {pendingRowCount > 0 && ( - - )} {evaluation.failed_rows > 0 && ( - - - - - - - - - -
- - -
- - {/* - High-level dataset segregation lives at the top of the page so users can - scope all filtering/searching that follows to a specific dataset. We - intentionally render this above the main card to make it visually - distinct from the in-card status/tag filters. - */} -
- - - {datasetFilter && ( - - Showing imports tagged with dataset “{datasetFilter}”. - - )} -
- -
-
-
-
- - -
- {allTags.length > 0 && ( -
- Tags: - {allTags.map((tag: CallImportTag) => { - const active = tagFilter.includes(tag.id) - return ( - - ) - })} - {tagFilter.length > 0 && ( - - )} -
- )} -
-

- {total} {activeTab === 'audio' ? 'manual upload' : 'dataset import'} - {total === 1 ? '' : 's'} -

-
- - {isLoading ? ( -
- -

Loading imports...

-
- ) : items.length === 0 ? ( -
- -

- {statusFilter - ? 'No imports match this filter.' - : activeTab === 'audio' - ? 'No manual audio uploads yet.' - : 'No dataset uploads yet.'} -

- {!statusFilter && ( - - )} -
- ) : ( -
- - - - - - - - - - - - - {items.map((item: CallImport) => { - const isDeleting = item.status === 'deleting' - return ( - { - if (isDeleting) return - navigate(`/call-imports/${item.id}`) - }} - > - - - - - - - - ) - })} - -
- Filename - - Provider - - Dataset / Tags - - Progress - - Status - - Actions -
-
- {item.original_filename || '(unnamed)'} -
-
- {item.id.slice(0, 8)} -
- 		- {item.source_format === 'audio' ? ( - - - Manual upload - - ) : item.provider || ( - - — - - )} - -
- {item.dataset ? ( - - {item.dataset} - - ) : ( - - no dataset - - )} - {item.tags.length > 0 && ( -
- {item.tags.map((tag) => ( - - {tag.name} - - ))} -
- )} -
- 		- - - - e.stopPropagation()} - > -
- - View - - -
-
- - {totalPages > 1 && ( -
-

- Page {page} of {totalPages} -

-
- - -
-
- )} -
- )} -
- - setShowUpload(false)} /> - setShowAudioUpload(false)} - /> - - { - if (!pendingDelete) return '' - const name = pendingDelete.original_filename || '(unnamed)' - const total = pendingDelete.total_rows - const completed = pendingDelete.completed_rows - const inFlight = - pendingDelete.status === 'pending' || - pendingDelete.status === 'processing' || - pendingDelete.status === 'deleting' - const lines = [ - `“${name}” will be permanently deleted, along with all ${total} row record${total === 1 ? '' : 's'} and ${completed} stored recording${completed === 1 ? '' : 's'} in S3.`, - inFlight - ? 'This batch is still processing — pending tasks will be revoked before deletion.' - : '', - 'This cannot be undone.', - deleteError ? `Error: ${deleteError}` : '', - ] - return lines.filter(Boolean).join('\n\n') - })()} - confirmLabel="Delete" - cancelLabel="Cancel" - variant="danger" - isLoading={deleteMutation.isPending} - onConfirm={() => { - if (pendingDelete) deleteMutation.mutate(pendingDelete.id) - }} - onCancel={() => { - if (deleteMutation.isPending) return - setPendingDelete(null) - setDeleteError(null) - }} - /> - - ) -} +import { useMemo, useState } from 'react' +import { Link, useNavigate } from 'react-router-dom' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { + ChevronLeft, + ChevronRight, + FileAudio, + FileSpreadsheet, + Layers, + Phone, + RefreshCw, + Trash2, + Upload, +} from 'lucide-react' +import { Tag as TagIcon } from 'lucide-react' +import { apiClient } from '../../lib/api' +import { getApiErrorMessage } from '../../lib/apiErrors' +import { useToast } from '../../hooks/useToast' +import { useWorkspaceStore } from '../../store/workspaceStore' +import type { CallImport, CallImportStatus, CallImportTag } from '../../types/api' +import Button from '../../components/Button' +import ConfirmModal from '../../components/ConfirmModal' +import StatusBadge from '../../components/shared/StatusBadge' +import CallImportProgressBar from './components/CallImportProgressBar' +import UploadAudioModal from './components/UploadAudioModal' +import UploadCsvModal from './components/UploadCsvModal' + +const PAGE_SIZE = 20 + +const STATUS_OPTIONS: Array<{ label: string; value: '' | CallImportStatus }> = [ + { label: 'All statuses', value: '' }, + { label: 'Uploaded', value: 'uploaded' }, + { label: 'Mapped', value: 'mapped' }, + { label: 'Pending', value: 'pending' }, + { label: 'Processing', value: 'processing' }, + { label: 'Completed', value: 'completed' }, + { label: 'Partial', value: 'partial' }, + { label: 'Failed', value: 'failed' }, + { label: 'Deleting', value: 'deleting' }, +] + +type UploadTab = 'datasets' | 'audio' + +export default function CallImports() { + const navigate = useNavigate() + const queryClient = useQueryClient() + const { showToast, ToastContainer } = useToast() + // Active workspace is part of every workspace-scoped queryKey so a + // workspace switch produces a clean cache miss instead of leaking + // rows from the previously-active workspace. + const activeWorkspaceId = useWorkspaceStore((s) => s.activeWorkspaceId) + const [page, setPage] = useState(1) + const [statusFilter, setStatusFilter] = useState<'' | CallImportStatus>('') + const [datasetFilter, setDatasetFilter] = useState('') + const [tagFilter, setTagFilter] = useState([]) + const [activeTab, setActiveTab] = useState('datasets') + const [showUpload, setShowUpload] = useState(false) + const [showAudioUpload, setShowAudioUpload] = useState(false) + const [pendingDelete, setPendingDelete] = useState(null) + const [deleteError, setDeleteError] = useState(null) + + const { data: datasets = [] } = useQuery({ + queryKey: ['call-import-datasets', activeWorkspaceId], + queryFn: () => apiClient.listCallImportDatasets(), + }) + + const { data: allTags = [] } = useQuery({ + queryKey: ['call-import-tags', activeWorkspaceId], + queryFn: () => apiClient.listCallImportTags(), + }) + + const deleteMutation = useMutation({ + mutationFn: (id: string) => apiClient.deleteCallImport(id), + onSuccess: (result) => { + queryClient.invalidateQueries({ queryKey: ['call-imports'] }) + setPendingDelete(null) + setDeleteError(null) + if (result.status === 'accepted') { + showToast( + 'Deletion started — large imports may take a minute.', + 'success', + ) + } + }, + onError: (err: unknown) => { + const message = getApiErrorMessage(err, 'Failed to delete import.') + setDeleteError(message) + showToast(message, 'error') + }, + }) + + const queryParams = useMemo( + () => ({ + page, + page_size: PAGE_SIZE, + ...(statusFilter ? { status: statusFilter } : {}), + ...(datasetFilter ? { dataset: datasetFilter } : {}), + ...(tagFilter.length > 0 ? { tag_id: tagFilter } : {}), + source_format: activeTab === 'audio' ? 'audio' : '__non_audio__', + }), + [page, statusFilter, datasetFilter, tagFilter, activeTab], + ) + + const { data, isLoading, isFetching, refetch } = useQuery({ + queryKey: ['call-imports', activeWorkspaceId, queryParams], + queryFn: () => apiClient.listCallImports(queryParams), + refetchInterval: (query) => { + const items = query.state.data?.items ?? [] + const hasActive = items.some( + (i: CallImport) => i.status === 'pending' || i.status === 'processing', + ) + const hasDeleting = items.some( + (i: CallImport) => i.status === 'deleting', + ) + if (hasDeleting) return 3000 + return hasActive ? 5000 : false + }, + }) + + const items = data?.items ?? [] + const total = data?.total ?? 0 + const totalPages = Math.max(1, Math.ceil(total / PAGE_SIZE)) + + return ( +
+ +
+
+

Call Imports

+

+ Upload datasets from CSV / Excel, or add manual call recordings + directly and use the same diarisation and evaluation tools. +

+
+
+ + + + + + + + +
+
+ +
+ + +
+ + {/* + High-level dataset segregation lives at the top of the page so users can + scope all filtering/searching that follows to a specific dataset. We + intentionally render this above the main card to make it visually + distinct from the in-card status/tag filters. + */} +
+ + + {datasetFilter && ( + + Showing imports tagged with dataset “{datasetFilter}”. + + )} +
+ +
+
+
+
+ + +
+ {allTags.length > 0 && ( +
+ Tags: + {allTags.map((tag: CallImportTag) => { + const active = tagFilter.includes(tag.id) + return ( + + ) + })} + {tagFilter.length > 0 && ( + + )} +
+ )} +
+

+ {total} {activeTab === 'audio' ? 'manual upload' : 'dataset import'} + {total === 1 ? '' : 's'} +

+
+ + {isLoading ? ( +
+ +

Loading imports...

+
+ ) : items.length === 0 ? ( +
+ +

+ {statusFilter + ? 'No imports match this filter.' + : activeTab === 'audio' + ? 'No manual audio uploads yet.' + : 'No dataset uploads yet.'} +

+ {!statusFilter && ( + + )} +
+ ) : ( +
+ + + + + + + + + + + + + {items.map((item: CallImport) => { + const isDeleting = item.status === 'deleting' + return ( + { + if (isDeleting) return + navigate(`/call-imports/${item.id}`) + }} + > + + + + + + + + ) + })} + +
+ Filename + + Provider + + Dataset / Tags + + Progress + + Status + + Actions +
+
+ {item.original_filename || '(unnamed)'} +
+
+ {item.id.slice(0, 8)} +
+ 		+ {item.source_format === 'audio' ? ( + + + Manual upload + + ) : item.provider || ( + + — + + )} + +
+ {item.dataset ? ( + + {item.dataset} + + ) : ( + + no dataset + + )} + {item.tags.length > 0 && ( +
+ {item.tags.map((tag) => ( + + {tag.name} + + ))} +
+ )} +
+ 		+ + + + e.stopPropagation()} + > +
+ + View + + +
+
+ + {totalPages > 1 && ( +
+

+ Page {page} of {totalPages} +

+
+ + +
+
+ )} +
+ )} +
+ + setShowUpload(false)} /> + setShowAudioUpload(false)} + /> + + { + if (!pendingDelete) return '' + const name = pendingDelete.original_filename || '(unnamed)' + const total = pendingDelete.total_rows + const completed = pendingDelete.completed_rows + const inFlight = + pendingDelete.status === 'pending' || + pendingDelete.status === 'processing' || + pendingDelete.status === 'deleting' + const lines = [ + `“${name}” will be permanently deleted, along with all ${total} row record${total === 1 ? '' : 's'} and ${completed} stored recording${completed === 1 ? '' : 's'} in S3.`, + inFlight + ? 'This batch is still processing — pending tasks will be revoked before deletion.' + : '', + 'This cannot be undone.', + deleteError ? `Error: ${deleteError}` : '', + ] + return lines.filter(Boolean).join('\n\n') + })()} + confirmLabel="Delete" + cancelLabel="Cancel" + variant="danger" + isLoading={deleteMutation.isPending} + onConfirm={() => { + if (pendingDelete) deleteMutation.mutate(pendingDelete.id) + }} + onCancel={() => { + if (deleteMutation.isPending) return + setPendingDelete(null) + setDeleteError(null) + }} + /> +
+ ) +} diff --git a/frontend/src/pages/callImports/Schemas.tsx b/frontend/src/pages/callImports/Schemas.tsx index 0fb167de..281e04f8 100644 --- a/frontend/src/pages/callImports/Schemas.tsx +++ b/frontend/src/pages/callImports/Schemas.tsx @@ -68,12 +68,12 @@ function makeRecordingUrlParameter(): EditableParameter { name: 'recording_url', type: 'recording_url', description: 'URL of the call recording for each imported row.', - is_required: true, + is_required: false, } } function isSystemRequiredParameter(param: EditableParameter): boolean { - return param.type === 'conversation_id' || param.type === 'recording_url' + return param.type === 'conversation_id' } function parametersFromSchema( @@ -109,15 +109,12 @@ function validateParameters(params: EditableParameter[]): string | null { if (convCount !== 1) { return 'Exactly one parameter must be of type "conversation_id".' } - if (recordingCount !== 1) { - return 'Exactly one parameter must be of type "recording_url".' + if (recordingCount > 1) { + return 'At most one parameter can be of type "recording_url".' } if (recordingDateCount > 1) { return 'At most one parameter can be of type "recording_date".' } - if (recordingCount > 1) { - return 'At most one parameter can be of type "recording_url".' - } if (transcriptCount > 1) { return 'At most one parameter can be of type "transcript".' } @@ -160,7 +157,7 @@ function SchemaEditor({ open, schema, onClose, onSaved }: SchemaEditorProps) { setParameters( schema ? parametersFromSchema(schema.parameters) - : [makeConversationIdParameter(), makeRecordingUrlParameter()], + : [makeConversationIdParameter()], ) setErrorMsg(null) } @@ -243,10 +240,7 @@ function SchemaEditor({ open, schema, onClose, onSaved }: SchemaEditorProps) { const removeParameter = (idx: number) => { setParameters((prev) => { const param = prev[idx] - if ( - param?.type === 'conversation_id' || - param?.type === 'recording_url' - ) { + if (param?.type === 'conversation_id') { return prev } return prev.filter((_, i) => i !== idx) @@ -258,14 +252,11 @@ function SchemaEditor({ open, schema, onClose, onSaved }: SchemaEditorProps) { const next = [...prev] const target = direction === 'up' ? idx - 1 : idx + 1 if (target < 0 || target >= next.length) return prev - // Keep the system rows pinned — conversation_id at the top and - // recording_url immediately after it. - const pinnedTypes = new Set(['conversation_id', 'recording_url']) + // Keep conversation_id pinned at the top. if ( - pinnedTypes.has(next[idx].type) || - pinnedTypes.has(next[target].type) || - target <= conversationIdIdx || - (recordingUrlIdx >= 0 && target <= recordingUrlIdx) + next[idx].type === 'conversation_id' || + next[target].type === 'conversation_id' || + target <= conversationIdIdx ) { return prev } @@ -274,6 +265,13 @@ function SchemaEditor({ open, schema, onClose, onSaved }: SchemaEditorProps) { }) } + const addRecordingUrlParameter = () => { + setParameters((prev) => { + if (prev.some((p) => p.type === 'recording_url')) return prev + return [...prev, makeRecordingUrlParameter()] + }) + } + const addParameter = () => { setParameters((prev) => [...prev, makeEmptyParameter()]) } @@ -328,15 +326,27 @@ function SchemaEditor({ open, schema, onClose, onSaved }: SchemaEditorProps) {

Parameters *

- +
+ {recordingUrlIdx < 0 && ( + + )} + +
@@ -424,8 +434,7 @@ function SchemaEditor({ open, schema, onClose, onSaved }: SchemaEditorProps) { updateParameter(idx, { type: nextType, is_required: - nextType === 'conversation_id' || - nextType === 'recording_url' + nextType === 'conversation_id' ? true : p.is_required, }) @@ -471,7 +480,7 @@ function SchemaEditor({ open, schema, onClose, onSaved }: SchemaEditorProps) { />
- {!typeLocked && ( + {!isConversationId && (
+ +
+ + {allTags.length === 0 ? ( +

+ No tags created yet.{' '} + - {tag.name} - - ) - })} + Create tags + + . +

+ ) : ( +
+ {allTags.map((tag: CallImportTag) => { + const active = selectedTagIds.includes(tag.id) + return ( + + ) + })} +
+ )}
- )} -
+ + )} {submitError && (
@@ -332,6 +395,13 @@ export default function UploadAudioModal({ open, onClose }: UploadAudioModalProp
)} + + {uploadProgress && uploadMutation.isPending && ( +
+ Uploading chunk {uploadProgress.chunkIndex} of {uploadProgress.totalChunks} ( + {uploadProgress.uploadedFiles}/{uploadProgress.totalFiles} files) +
+ )}
@@ -349,7 +419,7 @@ export default function UploadAudioModal({ open, onClose }: UploadAudioModalProp isLoading={uploadMutation.isPending} disabled={!canSubmit} > - Upload audio + {isAppendMode ? 'Add recordings' : 'Upload audio'}
diff --git a/frontend/src/pages/callImports/evaluationBulkOperation.ts b/frontend/src/pages/callImports/evaluationBulkOperation.ts index 1270dcc6..1922f62b 100644 --- a/frontend/src/pages/callImports/evaluationBulkOperation.ts +++ b/frontend/src/pages/callImports/evaluationBulkOperation.ts @@ -11,7 +11,7 @@ export function evaluationBulkOperationLabel( case 'abort': return 'Aborting run…' case 'force_fail_pending': - return 'Force-failing pending rows…' + return 'Processing bulk operation…' case 'retry': return 'Retrying failed rows…' } @@ -24,7 +24,7 @@ export function evaluationBulkOperationDescription( case 'abort': return 'Stopping in-flight and queued rows. Other actions stay disabled until this finishes.' case 'force_fail_pending': - return 'Marking pending rows as failed. Other actions stay disabled until this finishes.' + return 'A background operation is in progress. Other actions stay disabled until it finishes.' case 'retry': return 'Re-enqueuing rows for evaluation. Other actions stay disabled until this finishes.' } diff --git a/frontend/src/pages/metrics/MetricsLayout.tsx b/frontend/src/pages/metrics/MetricsLayout.tsx new file mode 100644 index 00000000..e9ed4ead --- /dev/null +++ b/frontend/src/pages/metrics/MetricsLayout.tsx @@ -0,0 +1,18 @@ +import { Outlet } from 'react-router-dom' +import MetricsTabBar from './components/MetricsTabBar' + +export default function MetricsLayout() { + return ( +
+
+

Metrics

+

+ Define quality metrics for your agents, or experiment with draft + metrics and ad-hoc evaluations in Studio. +

+
+ + +
+ ) +} diff --git a/frontend/src/pages/metrics/MetricsManagement.tsx b/frontend/src/pages/metrics/MetricsManagement.tsx index c359679c..a74d42d2 100644 --- a/frontend/src/pages/metrics/MetricsManagement.tsx +++ b/frontend/src/pages/metrics/MetricsManagement.tsx @@ -33,6 +33,10 @@ import { serializeMetricToClipboard, singleFormFromMetricClipboard, } from './metricClipboardUtils' +import { + buildSingleMetricValuePayload, + formatMetricValuePayloadJson, +} from './metricValuePayloadUtils' import { categoryChildrenFromPartial, createCategoryChildrenFromPartial, @@ -173,7 +177,21 @@ function buildMetricPartialContentForTarget( } } -export default function MetricsManagement() { +export interface MetricsManagementProps { + createModalOnly?: boolean + draftMode?: boolean + createModalOpen?: boolean + onCreateModalClose?: () => void + onMetricCreated?: (metric: Metric) => void +} + +export default function MetricsManagement({ + createModalOnly = false, + draftMode = false, + createModalOpen = false, + onCreateModalClose, + onMetricCreated, +}: MetricsManagementProps = {}) { const queryClient = useQueryClient() // Adding the active workspace id to every metrics queryKey so a // workspace switch produces a clean cache miss instead of showing @@ -356,6 +374,30 @@ export default function MetricsManagement() { scope: 'workspace' as 'workspace' | 'organization', }) + const singleMetricValuePayloadJson = useMemo(() => { + return formatMetricValuePayloadJson( + buildSingleMetricValuePayload({ + name: formData.name, + description: formData.description, + metric_type: formData.metric_type, + custom_data_type: formData.custom_data_type, + enum_options_csv: formData.enum_options_csv, + number_min: formData.number_min, + number_max: formData.number_max, + capture_rationale: formData.capture_rationale, + }), + ) + }, [ + formData.name, + formData.description, + formData.metric_type, + formData.custom_data_type, + formData.enum_options_csv, + formData.number_min, + formData.number_max, + formData.capture_rationale, + ]) + // --- Prompt-partial import sub-modal -------------------------------------- // The metric editors carry several "Description (Prompt)" textareas that // all feed into the LLM evaluation prompt: the single-metric form @@ -627,17 +669,38 @@ export default function MetricsManagement() { }) useEffect(() => { - if (metrics.length === 0 && !isLoading) { + if (metrics.length === 0 && !isLoading && !createModalOnly) { seedMutation.mutate() } - }, [metrics.length, isLoading]) + }, [metrics.length, isLoading, createModalOnly]) + + useEffect(() => { + if (createModalOnly && createModalOpen) { + setShowCreateModal(true) + setIsCustomMetricMode(true) + setEditingMetric(null) + setIsEditingCategory(false) + setCreateMode('single') + setPasteMetricError(null) + } else if (createModalOnly && !createModalOpen) { + setShowCreateModal(false) + } + }, [createModalOnly, createModalOpen]) const createMutation = useMutation({ - mutationFn: (data: typeof formData) => apiClient.createMetric(data), - onSuccess: () => { + mutationFn: (data: typeof formData) => + draftMode ? apiClient.createMetricDraft(data as any) : apiClient.createMetric(data), + onSuccess: (metric) => { queryClient.invalidateQueries({ queryKey: ['metrics'] }) - setShowCreateModal(false) - resetForm() + onMetricCreated?.(metric) + showToast( + draftMode ? 'Draft metric created' : 'Metric created', + 'success', + ) + closeModal() + }, + onError: (err: unknown) => { + showToast(getApiErrorMessage(err, 'Failed to create metric'), 'error') }, }) @@ -646,8 +709,11 @@ export default function MetricsManagement() { apiClient.updateMetric(id, data), onSuccess: () => { queryClient.invalidateQueries({ queryKey: ['metrics'] }) - setEditingMetric(null) - resetForm() + showToast('Metric updated', 'success') + closeModal() + }, + onError: (err: unknown) => { + showToast(getApiErrorMessage(err, 'Failed to update metric'), 'error') }, }) @@ -717,11 +783,17 @@ export default function MetricsManagement() { // parent + every child with workspace_id=NULL so the whole // category subtree appears in every workspace. scope?: 'workspace' | 'organization' - }) => apiClient.createMetricWithChildren(payload), - onSuccess: () => { + }) => + draftMode + ? apiClient.createMetricDraftWithChildren(payload) + : apiClient.createMetricWithChildren(payload), + onSuccess: (metric) => { queryClient.invalidateQueries({ queryKey: ['metrics'] }) + onMetricCreated?.(metric) closeModal() - showToast('Category metric created', 'success') + if (!draftMode) { + showToast('Category metric created', 'success') + } }, onError: (err: any) => { const detail = err?.response?.data?.detail || 'Failed to create category metric' @@ -1064,7 +1136,7 @@ export default function MetricsManagement() { const handleCreate = () => { if (!formData.name.trim()) { - alert('Please enter a metric name') + showToast('Please enter a metric name', 'error') return } createMutation.mutate(buildPayload() as any) @@ -1089,6 +1161,12 @@ export default function MetricsManagement() { }) } + const handleCopyValuePayload = () => { + copyTextToClipboard(singleMetricValuePayloadJson, () => { + showToast('Value payload copied to clipboard', 'success') + }) + } + const handlePasteMetric = async () => { setPasteMetricError(null) try { @@ -1213,7 +1291,7 @@ export default function MetricsManagement() { const handleUpdate = () => { if (!editingMetric) return if (!formData.name.trim()) { - alert('Please enter a metric name') + showToast('Please enter a metric name', 'error') return } updateMutation.mutate({ id: editingMetric.id, data: buildPayload() as any }) @@ -1313,6 +1391,9 @@ export default function MetricsManagement() { resetForm() resetAIForm() resetCategoryForm() + if (createModalOnly) { + onCreateModalClose?.() + } } const handleSort = (field: 'type' | 'method') => { @@ -1385,10 +1466,11 @@ export default function MetricsManagement() { return (
+ {!createModalOnly && ( + <>
-

Metrics

-

+

Manage evaluation metrics for your conversations

@@ -1825,9 +1907,12 @@ export default function MetricsManagement() { )}

+ + )} + {/* Create/Edit Modal */} {showCreateModal && ( -
+
{!editingMetric && ( @@ -2254,6 +2343,49 @@ export default function MetricsManagement() {
)} +
+
+ + Value payload (JSON) + + +

+ Example of one row's score object as stored in{' '} + + metric_scores + + . Actual{' '} + + value + {' '} + comes from evaluation. + {editingMetric?.id ? ( + <> + {' '} + Metric ID:{' '} + + {editingMetric.id} + + + ) : null} +

+ 
+                        {singleMetricValuePayloadJson}
+
+
+
+ {/* Call Imports configuration: previously housed a per-metric "Input columns" picker that turned the diff --git a/frontend/src/pages/metrics/MetricsStudio.tsx b/frontend/src/pages/metrics/MetricsStudio.tsx new file mode 100644 index 00000000..121dadb9 --- /dev/null +++ b/frontend/src/pages/metrics/MetricsStudio.tsx @@ -0,0 +1,591 @@ +import { useMemo, useState } from 'react' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { useNavigate } from 'react-router-dom' +import { format } from 'date-fns' +import { Play, Plus, RefreshCw, Sparkles, Trash2 } from 'lucide-react' +import { apiClient } from '../../lib/api' +import { getApiErrorMessage } from '../../lib/apiErrors' +import Button from '../../components/Button' +import AIProviderModelPicker from '../../components/AIProviderModelPicker' +import MetricPickerPanel from './components/MetricPickerPanel' +import { formatStudioModelLabel } from './components/MetricsStudioRunHeader' +import MetricsManagement from './MetricsManagement' +import { useWorkspaceStore } from '../../store/workspaceStore' +import type { LLMGenerationConfig } from '../../config/llmGenerationParams' +import { + getCallImportBatchLabel, + getCallImportRowLabel, + getCallImportRowSubtitle, + getObservabilityCallLabel, + getPlaygroundRecordingLabel, + getSimulatedResultLabel, + getSimulatedResultSubtitle, +} from './utils/sourceLabels' + +type SourceKind = 'call_import_row' | 'call_recording' | 'evaluator_result' + +type StudioSource = { + source_kind: SourceKind + source_ref: string + display_label: string +} + +type SourceTab = 'imports' | 'recordings' | 'simulated' + +export default function MetricsStudio() { + const navigate = useNavigate() + const queryClient = useQueryClient() + const activeWorkspaceId = useWorkspaceStore((s) => s.activeWorkspaceId) + const [metricTab, setMetricTab] = useState<'active' | 'drafts'>('active') + const [sourceTab, setSourceTab] = useState('imports') + const [selectedMetricIds, setSelectedMetricIds] = useState([]) + const [runName, setRunName] = useState('') + const [transcriptSource, setTranscriptSource] = useState<'production' | 'diarised'>('diarised') + const [llmProvider, setLlmProvider] = useState('') + const [llmModel, setLlmModel] = useState('') + const [llmConfig, setLlmConfig] = useState({}) + const [showDraftModal, setShowDraftModal] = useState(false) + + const [selectedImportId, setSelectedImportId] = useState('') + const [selectedImportRowIds, setSelectedImportRowIds] = useState>(new Set()) + const [selectedRecordingIds, setSelectedRecordingIds] = useState>(new Set()) + const [selectedSimulatedIds, setSelectedSimulatedIds] = useState>(new Set()) + + const { data: activeMetrics = [] } = useQuery({ + queryKey: ['metrics', 'studio', 'active', activeWorkspaceId], + queryFn: () => + apiClient.listMetrics(undefined, true, { + includeDrafts: false, + enabledOnly: true, + }), + }) + + const { data: draftMetrics = [] } = useQuery({ + queryKey: ['metrics', 'studio', 'drafts', activeWorkspaceId], + queryFn: () => apiClient.listMetrics(undefined, true, { draftsOnly: true }), + }) + + const metricsForPicker = metricTab === 'drafts' ? draftMetrics : activeMetrics + + const { data: importsData } = useQuery({ + queryKey: ['call-imports', 'studio'], + queryFn: () => apiClient.listCallImports({ page: 1, page_size: 100 }), + }) + + const { data: importDetail } = useQuery({ + queryKey: ['call-import', selectedImportId, 'studio-rows'], + queryFn: () => apiClient.getCallImport(selectedImportId, { row_limit: 100 }), + enabled: !!selectedImportId, + }) + + const { data: playgroundRecordings = [] } = useQuery({ + queryKey: ['call-recordings', 'studio'], + queryFn: () => apiClient.listCallRecordings(0, 100), + }) + + const { data: observabilityCalls = [] } = useQuery({ + queryKey: ['observability-calls', 'studio'], + queryFn: () => apiClient.listObservabilityCalls(0, 100), + }) + + const { data: simulatedResults } = useQuery({ + queryKey: ['evaluator-results', 'studio-simulated'], + queryFn: () => apiClient.listEvaluatorResults({ testAgentsOnly: true, limit: 100 }), + }) + + const { data: runsData, isLoading: runsLoading } = useQuery({ + queryKey: ['metric-studio-runs'], + queryFn: () => apiClient.listMetricStudioRuns(), + refetchInterval: (query) => { + const items = query.state.data?.items ?? [] + return items.some((r: any) => r.status === 'running' || r.status === 'pending') + ? 3000 + : false + }, + }) + + const promoteMutation = useMutation({ + mutationFn: (metricId: string) => apiClient.promoteMetric(metricId), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['metrics'] }) + }, + }) + + const runMutation = useMutation({ + mutationFn: () => + apiClient.createMetricStudioRun({ + name: runName.trim() || undefined, + metric_ids: selectedMetricIds, + sources: selectedSources.map(({ source_kind, source_ref, display_label }) => ({ + source_kind, + source_ref, + display_label, + })), + transcript_source: transcriptSource, + ...(llmProvider && llmModel + ? { + llm_provider: llmProvider, + llm_model: llmModel, + llm_config: llmConfig, + } + : {}), + }), + onSuccess: (run) => { + queryClient.invalidateQueries({ queryKey: ['metric-studio-runs'] }) + navigate(`/metrics-management/studio/runs/${run.id}`) + }, + }) + + const deleteRunMutation = useMutation({ + mutationFn: (runId: string) => apiClient.deleteMetricStudioRun(runId), + onSuccess: () => queryClient.invalidateQueries({ queryKey: ['metric-studio-runs'] }), + }) + + const importRows = importDetail?.rows ?? [] + const recordings = useMemo( + () => [ + ...playgroundRecordings.map((r: any) => ({ + id: r.call_short_id, + label: getPlaygroundRecordingLabel(r), + subtitle: r.call_short_id, + kind: 'playground' as const, + })), + ...observabilityCalls.map((r: any) => ({ + id: r.call_short_id, + label: getObservabilityCallLabel(r), + subtitle: r.call_short_id, + kind: 'webhook' as const, + })), + ], + [playgroundRecordings, observabilityCalls], + ) + + const simulatedItems = simulatedResults?.items ?? [] + + const selectedSources = useMemo(() => { + const next: StudioSource[] = [] + + for (const rowId of selectedImportRowIds) { + const row = importRows.find((r: any) => r.id === rowId) + next.push({ + source_kind: 'call_import_row', + source_ref: rowId, + display_label: row ? getCallImportRowLabel(row) : `Import row ${rowId.slice(0, 8)}`, + }) + } + for (const callShortId of selectedRecordingIds) { + const rec = recordings.find((r) => r.id === callShortId) + next.push({ + source_kind: 'call_recording', + source_ref: callShortId, + display_label: rec?.label ?? callShortId, + }) + } + for (const resultId of selectedSimulatedIds) { + const item = simulatedItems.find( + (r: any) => r.id === resultId || r.result_id === resultId, + ) + next.push({ + source_kind: 'evaluator_result', + source_ref: item?.id ?? resultId, + display_label: item ? getSimulatedResultLabel(item) : resultId.slice(0, 8), + }) + } + return next + }, [ + selectedImportRowIds, + selectedRecordingIds, + selectedSimulatedIds, + importRows, + recordings, + simulatedItems, + ]) + + const runError = runMutation.isError + ? getApiErrorMessage(runMutation.error, 'Failed to start Studio run.') + : null + + const canRun = selectedMetricIds.length > 0 && selectedSources.length > 0 + + return ( +
+
+
+
+

Metrics

+ +
+
+ {(['active', 'drafts'] as const).map((tab) => ( + + ))} +
+ + {metricTab === 'drafts' && draftMetrics.length > 0 && ( +
+ {draftMetrics.map((m: any) => ( +
+ {m.name} + +
+ ))} +
+ )} +
+ +
+

Call sources

+
+ {( + [ + { id: 'imports', label: 'Call Imports' }, + { id: 'recordings', label: 'Recordings' }, + { id: 'simulated', label: 'Simulated' }, + ] as const + ).map((tab) => ( + + ))} +
+ + {sourceTab === 'imports' && ( +
+ +
+ {importRows.map((row: any) => ( + + ))} +
+
+ )} + + {sourceTab === 'recordings' && ( +
+
+ {recordings.map((rec) => ( + + ))} +
+
+ )} + + {sourceTab === 'simulated' && ( +
+
+ {simulatedItems.map((item: any) => ( + + ))} +
+
+ )} + + {selectedSources.length > 0 && ( +
+ {selectedSources.map((src) => ( + + {src.display_label} + + + ))} +
+ )} +
+ +
+

Run configuration

+
+ + setRunName(e.target.value)} + className="w-full rounded-md border border-gray-300 px-3 py-2 text-sm" + placeholder="Optional label" + /> +
+
+ + +
+ setLlmConfig(next ?? {})} + /> + {runError &&

{runError}

} + {!canRun && ( +

+ Select at least one metric and one call source to run. +

+ )} + +
+
+ +
+
+

+ + Recent Studio runs +

+ +
+ {runsLoading ? ( +
Loading runs…
+ ) : (runsData?.items ?? []).length === 0 ? ( +
+ No Studio runs yet. Configure metrics and sources above, then run an evaluation. +
+ ) : ( +
+ {(runsData?.items ?? []).map((run: any) => { + const progress = + run.total_items > 0 + ? Math.round((run.completed_items / run.total_items) * 100) + : 0 + const inProgress = run.status === 'running' || run.status === 'pending' + + return ( +
navigate(`/metrics-management/studio/runs/${run.id}`)} + onKeyDown={(event) => { + if (event.key === 'Enter' || event.key === ' ') { + event.preventDefault() + navigate(`/metrics-management/studio/runs/${run.id}`) + } + }} + > +
+
+

+ {run.name || `Run ${run.id.slice(0, 8)}`} +

+

+ {format(new Date(run.created_at), 'MMM d, yyyy HH:mm')} ·{' '} + {run.completed_items}/{run.total_items} completed · {run.status} +

+

+ {formatStudioModelLabel(run)} +

+ {inProgress && ( +
+
+
+
+ {progress}% +
+ )} +
+ +
+
+ )})} +
+ )} +
+ + setShowDraftModal(false)} + onMetricCreated={(metric: any) => { + const childIds = (metric.children ?? []).map((c: any) => c.id) + const ids = [metric.id, ...childIds] + setSelectedMetricIds((prev) => Array.from(new Set([...prev, ...ids]))) + setMetricTab('drafts') + setShowDraftModal(false) + }} + /> +
+ ) +} diff --git a/frontend/src/pages/metrics/MetricsStudioRunDetail.tsx b/frontend/src/pages/metrics/MetricsStudioRunDetail.tsx new file mode 100644 index 00000000..6156efb6 --- /dev/null +++ b/frontend/src/pages/metrics/MetricsStudioRunDetail.tsx @@ -0,0 +1,152 @@ +import { useMemo, useState } from 'react' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { Link, useParams } from 'react-router-dom' +import { apiClient } from '../../lib/api' +import { getApiErrorMessage } from '../../lib/apiErrors' +import { useWorkspaceStore } from '../../store/workspaceStore' +import MetricsStudioRunHeader from './components/MetricsStudioRunHeader' +import MetricsStudioSourceList from './components/MetricsStudioSourceList' +import MetricsStudioScorePanel from './components/MetricsStudioScorePanel' +import { buildChildMetricIds } from './utils/metricScoreFilters' + +function flattenMetricNames(metrics: any[]): Record { + const map: Record = {} + for (const metric of metrics) { + map[metric.id] = metric.name + for (const child of metric.children ?? []) { + map[child.id] = child.name + } + } + return map +} + +export default function MetricsStudioRunDetail() { + const { runId = '' } = useParams<{ runId: string }>() + const queryClient = useQueryClient() + const activeWorkspaceId = useWorkspaceStore((s) => s.activeWorkspaceId) + const [selectedResultId, setSelectedResultId] = useState(null) + + const { data: run, isLoading: runLoading } = useQuery({ + queryKey: ['metric-studio-run', runId], + queryFn: () => apiClient.getMetricStudioRun(runId), + enabled: !!runId, + refetchInterval: (query) => + query.state.data?.status === 'running' ? 3000 : false, + }) + + const { data: resultsData, isLoading: resultsLoading } = useQuery({ + queryKey: ['metric-studio-run-results', runId], + queryFn: () => apiClient.listMetricStudioRunResults(runId), + enabled: !!runId, + refetchInterval: () => (run?.status === 'running' ? 3000 : false), + }) + + const { data: activeMetrics = [] } = useQuery({ + queryKey: ['metrics', 'studio', 'active', activeWorkspaceId], + queryFn: () => + apiClient.listMetrics(undefined, true, { + includeDrafts: false, + enabledOnly: true, + }), + }) + + const { data: draftMetrics = [] } = useQuery({ + queryKey: ['metrics', 'studio', 'drafts', activeWorkspaceId], + queryFn: () => apiClient.listMetrics(undefined, true, { draftsOnly: true }), + }) + + const retryMutation = useMutation({ + mutationFn: (resultIds?: string[]) => apiClient.retryMetricStudioRun(runId, resultIds), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['metric-studio-run', runId] }) + queryClient.invalidateQueries({ queryKey: ['metric-studio-run-results', runId] }) + }, + }) + + const promoteMutation = useMutation({ + mutationFn: (metricId: string) => apiClient.promoteMetric(metricId), + onSuccess: () => queryClient.invalidateQueries({ queryKey: ['metrics'] }), + }) + + const results = resultsData?.items ?? [] + const selectedResult = + results.find((r: any) => r.id === selectedResultId) ?? results[0] + + const metricNameById = useMemo( + () => flattenMetricNames([...activeMetrics, ...draftMetrics]), + [activeMetrics, draftMetrics], + ) + + const draftMetricIds = useMemo( + () => new Set(draftMetrics.map((m: any) => m.id)), + [draftMetrics], + ) + + const childMetricIds = useMemo( + () => buildChildMetricIds([...activeMetrics, ...draftMetrics]), + [activeMetrics, draftMetrics], + ) + + if (runLoading || resultsLoading) { + return ( +
+
+
+ ) + } + + if (!run) { + return ( +
+ + ← Back to Studio + +

Run not found.

+
+ ) + } + + const retryError = retryMutation.isError + ? getApiErrorMessage(retryMutation.error, 'Retry failed.') + : null + + return ( +
+ + ← Back to Studio + + + 0 ? () => retryMutation.mutate(undefined) : undefined} + retryPending={retryMutation.isPending} + /> + + {retryError &&

{retryError}

} + +
+ +
+ promoteMutation.mutate(metricId)} + /> +
+
+
+ ) +} diff --git a/frontend/src/pages/metrics/components/MetricPickerPanel.tsx b/frontend/src/pages/metrics/components/MetricPickerPanel.tsx new file mode 100644 index 00000000..7add6eb7 --- /dev/null +++ b/frontend/src/pages/metrics/components/MetricPickerPanel.tsx @@ -0,0 +1,160 @@ +import { ChevronDown, ChevronRight } from 'lucide-react' +import { useMemo, useState } from 'react' + +export interface MetricPickerItem { + id: string + name: string + enabled?: boolean + lifecycle?: string + selection_mode?: string | null + children?: MetricPickerItem[] +} + +interface MetricPickerPanelProps { + metrics: MetricPickerItem[] + selectedMetricIds: string[] + onChange: (ids: string[]) => void + emptyMessage?: string +} + +export default function MetricPickerPanel({ + metrics, + selectedMetricIds, + onChange, + emptyMessage = 'No metrics available.', +}: MetricPickerPanelProps) { + const [expandedParents, setExpandedParents] = useState>(new Set()) + const selectedSet = useMemo(() => new Set(selectedMetricIds), [selectedMetricIds]) + + const toggleMetric = (id: string) => { + const next = new Set(selectedSet) + if (next.has(id)) next.delete(id) + else next.add(id) + onChange(Array.from(next)) + } + + const enabledChildIds = (parent: MetricPickerItem): string[] => + (parent.children ?? []) + .filter((c) => c.enabled !== false || c.lifecycle === 'draft') + .map((c) => c.id) + + const toggleParentMetric = (parent: MetricPickerItem) => { + const childIds = enabledChildIds(parent) + const next = new Set(selectedSet) + const parentSelected = next.has(parent.id) + const someChildren = childIds.some((cid) => next.has(cid)) + if (parentSelected || someChildren) { + next.delete(parent.id) + for (const cid of childIds) next.delete(cid) + } else { + next.add(parent.id) + for (const cid of childIds) next.add(cid) + } + onChange(Array.from(next)) + } + + const toggleParentExpanded = (id: string) => { + setExpandedParents((prev) => { + const next = new Set(prev) + if (next.has(id)) next.delete(id) + else next.add(id) + return next + }) + } + + if (metrics.length === 0) { + return

{emptyMessage}

+ } + + return ( +
+ {metrics.map((metric) => { + const children = (metric.children ?? []).filter( + (c) => c.enabled !== false || c.lifecycle === 'draft', + ) + const isParent = Boolean(metric.selection_mode && children.length > 0) + const isExpanded = expandedParents.has(metric.id) + const childIds = enabledChildIds(metric) + const selectedChildCount = childIds.filter((cid) => selectedSet.has(cid)).length + const parentChecked = + selectedSet.has(metric.id) || + (childIds.length > 0 && childIds.every((cid) => selectedSet.has(cid))) + const parentIndeterminate = + !parentChecked && childIds.some((cid) => selectedSet.has(cid)) + + if (isParent) { + return ( +
+
+ + +
+ {isExpanded && ( +
+ {children.map((child) => ( +
+ {child.name} +
+ ))} +
+ )} +
+ ) + } + + return ( + + ) + })} +
+ ) +} diff --git a/frontend/src/pages/metrics/components/MetricScoreGrid.tsx b/frontend/src/pages/metrics/components/MetricScoreGrid.tsx new file mode 100644 index 00000000..445c6af1 --- /dev/null +++ b/frontend/src/pages/metrics/components/MetricScoreGrid.tsx @@ -0,0 +1,271 @@ +import type { ReactNode } from 'react' +import { AudioWaveform, Brain, Sparkles } from 'lucide-react' +import { + filterVisibleMetricScores, + type MetricScoreEntry, +} from '../utils/metricScoreFilters' + +type MetricScoreGridProps = { + metricScores: Record + metricNameById?: Record + childMetricIds?: Set + draftMetricIds?: Set + onPromoteDraft?: (metricId: string) => void +} + +const METRIC_CATEGORIES: Record = { + 'Pitch Variance': 'acoustic', + Jitter: 'acoustic', + Shimmer: 'acoustic', + HNR: 'acoustic', + 'MOS Score': 'ai_voice', + 'Emotion Category': 'ai_voice', + 'Emotion Confidence': 'ai_voice', + Valence: 'ai_voice', + Arousal: 'ai_voice', + 'Speaker Consistency': 'ai_voice', + 'Prosody Score': 'ai_voice', +} + +function getCategory(metricName: string): 'acoustic' | 'ai_voice' | 'llm' { + return METRIC_CATEGORIES[metricName] ?? 'llm' +} + +function formatMetricValue(value: unknown, type: string | undefined, _metricName: string): ReactNode { + if (value === null || value === undefined) return — + + const normalizedType = type?.toLowerCase() + + if (normalizedType === 'boolean') { + const boolValue = value === true || value === 1 || value === '1' || value === 'true' + return ( + + {boolValue ? 'Yes' : 'No'} + + ) + } + + if (normalizedType === 'rating') { + if (typeof value === 'string' && Number.isNaN(parseFloat(value))) { + return ( + + {value} + + ) + } + const numValue = typeof value === 'number' ? value : parseFloat(String(value)) + if (Number.isNaN(numValue)) return — + const percentage = Math.round(Math.max(0, Math.min(1, numValue)) * 100) + const barColor = + percentage >= 80 ? 'bg-emerald-500' : percentage >= 60 ? 'bg-amber-500' : 'bg-rose-500' + const textColor = + percentage >= 80 ? 'text-emerald-700' : percentage >= 60 ? 'text-amber-700' : 'text-rose-700' + return ( +
+
+ {percentage} + % +
+
+
+
+
+ ) + } + + if (normalizedType === 'number') { + const numValue = typeof value === 'number' ? value : parseFloat(String(value)) + if (Number.isNaN(numValue)) return — + return {numValue.toFixed(2)} + } + + if (normalizedType === 'text') { + return ( +

+ {String(value)} +

+ ) + } + + if (normalizedType === 'category') { + return ( + + {String(value)} + + ) + } + + return ( + + {String(value)} + + ) +} + +function MetricTile({ + metricId, + metric, + metricNameById, + draftMetricIds, + onPromoteDraft, + accent, +}: { + metricId: string + metric: MetricScoreEntry + metricNameById?: Record + draftMetricIds?: Set + onPromoteDraft?: (metricId: string) => void + accent: 'purple' | 'violet' | 'indigo' +}) { + const name = metric.metric_name || metricNameById?.[metricId] || metricId.slice(0, 8) + const displayValue = metric.skipped ?? metric.value + const borderClass = + accent === 'purple' + ? 'border-purple-200 bg-purple-50/50' + : accent === 'violet' + ? 'border-violet-200 bg-violet-50/50' + : 'border-indigo-200 bg-indigo-50/50' + const titleClass = + accent === 'purple' + ? 'text-purple-800' + : accent === 'violet' + ? 'text-violet-800' + : 'text-indigo-800' + + return ( +
+
+ {name} +
+
{formatMetricValue(displayValue, metric.type, name)}
+ {metric.rationale?.trim() && ( +

+ {metric.rationale.trim()} +

+ )} + {draftMetricIds?.has(metricId) && onPromoteDraft && ( + + )} +
+ ) +} + +function MetricSection({ + title, + icon: Icon, + badge, + accent, + entries, + metricNameById, + draftMetricIds, + onPromoteDraft, +}: { + title: string + icon: typeof Sparkles + badge: string + accent: 'purple' | 'violet' | 'indigo' + entries: Array<[string, MetricScoreEntry]> + metricNameById?: Record + draftMetricIds?: Set + onPromoteDraft?: (metricId: string) => void +}) { + if (entries.length === 0) return null + const badgeClass = + accent === 'purple' + ? 'bg-purple-100 text-purple-700' + : accent === 'violet' + ? 'bg-violet-100 text-violet-700' + : 'bg-indigo-100 text-indigo-700' + const titleClass = + accent === 'purple' + ? 'text-purple-800' + : accent === 'violet' + ? 'text-violet-800' + : 'text-indigo-800' + + return ( +
+
+ +

{title}

+ {badge} +
+
+ {entries.map(([metricId, metric]) => ( + + ))} +
+
+ ) +} + +export default function MetricScoreGrid({ + metricScores, + metricNameById, + childMetricIds = new Set(), + draftMetricIds, + onPromoteDraft, +}: MetricScoreGridProps) { + const entries = filterVisibleMetricScores(metricScores, childMetricIds) + + if (entries.length === 0) { + return

No metric scores yet.

+ } + + const aiVoice = entries.filter(([, m]) => getCategory(m.metric_name || '') === 'ai_voice') + const acoustic = entries.filter(([, m]) => getCategory(m.metric_name || '') === 'acoustic') + const llm = entries.filter(([, m]) => getCategory(m.metric_name || '') === 'llm') + + return ( +
+ + + +
+ ) +} diff --git a/frontend/src/pages/metrics/components/MetricsStudioAudioPlayer.tsx b/frontend/src/pages/metrics/components/MetricsStudioAudioPlayer.tsx new file mode 100644 index 00000000..d56e659b --- /dev/null +++ b/frontend/src/pages/metrics/components/MetricsStudioAudioPlayer.tsx @@ -0,0 +1,105 @@ +import { useEffect, useState } from 'react' +import { Volume2 } from 'lucide-react' +import { apiClient } from '../../../lib/api' + +type MetricsStudioAudioPlayerProps = { + sourceKind: string + sourceRef: string + metadata: Record +} + +function pickString(value: unknown): string | null { + return typeof value === 'string' && value.trim() ? value.trim() : null +} + +export default function MetricsStudioAudioPlayer({ + sourceKind, + sourceRef, + metadata, +}: MetricsStudioAudioPlayerProps) { + const [audioUrl, setAudioUrl] = useState(null) + const [loading, setLoading] = useState(false) + const [error, setError] = useState(null) + + useEffect(() => { + let cancelled = false + let objectUrl: string | null = null + + async function loadAudio() { + setLoading(true) + setError(null) + setAudioUrl(null) + + try { + const providerUrl = pickString(metadata.recording_url) + if (providerUrl) { + if (!cancelled) setAudioUrl(providerUrl) + return + } + + const s3Key = + pickString(metadata.audio_s3_key) ?? pickString(metadata.recording_s3_key) + if (s3Key) { + const { url } = await apiClient.getS3PresignedUrl(s3Key) + if (!cancelled) setAudioUrl(url) + return + } + + if (sourceKind === 'call_recording' && sourceRef) { + const source = pickString(metadata.source)?.toLowerCase() + objectUrl = + source === 'webhook' + ? await apiClient.getObservabilityCallAudioUrl(sourceRef) + : await apiClient.getCallRecordingAudioUrl(sourceRef) + if (!cancelled) setAudioUrl(objectUrl) + return + } + + if (sourceKind === 'evaluator_result' && sourceRef) { + const result = await apiClient.getEvaluatorResult(sourceRef) + const resultS3Key = pickString(result?.audio_s3_key) + if (resultS3Key) { + const { url } = await apiClient.getS3PresignedUrl(resultS3Key) + if (!cancelled) setAudioUrl(url) + return + } + const callDataUrl = pickString(result?.call_data?.recording_url) + if (callDataUrl && !cancelled) { + setAudioUrl(callDataUrl) + } + } + } catch { + if (!cancelled) setError('Could not load recording') + } finally { + if (!cancelled) setLoading(false) + } + } + + void loadAudio() + + return () => { + cancelled = true + if (objectUrl) URL.revokeObjectURL(objectUrl) + } + }, [sourceKind, sourceRef, metadata]) + + if (loading) { + return ( +

+ Loading recording… +

+ ) + } + + if (error || !audioUrl) return null + + return ( +
+
+ +

Call recording

+
+
+ ) +} diff --git a/frontend/src/pages/metrics/components/MetricsStudioRunHeader.tsx b/frontend/src/pages/metrics/components/MetricsStudioRunHeader.tsx new file mode 100644 index 00000000..3f5a87c8 --- /dev/null +++ b/frontend/src/pages/metrics/components/MetricsStudioRunHeader.tsx @@ -0,0 +1,199 @@ +import { format } from 'date-fns' +import { + Brain, + CheckCircle, + Clock, + FileText, + Loader, + RefreshCw, + XCircle, +} from 'lucide-react' +import Button from '../../../components/Button' + +export type StudioRunModelInfo = { + llm_provider?: string | null + llm_model?: string | null +} + +type StudioRun = StudioRunModelInfo & { + id: string + name?: string | null + status: string + total_items: number + completed_items: number + failed_items: number + created_at: string + transcript_source?: string | null +} + +type MetricsStudioRunHeaderProps = { + run: StudioRun + onRetryFailed?: () => void + retryPending?: boolean +} + +function RunStatusChip({ status }: { status: string }) { + const configs: Record = { + pending: { + label: 'Pending', + className: 'bg-slate-100 text-slate-700', + icon: , + }, + running: { + label: 'Running', + className: 'bg-amber-100 text-amber-800', + icon: , + }, + partial: { + label: 'Partial', + className: 'bg-orange-100 text-orange-800', + icon: , + }, + completed: { + label: 'Completed', + className: 'bg-emerald-100 text-emerald-800', + icon: , + }, + failed: { + label: 'Failed', + className: 'bg-rose-100 text-rose-800', + icon: , + }, + } + const config = configs[status] ?? { + label: status, + className: 'bg-gray-100 text-gray-700', + icon: null, + } + + return ( + + {config.icon} + {config.label} + + ) +} + +function MetaPill({ + icon, + label, + value, +}: { + icon: React.ReactNode + label: string + value: string +}) { + return ( +
+ {icon} + {label} + {value} +
+ ) +} + +export function formatStudioModelLabel(run: StudioRunModelInfo): string { + const provider = run.llm_provider?.trim() + const model = run.llm_model?.trim() + if (provider && model) return `${provider} / ${model}` + if (provider) return provider + if (model) return model + return 'Organization default' +} + +function formatTranscriptSourceLabel(source?: string | null): string { + return source === 'production' ? 'Production (CSV)' : 'Diarised' +} + +export default function MetricsStudioRunHeader({ + run, + onRetryFailed, + retryPending, +}: MetricsStudioRunHeaderProps) { + const progress = + run.total_items > 0 ? Math.round((run.completed_items / run.total_items) * 100) : 0 + const showProgressBar = run.status === 'running' || run.status === 'pending' + + return ( +
+
+
+

+ {run.name || `Studio run ${run.id.slice(0, 8)}`} +

+

+ {format(new Date(run.created_at), 'MMM d, yyyy HH:mm')} +

+
+
+ + {run.failed_items > 0 && onRetryFailed && ( + + )} +
+
+ +
+ } + label="Model" + value={formatStudioModelLabel(run)} + /> + } + label="Transcript" + value={formatTranscriptSourceLabel(run.transcript_source)} + /> + } + label="Progress" + value={`${run.completed_items}/${run.total_items}${ + run.failed_items > 0 ? ` (${run.failed_items} failed)` : '' + }`} + /> +
+ + {showProgressBar && ( +
+
+
+
+ {progress}% +
+ )} +
+ ) +} + +export function ResultStatusChip({ status }: { status: string }) { + const styles: Record = { + pending: 'bg-slate-100 text-slate-600', + running: 'bg-amber-100 text-amber-700', + partial: 'bg-orange-100 text-orange-700', + completed: 'bg-emerald-100 text-emerald-700', + failed: 'bg-rose-100 text-rose-700', + } + + return ( + + {status} + + ) +} diff --git a/frontend/src/pages/metrics/components/MetricsStudioScorePanel.tsx b/frontend/src/pages/metrics/components/MetricsStudioScorePanel.tsx new file mode 100644 index 00000000..cc9622ef --- /dev/null +++ b/frontend/src/pages/metrics/components/MetricsStudioScorePanel.tsx @@ -0,0 +1,157 @@ +import { Brain } from 'lucide-react' +import { Link } from 'react-router-dom' +import MetricScoreGrid from './MetricScoreGrid' +import MetricsStudioAudioPlayer from './MetricsStudioAudioPlayer' +import MetricsStudioTranscriptPanel from './MetricsStudioTranscriptPanel' +import { formatStudioModelLabel, type StudioRunModelInfo } from './MetricsStudioRunHeader' + +type StudioRunResult = { + id: string + source_kind: string + source_ref: string + display_label?: string | null + source_metadata?: Record | null + status: string + metric_scores?: Record< + string, + { + value?: unknown + type?: string + metric_name?: string + rationale?: string | null + skipped?: unknown + } + > + error_message?: string | null +} + +type MetricsStudioScorePanelProps = { + result: StudioRunResult | null | undefined + run: StudioRunModelInfo + transcriptSource: string + metricNameById: Record + childMetricIds: Set + draftMetricIds: Set + onPromoteDraft: (metricId: string) => void +} + +export default function MetricsStudioScorePanel({ + result, + run, + transcriptSource, + metricNameById, + childMetricIds, + draftMetricIds, + onPromoteDraft, +}: MetricsStudioScorePanelProps) { + if (!result) { + return ( +
+ Select a source on the left to inspect scores and transcript. +
+ ) + } + + const metadata = result.source_metadata ?? {} + const evaluationTranscript = + typeof metadata.evaluation_transcript === 'string' + ? metadata.evaluation_transcript + : null + const transcriptSourceUsed = + typeof metadata.transcript_source_used === 'string' + ? metadata.transcript_source_used + : transcriptSource + + return ( +
+
+
+
+

Evaluation details

+

+ {result.display_label || result.source_ref} +

+
+
+ +
+

{formatStudioModelLabel(run)}

+

Evaluation model

+
+
+
+ +
+ {typeof metadata.persona_name === 'string' && metadata.persona_name && ( +

+ Persona: {metadata.persona_name} +

+ )} + {typeof metadata.scenario_name === 'string' && metadata.scenario_name && ( +

+ Scenario: {metadata.scenario_name} +

+ )} + {typeof metadata.call_import_id === 'string' && metadata.call_import_id && ( + + View call import → + + )} + {result.source_kind === 'evaluator_result' && ( + + View full simulation → + + )} + {result.source_kind === 'call_recording' && ( + + View recording → + + )} +
+ + {result.error_message && ( +

+ {result.error_message} +

+ )} +
+ + {result.status === 'completed' && ( +
+

+ Analysis results +

+ +
+ )} + + + + {result.status === 'completed' && ( + + )} +
+ ) +} diff --git a/frontend/src/pages/metrics/components/MetricsStudioSourceList.tsx b/frontend/src/pages/metrics/components/MetricsStudioSourceList.tsx new file mode 100644 index 00000000..16b8dfb4 --- /dev/null +++ b/frontend/src/pages/metrics/components/MetricsStudioSourceList.tsx @@ -0,0 +1,109 @@ +import { Bot, Mic, Upload } from 'lucide-react' +import { ResultStatusChip } from './MetricsStudioRunHeader' + +export type StudioRunResult = { + id: string + source_kind: string + source_ref: string + display_label?: string | null + status: string + metric_scores?: Record +} + +type MetricsStudioSourceListProps = { + results: StudioRunResult[] + selectedResultId: string | null + onSelect: (resultId: string) => void +} + +function sourceIcon(sourceKind: string) { + switch (sourceKind) { + case 'call_import_row': + return Upload + case 'call_recording': + return Mic + case 'evaluator_result': + return Bot + default: + return Upload + } +} + +function sourceKindLabel(sourceKind: string): string { + switch (sourceKind) { + case 'call_import_row': + return 'Call import' + case 'call_recording': + return 'Recording' + case 'evaluator_result': + return 'Simulation' + default: + return sourceKind + } +} + +export default function MetricsStudioSourceList({ + results, + selectedResultId, + onSelect, +}: MetricsStudioSourceListProps) { + if (results.length === 0) { + return ( +
+ No source results yet. +
+ ) + } + + return ( +
+
+

+ Sources · {results.length} +

+
+
+ {results.map((result) => { + const Icon = sourceIcon(result.source_kind) + const selected = (selectedResultId ?? results[0]?.id) === result.id + const scoreCount = Object.keys(result.metric_scores ?? {}).length + + return ( + + ) + })} +
+
+ ) +} diff --git a/frontend/src/pages/metrics/components/MetricsStudioTranscriptPanel.tsx b/frontend/src/pages/metrics/components/MetricsStudioTranscriptPanel.tsx new file mode 100644 index 00000000..9fe7a4ee --- /dev/null +++ b/frontend/src/pages/metrics/components/MetricsStudioTranscriptPanel.tsx @@ -0,0 +1,35 @@ +import { FileText } from 'lucide-react' +import TranscriptView from '../../callImports/components/TranscriptView' + +type MetricsStudioTranscriptPanelProps = { + transcript: string | null | undefined + transcriptSource: 'production' | 'diarised' | string +} + +function transcriptSourceLabel(source: string): string { + return source === 'production' ? 'Production (CSV)' : 'Diarised' +} + +export default function MetricsStudioTranscriptPanel({ + transcript, + transcriptSource, +}: MetricsStudioTranscriptPanelProps) { + if (!transcript?.trim()) return null + + return ( +
+
+ +

+ Evaluation transcript +

+ + {transcriptSourceLabel(transcriptSource)} + +
+
+ +
+
+ ) +} diff --git a/frontend/src/pages/metrics/components/MetricsTabBar.tsx b/frontend/src/pages/metrics/components/MetricsTabBar.tsx new file mode 100644 index 00000000..9bb41740 --- /dev/null +++ b/frontend/src/pages/metrics/components/MetricsTabBar.tsx @@ -0,0 +1,38 @@ +import { Link, useLocation } from 'react-router-dom' +import { BarChart3, Sparkles } from 'lucide-react' + +const TABS = [ + { id: 'metrics', label: 'Metrics', href: '/metrics-management', icon: BarChart3 }, + { id: 'studio', label: 'Studio', href: '/metrics-management/studio', icon: Sparkles }, +] as const + +export default function MetricsTabBar() { + const location = useLocation() + + const isStudio = location.pathname.startsWith('/metrics-management/studio') + + return ( +
+ +
+ ) +} diff --git a/frontend/src/pages/metrics/index.ts b/frontend/src/pages/metrics/index.ts index 2495edfb..c8cba8fb 100644 --- a/frontend/src/pages/metrics/index.ts +++ b/frontend/src/pages/metrics/index.ts @@ -1,2 +1,5 @@ export { default as Metrics } from './Metrics' export { default as MetricsManagement } from './MetricsManagement' +export { default as MetricsLayout } from './MetricsLayout' +export { default as MetricsStudio } from './MetricsStudio' +export { default as MetricsStudioRunDetail } from './MetricsStudioRunDetail' diff --git a/frontend/src/pages/metrics/metricValuePayloadUtils.test.ts b/frontend/src/pages/metrics/metricValuePayloadUtils.test.ts new file mode 100644 index 00000000..c8180992 --- /dev/null +++ b/frontend/src/pages/metrics/metricValuePayloadUtils.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from 'vitest' +import { + buildSingleMetricValuePayload, + formatMetricValuePayloadJson, + type SingleMetricFormSnapshot, +} from './metricValuePayloadUtils' + +const baseForm: SingleMetricFormSnapshot = { + name: 'Booking Confirmation', + description: 'True when the agent confirms the booking date and time.', + metric_type: 'boolean', + custom_data_type: 'boolean', + enum_options_csv: '', + number_min: 0, + number_max: 10, + capture_rationale: false, +} + +describe('buildSingleMetricValuePayload', () => { + it('builds boolean payload without rationale', () => { + const payload = buildSingleMetricValuePayload(baseForm) + expect(payload).toEqual({ + value: true, + type: 'boolean', + metric_name: 'Booking Confirmation', + description: 'True when the agent confirms the booking date and time.', + }) + }) + + it('includes rationale when capture_rationale is enabled', () => { + const payload = buildSingleMetricValuePayload({ + ...baseForm, + capture_rationale: true, + }) + expect(payload.rationale).toBeTruthy() + expect(payload.type).toBe('boolean') + }) + + it('builds enum payload with options and first option as value', () => { + const payload = buildSingleMetricValuePayload({ + ...baseForm, + metric_type: 'rating', + custom_data_type: 'enum', + enum_options_csv: 'Excellent, Good, Poor', + }) + expect(payload).toMatchObject({ + value: 'Excellent', + type: 'enum', + metric_name: 'Booking Confirmation', + description: 'True when the agent confirms the booking date and time.', + options: ['Excellent', 'Good', 'Poor'], + }) + }) + + it('builds number payload using midpoint of min and max', () => { + const payload = buildSingleMetricValuePayload({ + ...baseForm, + metric_type: 'number', + custom_data_type: 'number_range', + number_min: 0, + number_max: 10, + }) + expect(payload).toEqual({ + value: 5, + type: 'number', + metric_name: 'Booking Confirmation', + description: 'True when the agent confirms the booking date and time.', + }) + }) + + it('builds text payload without rationale even when capture_rationale is on', () => { + const payload = buildSingleMetricValuePayload({ + ...baseForm, + metric_type: 'text', + custom_data_type: 'boolean', + capture_rationale: true, + }) + expect(payload.type).toBe('text') + expect(typeof payload.value).toBe('string') + expect(payload.rationale).toBeUndefined() + }) + + it('uses empty description when description is blank', () => { + const payload = buildSingleMetricValuePayload({ + ...baseForm, + description: ' ', + }) + expect(payload.description).toBe('') + }) + + it('uses placeholder name when metric name is empty', () => { + const payload = buildSingleMetricValuePayload({ + ...baseForm, + name: ' ', + }) + expect(payload.metric_name).toBe('(unnamed metric)') + }) +}) + +describe('formatMetricValuePayloadJson', () => { + it('returns pretty-printed JSON', () => { + const json = formatMetricValuePayloadJson( + buildSingleMetricValuePayload(baseForm), + ) + expect(json).toContain('"type": "boolean"') + expect(JSON.parse(json)).toBeTruthy() + }) +}) diff --git a/frontend/src/pages/metrics/metricValuePayloadUtils.ts b/frontend/src/pages/metrics/metricValuePayloadUtils.ts new file mode 100644 index 00000000..ad8bb46d --- /dev/null +++ b/frontend/src/pages/metrics/metricValuePayloadUtils.ts @@ -0,0 +1,100 @@ +export interface SingleMetricFormSnapshot { + name: string + description: string + metric_type: 'number' | 'boolean' | 'rating' | 'text' + custom_data_type: 'boolean' | 'enum' | 'number_range' + enum_options_csv: string + number_min: number + number_max: number + capture_rationale: boolean +} + +const EXAMPLE_RATIONALE = + 'Brief justification referencing the transcript.' + +const EXAMPLE_TEXT_VALUE = + 'A brief 1-3 sentence summary describing what was observed.' + +const DEFAULT_ENUM_OPTIONS = ['Excellent', 'Good', 'Poor'] + +function parseEnumOptions(csv: string): string[] { + return csv + .split(',') + .map((opt) => opt.trim()) + .filter(Boolean) +} + +function exampleNumberValue(min: number, max: number): number { + const lo = Number(min) + const hi = Number(max) + if (Number.isFinite(lo) && Number.isFinite(hi)) { + return Math.round(((lo + hi) / 2) * 100) / 100 + } + if (Number.isFinite(lo)) return lo + return 5 +} + +function resolveStoredType(form: SingleMetricFormSnapshot): string { + if (form.metric_type === 'text') return 'text' + if (form.metric_type === 'boolean' || form.custom_data_type === 'boolean') { + return 'boolean' + } + if (form.custom_data_type === 'enum' || form.metric_type === 'rating') { + return 'enum' + } + return 'number' +} + +function exampleValue( + storedType: string, + form: SingleMetricFormSnapshot, + enumOptions: string[], +): unknown { + if (storedType === 'text') return EXAMPLE_TEXT_VALUE + if (storedType === 'boolean') return true + if (storedType === 'enum') { + return enumOptions[0] ?? DEFAULT_ENUM_OPTIONS[0] + } + return exampleNumberValue(form.number_min, form.number_max) +} + +/** + * Build a representative score entry matching what evaluation workers + * persist under metric_scores[] for standalone metrics. + */ +export function buildSingleMetricValuePayload( + form: SingleMetricFormSnapshot, +): Record { + const metricName = form.name.trim() || '(unnamed metric)' + const storedType = resolveStoredType(form) + const enumOptions = + storedType === 'enum' + ? (() => { + const parsed = parseEnumOptions(form.enum_options_csv) + return parsed.length > 0 ? parsed : [...DEFAULT_ENUM_OPTIONS] + })() + : [] + + const payload: Record = { + value: exampleValue(storedType, form, enumOptions), + type: storedType, + metric_name: metricName, + description: form.description.trim(), + } + + if (storedType === 'enum') { + payload.options = enumOptions + } + + if (form.capture_rationale && form.metric_type !== 'text') { + payload.rationale = EXAMPLE_RATIONALE + } + + return payload +} + +export function formatMetricValuePayloadJson( + payload: Record, +): string { + return JSON.stringify(payload, null, 2) +} diff --git a/frontend/src/pages/metrics/utils/metricScoreFilters.ts b/frontend/src/pages/metrics/utils/metricScoreFilters.ts new file mode 100644 index 00000000..95542e55 --- /dev/null +++ b/frontend/src/pages/metrics/utils/metricScoreFilters.ts @@ -0,0 +1,69 @@ +const LEGACY_CATEGORY_LABEL_METRIC_NAMES = new Set([ + 'yes', + 'no', + 'true', + 'false', + 'same', + 'different', +]) + +export type MetricScoreEntry = { + value?: unknown + type?: string + metric_name?: string + parent_metric_id?: string | null + rationale?: string | null + skipped?: unknown +} + +export function isLegacyCategoryLabelMetric(metric: { + type?: string | null + metric_name?: string | null +}): boolean { + if ((metric.type || '').toLowerCase() !== 'boolean') return false + const name = (metric.metric_name || '').trim().toLowerCase() + return LEGACY_CATEGORY_LABEL_METRIC_NAMES.has(name) +} + +export function buildChildMetricIds( + metrics: Array<{ id?: string; parent_metric_id?: string | null; children?: any[] }>, +): Set { + const ids = new Set() + const visit = (metric: { id?: string; parent_metric_id?: string | null; children?: any[] }) => { + if (metric.parent_metric_id && metric.id) ids.add(metric.id) + for (const child of metric.children || []) { + if (child?.id) ids.add(child.id) + visit(child) + } + } + for (const metric of metrics) visit(metric) + return ids +} + +/** Hide per-label child booleans; keep parent category + standalone metrics. */ +export function shouldHideMetricScore( + metricId: string, + metric: MetricScoreEntry, + childMetricIds: Set, +): boolean { + return Boolean( + metric.parent_metric_id || + childMetricIds.has(metricId) || + isLegacyCategoryLabelMetric(metric), + ) +} + +export function filterVisibleMetricScores( + metricScores: Record, + childMetricIds: Set, +): Array<[string, MetricScoreEntry]> { + return Object.entries(metricScores).filter(([metricId, metric]) => { + if (shouldHideMetricScore(metricId, metric, childMetricIds)) return false + const val = metric.skipped ?? metric.value + if (val === null || val === undefined) return false + if (val === '') return false + if (typeof val === 'string' && val.toLowerCase() === 'n/a') return false + if (typeof val === 'string' && val.trim() === '') return false + return true + }) +} diff --git a/frontend/src/pages/metrics/utils/sourceLabels.ts b/frontend/src/pages/metrics/utils/sourceLabels.ts new file mode 100644 index 00000000..eb90495c --- /dev/null +++ b/frontend/src/pages/metrics/utils/sourceLabels.ts @@ -0,0 +1,72 @@ +import type { CallImportRow } from '../../../types/api' + +export function basenameFromS3Key(key: string | null | undefined): string | null { + if (!key) return null + const parts = key.split('/') + const name = parts[parts.length - 1]?.trim() + return name || null +} + +export function getCallImportBatchLabel(importBatch: { + original_filename?: string | null + name?: string | null + id?: string +}): string { + return ( + importBatch.original_filename?.trim() || + importBatch.name?.trim() || + importBatch.id?.slice(0, 8) || + 'Unnamed import' + ) +} + +export function getCallImportRowLabel(row: Pick): string { + const filename = basenameFromS3Key(row.recording_s3_key) + if (filename) return filename + if (row.conversation_id?.trim()) return row.conversation_id.trim() + return `Import row ${row.row_index ?? row.id.slice(0, 8)}` +} + +export function getCallImportRowSubtitle(row: Pick): string { + if (row.conversation_id?.trim()) { + return `Row ${row.row_index} · ${row.conversation_id}` + } + return `Row ${row.row_index}` +} + +export function getPlaygroundRecordingLabel(recording: { + display_name?: string | null + call_short_id: string +}): string { + return recording.display_name?.trim() || recording.call_short_id +} + +export function getObservabilityCallLabel(call: { + display_name?: string | null + agent?: { name?: string | null } | null + provider_platform?: string | null + call_short_id: string +}): string { + return ( + call.display_name?.trim() || + call.agent?.name?.trim() || + (call.provider_platform ? `${call.provider_platform} call` : null) || + call.call_short_id + ) +} + +export function getSimulatedResultLabel(item: { + name?: string | null + result_id?: string | null + id: string +}): string { + return item.name?.trim() || item.result_id?.trim() || item.id.slice(0, 8) +} + +export function getSimulatedResultSubtitle(item: { + result_id?: string | null + id: string +}): string | null { + if (item.result_id && item.result_id !== item.id) return item.result_id + return null +} diff --git a/frontend/src/pages/platform/PlatformAdmin.tsx b/frontend/src/pages/platform/PlatformAdmin.tsx new file mode 100644 index 00000000..d4fdb604 --- /dev/null +++ b/frontend/src/pages/platform/PlatformAdmin.tsx @@ -0,0 +1,273 @@ +import { useCallback, useEffect, useState } from 'react' +import { useNavigate } from 'react-router-dom' +import { AlertCircle, Building2, KeyRound, Loader2, LogOut, Users } from 'lucide-react' +import { Button, Chip, Switch } from '@heroui/react' +import Logo from '../../components/Logo' +import { apiClient, type PlatformOrganizationItem, type PlatformOrganizationStats } from '../../lib/api' +import { isPlatformAdminAuthenticated, usePlatformAdminStore } from '../../store/platformAdminStore' +import PlatformOrgMembersModal from './PlatformOrgMembersModal' +import PlatformSignupCodes from './PlatformSignupCodes' + +type PlatformTab = 'organizations' | 'signup-codes' + +const NAV_ITEMS: Array<{ key: PlatformTab; label: string; icon: typeof Building2 }> = [ + { key: 'organizations', label: 'Organizations', icon: Building2 }, + { key: 'signup-codes', label: 'Signup codes', icon: KeyRound }, +] + +export default function PlatformAdmin() { + const navigate = useNavigate() + const { admin, logout } = usePlatformAdminStore() + const [activeTab, setActiveTab] = useState('organizations') + const [stats, setStats] = useState(null) + const [organizations, setOrganizations] = useState([]) + const [loading, setLoading] = useState(true) + const [error, setError] = useState('') + const [updatingOrgId, setUpdatingOrgId] = useState(null) + const [selectedOrg, setSelectedOrg] = useState(null) + + const loadOrganizations = useCallback(async () => { + setError('') + setLoading(true) + try { + const [statsRes, orgsRes] = await Promise.all([ + apiClient.getPlatformOrganizationStats(), + apiClient.listPlatformOrganizations({ limit: 200 }), + ]) + setStats(statsRes) + setOrganizations(orgsRes.items) + } catch (err: any) { + if (err?.response?.status === 401) { + logout() + navigate('/platform/login', { replace: true }) + return + } + setError(err?.response?.data?.detail || 'Failed to load organizations') + } finally { + setLoading(false) + } + }, [logout, navigate]) + + useEffect(() => { + if (!isPlatformAdminAuthenticated()) { + navigate('/platform/login', { replace: true }) + return + } + if (activeTab === 'organizations') { + loadOrganizations() + } + }, [activeTab, loadOrganizations, navigate]) + + const handleToggleOrg = async (org: PlatformOrganizationItem, enabled: boolean) => { + setUpdatingOrgId(org.id) + setError('') + try { + const updated = await apiClient.updatePlatformOrganization(org.id, { is_active: enabled }) + setOrganizations((prev) => prev.map((item) => (item.id === org.id ? updated : item))) + const statsRes = await apiClient.getPlatformOrganizationStats() + setStats(statsRes) + } catch (err: any) { + setError(err?.response?.data?.detail || 'Failed to update organization') + } finally { + setUpdatingOrgId(null) + } + } + + const handleLogout = () => { + logout() + navigate('/platform/login', { replace: true }) + } + + return ( +
+ + +
+
+
+ +

{admin?.email}

+
+ +
+ +
+ {NAV_ITEMS.map(({ key, label, icon: Icon }) => { + const isActive = activeTab === key + return ( + + ) + })} +
+ +
+
+

+ {NAV_ITEMS.find((item) => item.key === activeTab)?.label} +

+ {admin &&

Signed in as {admin.email}

} +
+ + {error && activeTab === 'organizations' && ( + } className="w-full max-w-full h-auto py-2"> + {error} + + )} + + {activeTab === 'organizations' && ( + <> + {stats && ( +
+ + + +
+ )} + +
+
+

Organizations

+ {loading && } +
+
+ + + + + + + + + + + + + {organizations.map((org) => ( + + + + + + + + + ))} + {!loading && organizations.length === 0 && ( + + + + )} + +
NameOrganization IDMembersStatusEnabledActions
+
+ + {org.name} +
+ 		{org.id}{org.member_count} + + {org.is_active ? 'Active' : 'Disabled'} + + + handleToggleOrg(org, enabled)} + aria-label={`Toggle ${org.name}`} + /> + + +
+ No organizations found. +
+
+
+ + )} + + {activeTab === 'signup-codes' && } +
+
+ + {selectedOrg && ( + setSelectedOrg(null)} /> + )} +
+ ) +} + +function StatCard({ label, value }: { label: string; value: number }) { + return ( +
+
{label}
+
{value}
+
+ ) +} diff --git a/frontend/src/pages/platform/PlatformLogin.tsx b/frontend/src/pages/platform/PlatformLogin.tsx new file mode 100644 index 00000000..d9c97c1c --- /dev/null +++ b/frontend/src/pages/platform/PlatformLogin.tsx @@ -0,0 +1,110 @@ +import { useState } from 'react' +import { useNavigate } from 'react-router-dom' +import { AlertCircle, Eye, EyeOff } from 'lucide-react' +import { Button, Chip } from '@heroui/react' +import Logo from '../../components/Logo' +import { apiClient } from '../../lib/api' +import { usePlatformAdminStore } from '../../store/platformAdminStore' + +export default function PlatformLogin() { + const navigate = useNavigate() + const { setSession } = usePlatformAdminStore() + const [email, setEmail] = useState('') + const [password, setPassword] = useState('') + const [showPassword, setShowPassword] = useState(false) + const [error, setError] = useState('') + const [isLoading, setIsLoading] = useState(false) + + const handleSubmit = async (e: React.FormEvent) => { + e.preventDefault() + setError('') + setIsLoading(true) + try { + const res = await apiClient.platformLogin(email, password) + setSession(res.access_token, res.admin) + navigate('/platform') + } catch (err: any) { + const status = err?.response?.status + const detail = err?.response?.data?.detail + if (status === 404 && detail === 'Not found') { + setError( + 'No platform admin exists in the database this API is connected to. ' + + 'Run migration 057, then: python -m scripts.create_platform_admin --email you@example.com', + ) + } else if (status === 404) { + setError( + 'Platform admin API was not found. Restart the backend with the latest code, ' + + 'then confirm POST /api/v1/platform/auth/login is reachable.', + ) + } else if (!err?.response) { + setError( + `Could not reach the API at ${import.meta.env.VITE_API_URL || 'http://localhost:8000'}. ` + + 'Is the backend running?', + ) + } else { + setError(detail || 'Sign in failed') + } + } finally { + setIsLoading(false) + } + } + + return ( +
+
+
+ +
+
+

Platform Admin

+

Sign in to manage organizations

+
+ setEmail(e.target.value)} + required + className="w-full px-4 py-3 text-base text-gray-900 bg-gray-50 border-2 border-gray-200 rounded-xl focus:outline-none focus:border-[#ca8a04] focus:bg-white" + /> +
+ setPassword(e.target.value)} + required + className="w-full px-4 py-3 pr-12 text-base text-gray-900 bg-gray-50 border-2 border-gray-200 rounded-xl focus:outline-none focus:border-[#ca8a04] focus:bg-white" + /> + +
+ {error && ( + } className="w-full max-w-full h-auto py-2"> + {error} + + )} + +
+
+
+
+ ) +} diff --git a/frontend/src/pages/platform/PlatformOrgMembersModal.tsx b/frontend/src/pages/platform/PlatformOrgMembersModal.tsx new file mode 100644 index 00000000..cc51078c --- /dev/null +++ b/frontend/src/pages/platform/PlatformOrgMembersModal.tsx @@ -0,0 +1,263 @@ +import { useEffect, useState } from 'react' +import { AlertCircle, Eye, EyeOff, KeyRound, Loader2, Users, X } from 'lucide-react' +import { Button, Chip } from '@heroui/react' +import { + apiClient, + type PlatformOrganizationItem, + type PlatformOrgUser, +} from '../../lib/api' +import { PASSWORD_POLICY_HINT, validatePasswordPolicy } from '../../lib/passwordPolicy' + +type Props = { + org: PlatformOrganizationItem + onClose: () => void +} + +function generateStrongPassword(): string { + const buf = new Uint8Array(16) + crypto.getRandomValues(buf) + const alphabet = 'ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz23456789!@#$%^&*' + let pwd = '' + for (let i = 0; i < buf.length; i++) { + pwd += alphabet[buf[i] % alphabet.length] + } + return pwd +} + +export default function PlatformOrgMembersModal({ org, onClose }: Props) { + const [members, setMembers] = useState([]) + const [loading, setLoading] = useState(true) + const [error, setError] = useState('') + + const [resetTarget, setResetTarget] = useState(null) + const [newPassword, setNewPassword] = useState('') + const [confirmPassword, setConfirmPassword] = useState('') + const [showPassword, setShowPassword] = useState(false) + const [resetError, setResetError] = useState('') + const [resetting, setResetting] = useState(false) + const [resetSuccess, setResetSuccess] = useState('') + + useEffect(() => { + let active = true + setLoading(true) + setError('') + apiClient + .listPlatformOrganizationUsers(org.id) + .then((rows) => active && setMembers(rows)) + .catch((err: any) => { + if (active) setError(err?.response?.data?.detail || 'Failed to load members') + }) + .finally(() => active && setLoading(false)) + return () => { + active = false + } + }, [org.id]) + + const openReset = (member: PlatformOrgUser) => { + setResetTarget(member) + setNewPassword('') + setConfirmPassword('') + setShowPassword(false) + setResetError('') + setResetSuccess('') + } + + const closeReset = () => { + setResetTarget(null) + setNewPassword('') + setConfirmPassword('') + setResetError('') + } + + const handleGeneratePassword = () => { + const pwd = generateStrongPassword() + setNewPassword(pwd) + setConfirmPassword(pwd) + setShowPassword(true) + setResetError('') + } + + const handleResetSubmit = async (e: React.FormEvent) => { + e.preventDefault() + if (!resetTarget) return + setResetError('') + setResetSuccess('') + + const policy = validatePasswordPolicy(newPassword) + if (!policy.valid) { + setResetError(policy.message || 'Invalid password') + return + } + if (newPassword !== confirmPassword) { + setResetError('Passwords do not match') + return + } + + setResetting(true) + try { + await apiClient.platformResetUserPassword(org.id, resetTarget.id, newPassword) + setResetSuccess(`Password reset for ${resetTarget.email}`) + closeReset() + } catch (err: any) { + setResetError(err?.response?.data?.detail || 'Failed to reset password') + } finally { + setResetting(false) + } + } + + return ( +
+
e.stopPropagation()} + > +
+
+

+ + {org.name} +

+

{org.id}

+
+ +
+ +
+ {error && ( + }> + {error} + + )} + {resetSuccess && ( + + {resetSuccess} + + )} + + {loading ? ( +
+ +
+ ) : members.length === 0 ? ( +

No members in this organization.

+ ) : ( + + + + + + + + + + + {members.map((member) => ( + + + + + + + ))} + +
EmailRoleStatusActions
{member.email}{member.role} + + {member.is_active ? 'Active' : 'Inactive'} + + + +
+ )} +
+
+ + {resetTarget && ( +
+
e.stopPropagation()} + > +

Reset password

+

+ Set a new password for {resetTarget.email}. Share + it securely — it is not emailed automatically. +

+
+
+ + +
+
+ setNewPassword(e.target.value)} + required + minLength={8} + maxLength={32} + placeholder={PASSWORD_POLICY_HINT} + className="w-full px-3 py-2 pr-10 border border-gray-300 rounded-lg focus:outline-none focus:border-amber-500" + /> + +
+ setConfirmPassword(e.target.value)} + required + placeholder="Confirm password" + className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:outline-none focus:border-amber-500" + /> + {resetError && ( + + {resetError} + + )} +
+ + +
+
+
+
+ )} +
+ ) +} diff --git a/frontend/src/pages/platform/PlatformSignupCodes.tsx b/frontend/src/pages/platform/PlatformSignupCodes.tsx new file mode 100644 index 00000000..76273d01 --- /dev/null +++ b/frontend/src/pages/platform/PlatformSignupCodes.tsx @@ -0,0 +1,281 @@ +import { useCallback, useEffect, useState } from 'react' +import { + AlertCircle, + Check, + Copy, + KeyRound, + Loader2, + Plus, + Trash2, +} from 'lucide-react' +import { Button, Chip, Switch } from '@heroui/react' +import { apiClient, type PlatformSignupCode } from '../../lib/api' + +export default function PlatformSignupCodes() { + const [codes, setCodes] = useState([]) + const [loading, setLoading] = useState(true) + const [error, setError] = useState('') + const [gatedSignup, setGatedSignup] = useState(null) + + const [showCreate, setShowCreate] = useState(false) + const [newCode, setNewCode] = useState('') + const [newLabel, setNewLabel] = useState('') + const [newMaxUses, setNewMaxUses] = useState('') + const [creating, setCreating] = useState(false) + const [createdCodePlaintext, setCreatedCodePlaintext] = useState(null) + const [copied, setCopied] = useState(false) + const [updatingId, setUpdatingId] = useState(null) + + const loadCodes = useCallback(async () => { + setError('') + setLoading(true) + try { + const [codeRows, authConfig] = await Promise.all([ + apiClient.listPlatformSignupCodes(), + apiClient.getAuthConfig(), + ]) + setCodes(codeRows) + setGatedSignup(Boolean(authConfig.gated_signup)) + } catch (err: any) { + setError(err?.response?.data?.detail || 'Failed to load signup codes') + } finally { + setLoading(false) + } + }, []) + + useEffect(() => { + loadCodes() + }, [loadCodes]) + + const handleCreate = async (e: React.FormEvent) => { + e.preventDefault() + setCreating(true) + setError('') + try { + const created = await apiClient.createPlatformSignupCode({ + code: newCode.trim(), + label: newLabel.trim() || undefined, + max_uses: newMaxUses ? parseInt(newMaxUses, 10) : undefined, + }) + setCreatedCodePlaintext(created.code || newCode.trim()) + setNewCode('') + setNewLabel('') + setNewMaxUses('') + setShowCreate(false) + await loadCodes() + } catch (err: any) { + setError(err?.response?.data?.detail || 'Failed to create code') + } finally { + setCreating(false) + } + } + + const handleToggleActive = async (row: PlatformSignupCode, active: boolean) => { + setUpdatingId(row.id) + setError('') + try { + await apiClient.updatePlatformSignupCode(row.id, { is_active: active }) + await loadCodes() + } catch (err: any) { + setError(err?.response?.data?.detail || 'Failed to update code') + } finally { + setUpdatingId(null) + } + } + + const handleDeactivate = async (row: PlatformSignupCode) => { + if (!confirm(`Deactivate reference code "${row.label || row.id}"?`)) return + setUpdatingId(row.id) + setError('') + try { + await apiClient.deactivatePlatformSignupCode(row.id) + await loadCodes() + } catch (err: any) { + setError(err?.response?.data?.detail || 'Failed to deactivate code') + } finally { + setUpdatingId(null) + } + } + + const copyCreatedCode = async () => { + if (!createdCodePlaintext) return + await navigator.clipboard.writeText(createdCodePlaintext) + setCopied(true) + setTimeout(() => setCopied(false), 2000) + } + + return ( +
+
+
+

+ + Signup reference codes +

+

+ {gatedSignup === null + ? 'Loading signup settings…' + : gatedSignup + ? 'Gated signup is enabled — new users must enter a valid code at registration.' + : 'Gated signup is off — set auth.local_password.gated_signup.enabled: true in config.yml to require codes.'} +

+
+ +
+ + {createdCodePlaintext && ( +
+
+

Code created — copy it now

+

{createdCodePlaintext}

+

+ This is the only time the plaintext code is shown. +

+
+ +
+ )} + + {error && ( + } className="w-full max-w-full h-auto py-2"> + {error} + + )} + +
+ {loading ? ( +
+ +
+ ) : codes.length === 0 ? ( +

No reference codes yet.

+ ) : ( + + + + + + + + + + + + {codes.map((row) => ( + + + + + + + + ))} + +
LabelUsesExpiresActiveActions
+
{row.label || '—'}
+
{row.id}
+ 		+ {row.use_count} + {row.max_uses != null ? ` / ${row.max_uses}` : ' / ∞'} + + {row.expires_at + ? new Date(row.expires_at).toLocaleString() + : 'Never'} + + handleToggleActive(row, v)} + aria-label="Toggle code active" + /> + + +
+ )} +
+ + {showCreate && ( +
setShowCreate(false)} + > +
e.stopPropagation()} + > +

Create reference code

+
+
+ + setNewCode(e.target.value)} + required + minLength={4} + maxLength={64} + placeholder="e.g. BETA2026" + className="w-full px-3 py-2 border border-gray-300 rounded-lg uppercase focus:outline-none focus:border-amber-500" + /> +
+
+ + setNewLabel(e.target.value)} + placeholder="Beta invite batch" + className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:outline-none focus:border-amber-500" + /> +
+
+ + setNewMaxUses(e.target.value)} + placeholder="Unlimited if empty" + className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:outline-none focus:border-amber-500" + /> +
+
+ + +
+
+
+
+ )} +
+ ) +} diff --git a/frontend/src/store/platformAdminStore.ts b/frontend/src/store/platformAdminStore.ts new file mode 100644 index 00000000..1b79b15c --- /dev/null +++ b/frontend/src/store/platformAdminStore.ts @@ -0,0 +1,49 @@ +import { create } from 'zustand' + +type PlatformAdminUser = { + id: string + email: string +} + +interface PlatformAdminState { + accessToken: string | null + admin: PlatformAdminUser | null + setSession: (token: string, admin: PlatformAdminUser) => void + logout: () => void +} + +const STORAGE_TOKEN = 'platformAccessToken' +const STORAGE_ADMIN = 'platformAdminUser' + +function readStoredAdmin(): PlatformAdminUser | null { + try { + const raw = localStorage.getItem(STORAGE_ADMIN) + return raw ? (JSON.parse(raw) as PlatformAdminUser) : null + } catch { + return null + } +} + +export const usePlatformAdminStore = create((set) => { + const storedToken = localStorage.getItem(STORAGE_TOKEN) + const storedAdmin = readStoredAdmin() + + return { + accessToken: storedToken, + admin: storedAdmin, + setSession: (token, admin) => { + localStorage.setItem(STORAGE_TOKEN, token) + localStorage.setItem(STORAGE_ADMIN, JSON.stringify(admin)) + set({ accessToken: token, admin }) + }, + logout: () => { + localStorage.removeItem(STORAGE_TOKEN) + localStorage.removeItem(STORAGE_ADMIN) + set({ accessToken: null, admin: null }) + }, + } +}) + +export function isPlatformAdminAuthenticated(): boolean { + return Boolean(localStorage.getItem(STORAGE_TOKEN)) +} diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index eb4a9baa..ce71af44 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -1,2165 +1,2167 @@ -// API Types matching the backend schemas - -export type { LLMGenerationConfig } from '../config/llmGenerationParams' -import type { LLMGenerationConfig } from '../config/llmGenerationParams' - -export enum EvaluationType { - ASR = 'asr', - TTS = 'tts', -} - -export enum EvaluationStatus { - PENDING = 'pending', - PROCESSING = 'processing', - COMPLETED = 'completed', - FAILED = 'failed', - CANCELLED = 'cancelled', -} - -export interface AudioFile { - id: string - filename: string - format: string - file_size: number - duration?: number | null - sample_rate?: number | null - channels?: number | null - uploaded_at: string -} - -export interface Evaluation { - id: string - audio_id: string - reference_text?: string | null - evaluation_type: EvaluationType - model_name?: string | null - status: EvaluationStatus - metrics_requested?: string[] | null - created_at: string - started_at?: string | null - completed_at?: string | null - error_message?: string | null -} - -export interface DashboardSummary { - evaluations: { - total: number - completed: number - pending: number - failed: number - } - resources: { - agents: number - personas: number - scenarios: number - integrations: number - voice_bundles: number - } - setup_progress: { - has_integration: boolean - has_voice_bundle: boolean - has_agent: boolean - has_evaluation: boolean - } - metrics: { - total: number - enabled: number - } - call_imports: { - total: number - } - call_import_evaluations: { - total: number - completed: number - running: number - failed: number - } - recent_evaluations: Evaluation[] -} - -export interface ModelConfigEntry { - provider: string - model_type: string - description?: string - featured?: boolean - featured_rank?: number - highlights?: string[] -} - -export interface EvaluationCreate { - audio_id: string - reference_text?: string | null - evaluation_type: EvaluationType - model_name?: string | null - metrics?: string[] -} - -export interface EvaluationResult { - evaluation_id: string - status: EvaluationStatus - transcript?: string | null - metrics: Record - processing_time?: number | null - model_used?: string | null - created_at: string -} - -export interface BatchEvaluationResult { - processed_files: number - failed_files: number - aggregated_metrics?: Record | null - individual_results: EvaluationResult[] -} - -/** Voice agent evaluator run (evaluator_results table). */ -export type EvaluatorResultStatus = - | 'queued' - | 'call_initiating' - | 'call_connecting' - | 'call_in_progress' - | 'call_ended' - | 'transcribing' - | 'evaluating' - | 'fetching_details' - | 'completed' - | 'failed' - -export interface EvaluatorResultMetricScore { - value: unknown - type: string - metric_name: string - parent_metric_id?: string | null -} - -export interface EvaluatorResultRow { - id: string - result_id: string - name: string | null - evaluator_id: string | null - agent_id?: string | null - persona_id?: string | null - scenario_id?: string | null - suite_id?: string | null - timestamp: string - duration_seconds: number | null - status: EvaluatorResultStatus - metric_scores: Record | null - error_message: string | null - agent?: { id: string; name: string } | null - scenario?: { id: string; name: string } | null -} - -export interface EvaluatorResultListResponse { - items: EvaluatorResultRow[] - total: number -} - -export interface EvaluatorResultCounts { - total: number - completed: number - failed: number - in_progress: number - last_run_at?: string | null -} - -export interface EvaluatorResultsScenarioSummary { - scenario_id: string - scenario_name: string - counts: EvaluatorResultCounts -} - -export interface EvaluatorResultsSuiteSummary { - suite_id: string - suite_name?: string | null - agent_id: string - persona_id?: string | null - counts: EvaluatorResultCounts - scenarios?: EvaluatorResultsScenarioSummary[] | null -} - -export interface EvaluatorResultsAgentSummary { - agent_id: string - agent_name: string - counts: EvaluatorResultCounts - suites?: EvaluatorResultsSuiteSummary[] | null -} - -export interface EvaluatorResultsOverviewResponse { - workspace_counts: EvaluatorResultCounts - agents: EvaluatorResultsAgentSummary[] - unassigned: { - counts: EvaluatorResultCounts - recent_result_ids: string[] - } -} - -export interface ListEvaluatorResultsParams { - skip?: number - limit?: number - evaluatorId?: string - agentId?: string - suiteId?: string - scenarioId?: string - status?: 'completed' | 'failed' | 'in_progress' - unassignedOnly?: boolean - playground?: boolean - testAgentsOnly?: boolean -} - -export interface APIKey { - id: string - key: string - name?: string | null - is_active: boolean - created_at: string - last_used?: string | null - message?: string -} - -export interface MessageResponse { - message: string -} - -// IAM & User Types -export enum Role { - READER = 'reader', - WRITER = 'writer', - ADMIN = 'admin', -} - -export enum InvitationStatus { - PENDING = 'pending', - ACCEPTED = 'accepted', - DECLINED = 'declined', - EXPIRED = 'expired', -} - -export interface User { - id: string - email: string - name?: string | null - is_active: boolean - created_at: string -} - -export interface OrganizationMember { - id: string - user_id: string - organization_id: string - role: Role - joined_at: string - user: User -} - -export interface Invitation { - id: string - organization_id: string - email: string - role: Role - status: InvitationStatus - expires_at: string - created_at: string - organization_name?: string | null -} - -export interface InvitationCreate { - email: string - role: Role -} - -export interface RoleUpdate { - role: Role -} - -export interface Profile { - id: string - email: string - name?: string | null - first_name?: string | null - last_name?: string | null - created_at: string - organizations: Array<{ - id: string - name: string - role: string - joined_at: string - }> -} - -export interface UserUpdate { - name?: string | null - first_name?: string | null - last_name?: string | null - email?: string | null -} - -export interface UserPreferences { - theme?: string - notifications_enabled?: boolean - email_notifications?: boolean - default_language?: string - [key: string]: any -} - -export interface UserPreferencesUpdate { - theme?: string - notifications_enabled?: boolean - email_notifications?: boolean - default_language?: string - [key: string]: any -} - -// Integration Types -export enum IntegrationPlatform { - RETELL = 'retell', - VAPI = 'vapi', - CARTESIA = 'cartesia', - ELEVENLABS = 'elevenlabs', - DEEPGRAM = 'deepgram', - MURF = 'murf', - SARVAM = 'sarvam', - VOICEMAKER = 'voicemaker', - SMALLEST = 'smallest', -} - -export enum TelephonyProvider { - PLIVO = 'plivo', - EXOTEL = 'exotel', - VOBIZ = 'vobiz', -} - -export type CredentialRoutingMode = 'inherit' | 'gateway' | 'direct' -export type GatewayInterfaceMode = 'inherit' | 'litellm_shim' | 'native_openai' - -export type EffectiveCredentialRouting = - | 'inherit' - | 'direct' - | 'gateway' - | 'bifrost' - | 'litellm_proxy' - -export interface Integration { - id: string - organization_id: string - platform: IntegrationPlatform - name?: string | null - public_key?: string | null - is_active: boolean - /** True if this row is the default credential for (org, platform). */ - is_default?: boolean - routing_mode?: CredentialRoutingMode - effective_routing?: EffectiveCredentialRouting - created_at: string - updated_at: string - last_tested_at?: string | null -} - -export interface IntegrationCreate { - platform: IntegrationPlatform - api_key: string - public_key?: string - name?: string | null - routing_mode?: CredentialRoutingMode - /** Mark the new credential as the default for (org, platform). */ - is_default?: boolean -} - -// VoiceBundle Types -export enum ModelProvider { - OPENAI = 'openai', - ANTHROPIC = 'anthropic', - GOOGLE = 'google', - XAI = 'xai', - FIREWORKS = 'fireworks', - COHERE = 'cohere', - MISTRAL = 'mistral', - META = 'meta', - TOGETHER = 'together', - PERPLEXITY = 'perplexity', - AZURE = 'azure', - AWS = 'aws', - DEEPGRAM = 'deepgram', - CARTESIA = 'cartesia', - ELEVENLABS = 'elevenlabs', - MURF = 'murf', - CUSTOM = 'custom', - SARVAM = 'sarvam', - VOICEMAKER = 'voicemaker', - SMALLEST = 'smallest', -} - -// AI Provider Types -export interface AIProvider { - id: string - provider: ModelProvider - api_key?: string | null - name?: string | null - endpoint_url?: string | null - is_active: boolean - /** True if this row is the default credential for (org, provider). */ - is_default?: boolean - routing_mode?: CredentialRoutingMode - gateway_model?: string | null - gateway_interface?: GatewayInterfaceMode - gateway_base_url?: string | null - gateway_auth_header?: string | null - gateway_auth_secret_env?: string | null - has_gateway_auth_secret?: boolean - gateway_extra_headers?: Record | null - /** True when provider secrets are resolved by the Bifrost gateway. */ - gateway_managed?: boolean - effective_routing?: EffectiveCredentialRouting - effective_gateway_interface?: 'litellm_shim' | 'native_openai' - created_at: string - updated_at: string - last_tested_at?: string | null -} - -export interface AIProviderCreate { - provider: ModelProvider - api_key?: string | null - name?: string | null - endpoint_url?: string | null - routing_mode?: CredentialRoutingMode - gateway_model?: string | null - gateway_interface?: GatewayInterfaceMode - gateway_base_url?: string | null - gateway_auth_header?: string | null - gateway_auth_secret_env?: string | null - gateway_auth_secret?: string | null - gateway_extra_headers?: Record | null - /** Mark the new credential as the default for (org, provider). */ - is_default?: boolean -} - -export interface AIProviderUpdate { - api_key?: string | null - name?: string | null - endpoint_url?: string | null - is_active?: boolean - routing_mode?: CredentialRoutingMode - gateway_model?: string | null - gateway_interface?: GatewayInterfaceMode - gateway_base_url?: string | null - gateway_auth_header?: string | null - gateway_auth_secret_env?: string | null - gateway_auth_secret?: string | null - clear_gateway_auth_secret?: boolean - gateway_extra_headers?: Record | null -} - -export enum VoiceBundleType { - STT_LLM_TTS = 'stt_llm_tts', - S2S = 's2s', -} - -export interface VoiceBundle { - id: string - name: string - description?: string | null - bundle_type: VoiceBundleType - stt_provider?: ModelProvider | null - stt_model?: string | null - /** - * Optional explicit AIProvider/Integration row id for STT. When null the - * runtime resolver picks the default credential for stt_provider. - */ - stt_credential_id?: string | null - llm_provider?: ModelProvider | null - llm_model?: string | null - llm_temperature?: number | null - llm_max_tokens?: number | null - llm_config?: Record | null - llm_credential_id?: string | null - tts_provider?: ModelProvider | null - tts_model?: string | null - tts_voice?: string | null - tts_config?: Record | null - tts_credential_id?: string | null - s2s_provider?: ModelProvider | null - s2s_model?: string | null - s2s_config?: Record | null - s2s_credential_id?: string | null - extra_metadata?: Record | null - is_active: boolean - created_at: string - updated_at: string - created_by?: string | null -} - -export interface VoiceBundleCreate { - name: string - description?: string | null - bundle_type?: VoiceBundleType - stt_provider?: ModelProvider | null - stt_model?: string | null - stt_credential_id?: string | null - llm_provider?: ModelProvider | null - llm_model?: string | null - llm_temperature?: number | null - llm_max_tokens?: number | null - llm_config?: Record | null - llm_credential_id?: string | null - tts_provider?: ModelProvider | null - tts_model?: string | null - tts_voice?: string | null - tts_config?: Record | null - tts_credential_id?: string | null - s2s_provider?: ModelProvider | null - s2s_model?: string | null - s2s_config?: Record | null - s2s_credential_id?: string | null - extra_metadata?: Record | null -} - -// Test Agent Types -export interface AgentPhoneAssignmentConflict { - agent_id: string - agent_name: string - phone_number: string -} - -export interface AgentPhoneAssignmentCheckResponse { - available: boolean - phone_number?: string | null - conflict?: AgentPhoneAssignmentConflict | null -} - -export interface TestAgent { - id: string - agent_id?: string | null - name: string - phone_number?: string | null - telephony_phone_number_id?: string | null - language: string - description: string | null - prompt_variables?: Record | null - silence_hangup_secs?: number - call_type: string - call_medium: string - voice_bundle_id?: string | null - voice_ai_integration_id?: string | null - voice_ai_agent_id?: string | null - provider_prompt?: string | null - provider_prompt_synced_at?: string | null - created_at: string - updated_at: string -} - -// Test Agent Conversation Types -export interface TestAgentConversation { - id: string - organization_id: string - agent_id: string - persona_id: string - scenario_id: string - voice_bundle_id: string - status: string - live_transcription?: Array<{ - speaker: string - text: string - timestamp: number - audio_segment_key?: string - }> | null - conversation_audio_key?: string | null - full_transcript?: string | null - started_at: string - ended_at?: string | null - duration_seconds?: number | null - conversation_metadata?: Record | null - created_at: string - updated_at: string - created_by?: string | null -} - -export interface TestAgentConversationCreate { - agent_id: string - persona_id: string - scenario_id: string - voice_bundle_id: string - conversation_metadata?: Record | null -} - -export interface TestAgentConversationUpdate { - status?: string | null - live_transcription?: Array> | null - full_transcript?: string | null - conversation_metadata?: Record | null -} - -export interface VoiceBundleUpdate { - name?: string - description?: string | null - stt_provider?: ModelProvider - stt_model?: string - stt_credential_id?: string | null - llm_provider?: ModelProvider - llm_model?: string - llm_temperature?: number | null - llm_max_tokens?: number | null - llm_config?: Record | null - llm_credential_id?: string | null - tts_provider?: ModelProvider - tts_model?: string - tts_voice?: string | null - tts_config?: Record | null - tts_credential_id?: string | null - s2s_provider?: ModelProvider | null - s2s_model?: string | null - s2s_config?: Record | null - s2s_credential_id?: string | null - extra_metadata?: Record | null - is_active?: boolean -} - -// Data Sources Types -export interface S3ConnectionTest { - bucket_name: string - region?: string - access_key_id: string - secret_access_key: string - endpoint_url?: string | null -} - -export interface S3ConnectionTestResponse { - success: boolean - message: string - bucket_name?: string | null -} - -export interface S3FileInfo { - key: string - filename: string - size: number - last_modified: string -} - -export interface S3FolderInfo { - name: string - path: string -} - -export interface S3ListFilesResponse { - files: S3FileInfo[] - total: number - prefix?: string | null -} - -export interface S3BrowseResponse { - folders: S3FolderInfo[] - files: S3FileInfo[] - current_path: string - organization_id: string -} - -export interface S3Status { - enabled: boolean - provider?: 's3' | 'gcs' | string - error?: string | null -} - -// Alert Types -export enum AlertMetricType { - NUMBER_OF_CALLS = 'number_of_calls', - CALL_DURATION = 'call_duration', - ERROR_RATE = 'error_rate', - SUCCESS_RATE = 'success_rate', - LATENCY = 'latency', - CUSTOM = 'custom', -} - -export enum AlertAggregation { - SUM = 'sum', - AVG = 'avg', - COUNT = 'count', - MIN = 'min', - MAX = 'max', -} - -export enum AlertOperator { - GREATER_THAN = '>', - LESS_THAN = '<', - GREATER_THAN_OR_EQUAL = '>=', - LESS_THAN_OR_EQUAL = '<=', - EQUAL = '=', - NOT_EQUAL = '!=', -} - -export enum AlertNotifyFrequency { - IMMEDIATE = 'immediate', - HOURLY = 'hourly', - DAILY = 'daily', - WEEKLY = 'weekly', -} - -export enum AlertStatus { - ACTIVE = 'active', - PAUSED = 'paused', - DISABLED = 'disabled', -} - -export enum AlertHistoryStatus { - TRIGGERED = 'triggered', - NOTIFIED = 'notified', - ACKNOWLEDGED = 'acknowledged', - RESOLVED = 'resolved', -} - -export interface Alert { - id: string - organization_id: string - name: string - description?: string | null - metric_type: AlertMetricType - aggregation: AlertAggregation - operator: AlertOperator - threshold_value: number - time_window_minutes: number - agent_ids?: string[] | null - notify_frequency: AlertNotifyFrequency - notify_emails?: string[] | null - notify_webhooks?: string[] | null - status: AlertStatus - created_at: string - updated_at: string - created_by?: string | null -} - -export interface AlertCreate { - name: string - description?: string | null - metric_type?: AlertMetricType - aggregation?: AlertAggregation - operator?: AlertOperator - threshold_value: number - time_window_minutes?: number - agent_ids?: string[] | null - notify_frequency?: AlertNotifyFrequency - notify_emails?: string[] - notify_webhooks?: string[] -} - -export interface AlertUpdate { - name?: string - description?: string | null - metric_type?: AlertMetricType - aggregation?: AlertAggregation - operator?: AlertOperator - threshold_value?: number - time_window_minutes?: number - agent_ids?: string[] | null - notify_frequency?: AlertNotifyFrequency - notify_emails?: string[] - notify_webhooks?: string[] - status?: AlertStatus -} - -export interface AlertHistoryItem { - id: string - organization_id: string - alert_id: string - triggered_at: string - triggered_value: number - threshold_value: number - status: AlertHistoryStatus - notified_at?: string | null - notification_details?: Record | null - acknowledged_at?: string | null - acknowledged_by?: string | null - resolved_at?: string | null - resolved_by?: string | null - resolution_notes?: string | null - context_data?: Record | null - created_at: string - updated_at: string - alert?: Alert -} - - -// Cron Job Types -export enum CronJobStatus { - ACTIVE = 'active', - PAUSED = 'paused', - COMPLETED = 'completed', -} - -export interface CronJob { - id: string - organization_id: string - name: string - cron_expression: string - timezone: string - max_runs: number - current_runs: number - evaluator_ids: string[] - status: CronJobStatus - next_run_at?: string | null - last_run_at?: string | null - created_at: string - updated_at: string - created_by?: string | null -} - -export interface CronJobCreate { - name: string - cron_expression: string - timezone: string - max_runs: number - evaluator_ids: string[] -} - -export interface CronJobUpdate { - name?: string - cron_expression?: string - timezone?: string - max_runs?: number - evaluator_ids?: string[] - status?: CronJobStatus -} - -// --- Call Imports --- - -/** - * Lifecycle for a call-import batch. - * - * - ``uploaded`` : file landed in S3, no mapping yet. - * - ``mapped`` : user picked a schema + sheet + column mapping; no - * rows materialised yet, no worker enqueued. - * - ``processing`` : rows materialised + workers enqueued. - * - ``pending`` : transient state used by the legacy one-shot - * ``POST /upload`` endpoint just before transitioning - * to ``processing``. - */ -export type CallImportStatus = - | 'pending' - | 'uploaded' - | 'mapped' - | 'processing' - | 'completed' - | 'partial' - | 'failed' - | 'deleting' - -export type CallImportRowStatus = - | 'pending' - | 'processing' - | 'completed' - | 'failed' - -/** Where the value in `transcript` came from. */ -export type CallImportTranscriptSource = - | 'csv' - | 'transcribed' - | 'edited' - | null -/** Lifecycle status for the post-hoc transcription workflow itself. */ -export type CallImportTranscriptStatus = - | 'idle' - | 'pending' - | 'running' - | 'completed' - | 'failed' - | null - -/** - * Which transcript an evaluation run scored against. - * - `production`: the CSV-supplied value on `CallImportRow.transcript`. - * - `diarised`: the worker-produced value on `CallImportRow.diarised_transcript`. - */ -export type CallImportEvaluationTranscriptSource = 'production' | 'diarised' - -/** - * One contiguous turn inside ``CallImportRow.diarised_segments``. - * - * The diarisation worker rewrites each pyannote ``Speaker N`` label - * into ``agent`` / ``user`` (first speaker = agent heuristic). Anything - * beyond two distinct speakers keeps a generic ``speaker_N`` label so - * multi-party recordings still render every voice. - */ -export interface CallImportDiarisedSegment { - speaker: string - text: string - start: number - end: number - /** Original pyannote label (``Speaker 1`` / ``Speaker 2`` / ...). */ - raw_speaker: string -} - -export interface CallImportRow { - id: string - row_index: number - /** Mandatory identifier per row. Renamed from ``external_call_id``. */ - conversation_id: string - recording_url: string | null - recording_date: string | null - /** Production transcript — the value supplied via the CSV upload. */ - transcript: string | null - /** Provenance of the stored production transcript (csv = CSV upload, edited = manual edit). */ - transcript_source: CallImportTranscriptSource - /** Legacy: provider recorded by the original transcription worker before the split. */ - transcript_provider: string | null - transcript_model: string | null - transcript_status: CallImportTranscriptStatus - transcript_error: string | null - transcribed_at: string | null - /** Diarised transcript — produced by the post-hoc diarisation worker. */ - diarised_transcript: string | null - /** Provider used by the diarisation worker (e.g. "deepgram"). */ - diarised_transcript_provider: string | null - diarised_transcript_model: string | null - diarised_transcript_status: CallImportTranscriptStatus - diarised_transcript_error: string | null - diarised_at: string | null - /** - * Structured speaker turns produced by the diarisation worker. Each - * entry is a single contiguous turn shaped as - * `{ speaker: 'agent' | 'user' | 'speaker_N', text, start, end, - * raw_speaker }`. ``diarised_transcript`` is a rendered - * `: ` view of this list with - * ``diarised_speaker_swap`` applied. ``null`` on legacy rows that - * were diarised before structured turns were persisted (or when the - * STT provider didn't surface segments). - */ - diarised_segments: CallImportDiarisedSegment[] | null - /** - * When ``true`` the ``agent`` <-> ``user`` mapping inside - * ``diarised_segments`` is inverted in the rendered transcript / - * CSV export. The worker writes the canonical mapping using a - * "first speaker is the agent" heuristic; reviewers can flip the - * toggle from the row detail panel without re-running diarisation. - */ - diarised_speaker_swap: boolean - /** - * LLM that turned the STT plain-text output into structured - * ``diarised_segments``. NULL on legacy rows (pre-LLM-diariser). - */ - diarised_llm_provider: string | null - diarised_llm_model: string | null - /** - * Exact prompt the LLM diariser ran with. Useful for the modal to - * pre-fill its textarea when the operator wants to iterate on a - * previously-diarised row. - */ - diarised_prompt: string | null - /** - * Diarisation pipeline that produced this row's turns. - * - `stt_llm` (default) — two-stage STT then LLM diariser. - * - `llm_only` — single-stage multimodal LLM (audio in). - * Read-only; written by the worker on each diarisation. - */ - transcribe_mode?: 'stt_llm' | 'llm_only' - /** - * Per-row preservation of the mapped source cells. Values land here - * as whatever type the schema parameter coerced them to — - * strings (text / url / conversation_id / recording_url / - * recording_date / transcript / datetime), numbers, booleans, or - * ``null`` for blanks. Always - * coerce with ``String(value)`` before string operations. - */ - raw_columns: Record | null - status: CallImportRowStatus - recording_s3_key: string | null - recording_content_type: string | null - recording_size_bytes: number | null - error_message: string | null - attempts: number - created_at: string - updated_at: string -} - -export interface CallImportTag { - id: string - name: string - color: string | null - created_at: string - updated_at: string -} - -/** - * Parameter type tag on a Call Import schema parameter. - * - * - ``conversation_id``: mandatory identifier (one per schema). - * - ``recording_url``: feeds ``CallImportRow.recording_url``. - * - ``recording_date``: date-only call recording date used for reports. - * - ``transcript``: feeds ``CallImportRow.transcript``. - * - ``text`` / ``number`` / ``boolean`` / ``datetime`` / ``url``: - * generic typed fields preserved per row in ``raw_columns`` and - * surfaced in the evaluation export under the parameter's name. - */ -export type CallImportSchemaParameterType = - | 'conversation_id' - | 'recording_url' - | 'recording_date' - | 'transcript' - | 'text' - | 'number' - | 'boolean' - | 'datetime' - | 'url' - -export interface CallImportSchemaParameter { - id?: string - name: string - type: CallImportSchemaParameterType - description: string | null - is_required: boolean - ordering?: number -} - -export interface CallImportSchema { - id: string - organization_id: string - workspace_id: string - name: string - description: string | null - parameters: CallImportSchemaParameter[] - /** How many CallImport batches reference this schema. */ - usage_count: number - created_at: string - updated_at: string -} - -export interface CallImportSchemaListResponse { - items: CallImportSchema[] - total: number -} - -export interface CallImportSchemaCreate { - name: string - description?: string | null - parameters: Array> -} - -export interface CallImportSchemaUpdate { - name?: string - description?: string | null - parameters?: Array> -} - -/** - * In-org Workspace - the active workspace scopes call imports and - * metrics in the UI. The org's Default workspace is auto-seeded by - * migration 033 and cannot be deleted. - */ -export interface Workspace { - id: string - organization_id: string - name: string - slug: string - is_default: boolean - created_at: string - updated_at: string - role_id?: string | null - role_name?: string | null - capabilities?: string[] -} - -export interface WorkspaceRole { - id: string - organization_id: string - name: string - description?: string | null - capabilities: string[] - is_system: boolean - created_at: string - updated_at: string -} - -export interface WorkspaceMember { - id: string - workspace_id: string - user_id: string - role_id: string - role_name: string - user_email: string - user_name?: string | null - added_by_user_id?: string | null - created_at: string -} - -export interface CapabilityInfo { - key: string - label: string -} - -export interface CapabilityDomain { - key: string - label: string - capabilities: CapabilityInfo[] -} - -export interface WorkspaceRoleCreate { - name: string - description?: string | null - capabilities: string[] -} - -export interface WorkspaceRoleUpdate { - name?: string - description?: string | null - capabilities?: string[] -} - -export interface CallImportSourceRowSkip { - source_row: number - reason: string - message: string -} - -export interface CallImport { - id: string - organization_id: string - /** Workspace this import belongs to. */ - workspace_id: string - /** - * Telephony provider key. ``null`` until the IMPORT stage in the - * staged flow (which is the first step that knows the provider). - * Always populated on post-import batches and on legacy one-shot - * uploads. - */ - provider: string | null - telephony_integration_id: string | null - original_filename: string | null - /** - * For Excel uploads, which worksheet this batch came from. ``null`` - * for CSV uploads (CSV files have no sheet concept) and for any - * imports created before multi-sheet support landed. - */ - sheet_name: string | null - /** Optional free-text dataset label (high-level segregation filter). */ - dataset: string | null - /** Tags currently attached to this import. Empty array if untagged. */ - tags: CallImportTag[] - /** - * Reusable Input Parameter schema the batch was uploaded against. - * NULL on legacy batches uploaded before the schema-driven flow shipped. - */ - schema_id: string | null - /** - * Schema-driven mapping: ``{parameter_name: csv_header}``. Empty on - * legacy batches; check ``column_mapping`` / ``extra_columns`` / - * ``custom_column_mapping`` instead for those. - */ - parameter_mapping: Record - /** Legacy free-form mapping kept for batches uploaded before schemas. */ - column_mapping: Record - /** Legacy extra-column list kept for backwards-compat. */ - extra_columns: string[] - /** Legacy uploader-named columns kept for backwards-compat. */ - custom_column_mapping: Record - /** - * Source headers the uploader explicitly skipped, captured at the - * MAP stage. Empty for legacy one-shot uploads where the value was - * ephemeral. - */ - skipped_columns: string[] - /** - * Source rows skipped at parse time (missing/invalid conversation ID or URL). - */ - source_row_skips?: CallImportSourceRowSkip[] - /** S3 key for the staged source file. ``null`` on legacy batches. */ - source_s3_key: string | null - /** ``'csv'`` / ``'xlsx'`` for staged files, or ``'audio'`` for manual uploads. */ - source_format: string | null - source_size_bytes: number | null - source_content_type: string | null - /** - * Snapshot of the file's sheets + headers captured at UPLOAD time so - * the MAP UI can render without re-fetching the source from S3. - * ``null`` on legacy batches. - */ - available_sheets: CallImportPreviewSheet[] | null - total_rows: number - completed_rows: number - failed_rows: number - status: CallImportStatus - error_message: string | null - created_at: string - updated_at: string - created_by_email?: string | null - last_updated_by_email?: string | null -} - -export interface CallImportDetail extends CallImport { - rows: CallImportRow[] - /** - * Total row count *after* applying the optional ``q`` search filter. - * ``null`` when no filter is active — paginate against ``total_rows`` - * in that case. - */ - filtered_total_rows: number | null - /** - * Batch-wide aggregates of ``CallImportRow.diarised_transcript_status``. - * The ``idle`` bucket (rows never touched by the transcribe/diarise - * worker) is implicit: ``total_rows - (pending + running + completed - * + failed)``. Lets the UI render a transcribe-and-diarise progress - * bar without paginating through every row. - */ - diarised_pending_rows: number - diarised_running_rows: number - diarised_completed_rows: number - diarised_failed_rows: number -} - -export interface CallImportListResponse { - items: CallImport[] - total: number - page: number - page_size: number -} - -export interface CallImportUploadResponse { - id: string - total_rows: number - status: CallImportStatus - dataset: string | null - tags: CallImportTag[] - message: string -} - -/** One worksheet (or one CSV file synthesized as a single sheet). */ -export interface CallImportPreviewSheet { - /** Sheet name for xlsx; filename for csv. */ - name: string - /** Column headers from the first non-empty row. */ - headers: string[] - /** Approximate count of data rows (excludes the header row). */ - row_count: number -} - -/** - * Sheets / headers extracted server-side from an uploaded CSV or Excel - * workbook. Drives the modal's column-mapping UI without forcing the - * frontend to parse the file itself. - */ -export interface CallImportPreviewResponse { - /** ``'csv'`` or ``'xlsx'``. */ - format: 'csv' | 'xlsx' - sheets: CallImportPreviewSheet[] -} - -export type MetricSelectionMode = 'single_choice' | 'multi_label' - -export interface CallImportMetricSummary { - id: string - name: string - metric_type: string | null - description: string | null - parent_metric_id?: string | null - selection_mode?: MetricSelectionMode | null - /** Only meaningful on multi_label parents; gates the Discovered - * Labels panel on the Flow tab. Defaults to false. */ - allow_discovery?: boolean -} - -/** Per-metric LLM override (provider+model+optional credential + generation params). */ -export interface CallImportEvaluationLLMOverride { - provider?: string | null - model?: string | null - credential_id?: string | null - llm_config?: LLMGenerationConfig | null -} - -export interface CallImportEvaluation { - id: string - call_import_id: string - organization_id: string - /** User-supplied label for the run; null when not named. */ - name: string | null - selected_metric_ids: string[] - /** parent_id -> [child_id, ...] snapshot captured at run time. */ - selected_metric_groups?: Record | null - metrics: CallImportMetricSummary[] - status: 'pending' | 'running' | 'completed' | 'partial' | 'failed' - total_rows: number - completed_rows: number - failed_rows: number - error_message: string | null - /** Run-level LLM provider chosen by the user (null = legacy default). */ - llm_provider: string | null - llm_model: string | null - llm_credential_id: string | null - llm_config?: LLMGenerationConfig | null - metric_llm_overrides: Record | null - stt_provider: string | null - stt_model: string | null - stt_credential_id: string | null - /** - * Run-level LLM diariser config used when the worker auto-diarises - * rows that are missing a diarised transcript. - */ - diarisation_llm_provider?: string | null - diarisation_llm_model?: string | null - diarisation_llm_credential_id?: string | null - diarisation_prompt?: string | null - /** - * Diarisation pipeline shape this run was created with. - * - `stt_llm` (default) — STT then an LLM diariser over the text. - * - `llm_only` — audio fed directly to a multimodal diariser LLM. - * Surfaced so the retry / re-run UI can preselect the right mode. - */ - transcribe_mode?: 'stt_llm' | 'llm_only' - /** - * Which transcript column this run scored against. - * Defaults to `production` on legacy runs. - */ - transcript_source: CallImportEvaluationTranscriptSource - /** - * Other evaluation ids created in the same Run Evaluation request. - * Populated only on the POST response when the user ticked both - * Production and Diarised. Empty array on all other reads. - */ - sibling_evaluation_ids: string[] - started_at: string | null - finished_at: string | null - created_at: string - updated_at: string - created_by_email?: string | null - last_updated_by_email?: string | null - /** - * Cached LLM-generated TLDR rendered above the Visualizations tab. - * Populated lazily via ``POST /evaluations/{id}/insights``; null on - * runs the user has not summarised yet. - */ - tldr_summary?: EvaluationTldrSummary | null - user_insights?: EvaluationUserInsightsState | null - metric_clusters?: EvaluationMetricClustersState | null - /** - * True when the user opted into top-level metric discovery on the - * Run Evaluation modal. Gates the Discovered metrics panel on the - * evaluation detail Flow tab. - */ - discover_new_metrics?: boolean - /** - * Set while a bulk background operation (abort, force-fail, retry) is - * still running. The UI disables other mutating actions until cleared. - */ - bulk_operation?: 'abort' | 'force_fail_pending' | 'retry' | null -} - -/** - * LLM-generated narrative + bullet patterns for a single evaluation - * run. Cached on the evaluation row so re-opening the Visualizations - * tab doesn't auto-burn LLM tokens. ``is_stale`` is computed by the - * backend at read time when ``completed_rows`` has grown since the - * summary was generated. - */ -export interface EvaluationTldrSummary { - narrative: string - patterns: string[] - metric_insights?: Record - generated_at: string - generated_at_completed_rows: number - provider?: string | null - model?: string | null - is_stale: boolean -} - -export interface UserInsightCategory { - label: string - count: number - share_pct: number -} - -export interface UserInsightEvidenceTurn { - speaker: string - text: string -} - -export interface UserInsightEvidence { - conversation_id?: string | null - quote: string - turns?: UserInsightEvidenceTurn[] -} - -export interface EvaluationUserInsightItem { - id: string - title: string - categories: UserInsightCategory[] - observation: string - evidence: UserInsightEvidence -} - -export interface EvaluationUserInsightsState { - status: 'idle' | 'running' | 'completed' | 'failed' - insights: EvaluationUserInsightItem[] - overview?: string | null - generated_at?: string | null - generated_at_completed_rows: number - progress?: { completed_llm_calls: number; total_llm_calls: number } | null - provider?: string | null - model?: string | null - llm_calls_used: number - max_llm_calls?: number | null - error_message?: string | null - is_stale: boolean -} - -export type MetricClusterGapLabel = - | 'LOGIC_GAP' - | 'UNDERSPEC' - | 'EXISTS_NO_TRIGGER' - | 'MISSING' - -export interface MetricSubCluster { - label: string - count: number - share_pct: number -} - -export interface MetricClusterEvidenceTurn { - speaker: string - text: string -} - -export interface MetricClusterEvidence { - conversation_id?: string | null - evaluation_row_id?: string | null - quote: string - turns?: MetricClusterEvidenceTurn[] -} - -export interface MetricCluster { - id: string - label: string - gap_label: MetricClusterGapLabel - level: number - count: number - share_pct: number - sub_clusters: MetricSubCluster[] - observation: string - failure_reason?: string - evidence: MetricClusterEvidence - is_discovered: boolean -} - -export interface MetricClusterGroup { - metric_id: string - metric_name: string - flagged_count: number - failure_reason?: string - clusters: MetricCluster[] -} - -export interface DiscoveredProblemCluster { - id: string - label: string - gap_label: MetricClusterGapLabel - count: number - share_pct: number - observation: string - failure_reason?: string - evidence: MetricClusterEvidence -} - -export interface RcaRepeatedPatternRow { - metric_id: string - metric_name: string - top_rca_patterns: string - evidence_share_pct: number - evidence_calls: number - evidence_cluster_count?: number - failure_reason: string -} - -export interface RcaMetricHotspotRow { - metric_id: string - metric_name: string - description: string - metric_rate_pct: number - flagged_calls: number -} - -export interface RcaPromptAreaRow { - label: string - share_pct: number - gap_label: MetricClusterGapLabel -} - -export interface MetricClustersRcaSummary { - total_clusters: number - total_clustered_instances: number - total_flagged_instances?: number - analysed_calls: number - repeated_patterns: RcaRepeatedPatternRow[] - metric_hotspots: RcaMetricHotspotRow[] - prompt_areas: RcaPromptAreaRow[] -} - -export interface MetricFailurePolicy { - metric_id: string - failure_values: string[] - failure_child_names?: string[] - numeric_rule?: { op: 'lt' | 'lte' | 'gt' | 'gte'; threshold: number } | null -} - -export interface MetricFailurePolicyValueCount { - label: string - count: number -} - -export interface MetricFailurePolicyMetricPreview { - metric_id: string - metric_name: string - metric_type?: string | null - selection_mode?: string | null - is_multi_label_parent: boolean - value_counts: MetricFailurePolicyValueCount[] - child_names: string[] - row_count_by_value: Record - suggested_policy: MetricFailurePolicy - effective_policy: MetricFailurePolicy -} - -export interface MetricFailurePoliciesResponse { - previews: MetricFailurePolicyMetricPreview[] - policies: Record - source: 'inferred' | 'user' - updated_at?: string | null -} - -export interface MetricClusterEligibleRow { - evaluation_row_id: string - conversation_id?: string | null - row_index?: number | null - flagged_metric_names: string[] -} - -export interface MetricClusterEligibleRowsResponse { - items: MetricClusterEligibleRow[] - total: number -} - -export interface EvaluationMetricClustersState { - status: 'idle' | 'running' | 'completed' | 'failed' | 'cancelled' - groups: MetricClusterGroup[] - discovered_problems: DiscoveredProblemCluster[] - overview?: string | null - generated_at?: string | null - generated_at_completed_rows: number - progress?: { completed_llm_calls: number; total_llm_calls: number } | null - provider?: string | null - model?: string | null - llm_calls_used: number - max_llm_calls?: number | null - error_message?: string | null - is_stale: boolean - selected_evaluation_row_ids?: string[] - failure_policies?: Record - failure_policies_source?: 'inferred' | 'user' - failure_policies_updated_at?: string | null - rca_summary?: MetricClustersRcaSummary | null -} - -export interface AgentFlowNode { - id: string - label: string - node_type: 'start' | 'decision' | 'action' | 'terminal' - position_x?: number | null - position_y?: number | null - prompt_excerpt?: string | null - start_offset?: number | null - end_offset?: number | null -} - -export interface AgentFlowEdge { - source: string - target: string - condition?: string | null -} - -export interface AgentFlowGraph { - nodes: AgentFlowNode[] - edges: AgentFlowEdge[] - generated_at?: string | null - provider?: string | null - model?: string | null - layout_saved_at?: string | null - prompt_content_hash?: string | null - mapping_error?: string | null - generation_error?: string | null -} - -export interface ImportedAgent { - id: string - organization_id: string - name: string - description: string | null - content: string - tags: string[] | null - current_version: number - agent_flowchart?: AgentFlowGraph | null - agent_flowchart_status?: string | null - created_at: string - updated_at: string - created_by: string | null -} - -export interface ImportedAgentDetail extends ImportedAgent { - versions: PromptPartialVersion[] -} - -export interface MetricPartialChild { - name: string - description: string - example: string -} - -export interface MetricPartialContent { - schema_version: 1 - metric_kind: 'single' | 'category' - description: string - children?: MetricPartialChild[] -} - -export interface MetricPartial { - id: string - organization_id: string - name: string - description: string | null - content: string - tags: string[] | null - current_version: number - created_at: string - updated_at: string - created_by: string | null -} - -export interface MetricPartialDetail extends MetricPartial { - versions: PromptPartialVersion[] -} - -export interface PromptPartialVersion { - id: string - prompt_partial_id: string - version: number - content: string - change_summary: string | null - created_at: string - created_by: string | null -} - -export interface PromptImprovementSuggestion { - id: string - metric_id: string - metric_name: string - cluster_id: string - cluster_label: string - gap_label: MetricClusterGapLabel - share_pct: number - priority: 'high' | 'medium' | 'low' - change_type?: 'edit' | 'add' - target_section: string - anchor_excerpt?: string - current_gap: string - suggested_text: string - rationale: string - flow_node_id?: string - flow_node_label?: string -} - -export interface EvaluationPromptImprovementsState { - status: 'idle' | 'running' | 'completed' | 'failed' - imported_agent_id?: string | null - imported_agent_name?: string | null - suggestions: PromptImprovementSuggestion[] - overview?: string | null - generated_at?: string | null - generated_at_completed_rows: number - provider?: string | null - model?: string | null - error_message?: string | null - is_stale: boolean -} - -export interface MetricPeriodDelta { - label: string - detail: string - why?: string | null -} - -export interface CallImportEvaluationListResponse { - items: CallImportEvaluation[] - total: number -} - -export interface CallImportEvaluationBaselineCandidate { - evaluation_id: string - name: string - dataset: string - period_label: string | null - period_start: string | null - period_end: string | null - period_display: string - completed_rows: number - created_at: string - is_default: boolean -} - -export interface CallImportEvaluationBaselineCandidatesResponse { - items: CallImportEvaluationBaselineCandidate[] - default_evaluation_id: string | null -} - -export interface CallImportEvaluationPdfReport { - id: string - filename: string - preview_url?: string | null - download_url?: string | null - created_at: string - created_by?: string | null - report_type: string - vendor_name: string - config_summary?: string | null - storage_available?: boolean - cache_hit?: boolean -} - -export interface CallImportEvaluationPdfReportListItem { - id: string - filename?: string | null - vendor_name: string - report_type: string - created_by?: string | null - created_at: string - config_summary?: string | null - cache_fingerprint?: string | null -} - -export interface CallImportEvaluationPdfReportListResponse { - items: CallImportEvaluationPdfReportListItem[] -} - -export interface CallImportEvaluationRow { - id: string - evaluation_id: string - call_import_row_id: string - row_index: number | null - /** Mandatory identifier from the source batch (renamed from ``external_call_id``). */ - conversation_id: string | null - transcript: string | null - raw_columns: Record | null - recording_url: string | null - recording_date: string | null - /** - * S3 object key for the downloaded recording. Prefer this over - * ``recording_url`` for playback — we resolve it to a presigned URL - * so audio plays from our storage instead of the (often expired) - * provider URL. - */ - recording_s3_key: string | null - diarised_transcript_status?: string | null - diarised_transcript_error?: string | null - status: 'pending' | 'running' | 'completed' | 'failed' | 'skipped' - metric_scores: Record - error_message: string | null - started_at: string | null - finished_at: string | null - created_at: string - updated_at: string -} - -export interface CallImportEvaluationRowListResponse { - items: CallImportEvaluationRow[] - total: number - page: number - page_size: number -} - -// --- Retry (re-enqueue failed rows on an existing evaluation run) --- - -export interface CallImportEvaluationRetryRequest { - /** - * Restrict the retry to a specific subset of evaluation rows. - * When omitted, every row with status='failed' in this run is - * re-enqueued. - */ - eval_row_ids?: string[] - - /** - * Optional LLM overrides. When provided, persisted onto the run so - * future retries default to the new config. ``llm_provider`` and - * ``llm_model`` must be sent together — the backend 400s on - * half-configured input. - */ - llm_provider?: string - llm_model?: string - llm_credential_id?: string | null - - /** - * Optional STT overrides (only meaningful when the run scores the - * diarised transcript). Same paired-field rule as LLM. - */ - stt_provider?: string - stt_model?: string - stt_credential_id?: string | null - - /** - * When true, wipe the diarised transcript on every retried row so - * the (possibly new) STT runs from scratch. Only takes effect for - * diarised runs that have STT config. - */ - transcribe_overwrite?: boolean -} - -export interface CallImportEvaluationRetrySkippedItem { - eval_row_id: string - /** - * Why this row was not re-enqueued. Known values: - * - 'unknown' (id not in this run) - * - 'in_progress' (status is pending/running) - * - 'completed' (already successful) - * - 'source_row_missing' - */ - reason: 'unknown' | 'in_progress' | 'completed' | 'source_row_missing' -} - -export interface CallImportEvaluationRetryResponse { - requeued: number - /** - * Of those, how many were chained through a diarisation task first - * because the diarised transcript was missing. - */ - transcribe_requeued: number - skipped: CallImportEvaluationRetrySkippedItem[] -} - -export interface CallImportEvaluationBulkActionResponse { - accepted: boolean - target_count: number - evaluation_id: string -} - -// --- Diarization / transcription --- - -export interface CallImportTranscribeRequest { - /** - * Diarisation pipeline shape. - * - `stt_llm` (default) — STT produces plain text, then an LLM - * diariser splits it into agent/user turns. STT fields required. - * - `llm_only` — skip STT entirely and feed the audio bytes - * directly to a multimodal `diarization_llm_*` model along with - * `diarization_prompt`. STT fields MUST be omitted in this mode. - */ - mode?: 'stt_llm' | 'llm_only' - /** Required when `mode === 'stt_llm'`; must be null/omitted in `llm_only`. */ - stt_provider?: string | null - /** Required when `mode === 'stt_llm'`; must be null/omitted in `llm_only`. */ - stt_model?: string | null - credential_id?: string | null - language?: string | null - only_missing?: boolean - overwrite_existing?: boolean - row_ids?: string[] - /** - * LLM diariser. In `stt_llm` mode it splits the STT plain-text into - * agent/user turns; in `llm_only` mode it directly receives the - * audio along with `diarization_prompt`. Always required. - */ - diarization_llm_provider: string - diarization_llm_model: string - diarization_llm_credential_id?: string | null - /** - * Operator-supplied system prompt for the diariser LLM. NULL/empty - * means "fall back to the canonical default" (see - * ``getDiarisationDefaultPrompt``). - */ - diarization_prompt?: string | null -} - -export interface CallImportTranscribeResponse { - queued: number - skipped_rows: number - skipped_reason_counts: Record - accepted?: boolean -} - -export interface CallImportRowBulkDeleteResponse { - deleted: number - status?: 'completed' | 'accepted' -} - -export interface CallImportRetryFailedRowsResponse { - requeued: number - enqueue_failed: number - skipped: number -} - -export interface CallImportDiarisationPromptDefaultResponse { - prompt: string -} - -// --- Aggregation / visualization payloads --- - -export interface CallImportMetricHistogramBucket { - x0: number - x1: number - count: number -} - -export interface CallImportMetricValueCount { - label: string - count: number -} - -/** - * One unordered pair-count cell from a multi-label parent's - * co-occurrence matrix. ``a`` and ``b`` are child label names with - * ``a < b`` lexicographically; ``count`` is the number of rows on - * which both labels fired together. - */ -export interface CallImportMetricLabelPair { - a: string - b: string - count: number -} - -export interface CallImportMetricAggregate { - metric_id: string - metric_name: string - metric_type: string | null - metric_category?: 'quality' | 'user_insight' | string - /** - * True when the metric is a multi-label classifier parent. - * ``value_counts`` then lists per-child label tallies and one row - * may contribute to several labels, so the chart layout has to - * ignore the pie toggle (slices wouldn't sum to 100%) and the - * n-badge represents rows scored, not label occurrences. - */ - is_multi_label_parent?: boolean - count: number - skipped_count: number - error_count: number - mean: number | null - median: number | null - p25: number | null - p75: number | null - p95: number | null - min: number | null - max: number | null - stddev: number | null - histogram_buckets: CallImportMetricHistogramBucket[] - value_counts: CallImportMetricValueCount[] - /** - * Pairwise label intersections for multi-label parent metrics. - * Empty for everything else. The Visualizations tab reconstructs - * a square symmetric matrix from these unordered pairs to render - * the co-occurrence heatmap chart type. - */ - co_occurrence?: CallImportMetricLabelPair[] -} - -export interface CallImportEvaluationAggregateResponse { - evaluation_id: string - total_rows: number - completed_rows: number - failed_rows: number - metrics: CallImportMetricAggregate[] - period_deltas?: Record - baseline_evaluation_id?: string | null - failure_policies_source?: 'inferred' | 'user' | null -} - -export interface EvaluatorResultsAggregateResponse { - scope: string - suite_id?: string | null - agent_id?: string | null - scenario_id?: string | null - total_rows: number - completed_rows: number - failed_rows: number - metrics: CallImportMetricAggregate[] -} - -export interface CallImportInsightsRunPoint { - evaluation_id: string - name: string | null - created_at: string - mean: number | null - completed_rows: number -} - -export interface CallImportInsightsMetric { - metric_id: string - metric_name: string - metric_type: string | null - latest: CallImportMetricAggregate | null - trend: CallImportInsightsRunPoint[] -} - -export interface CallImportInsightsResponse { - call_import_id: string - total_rows: number - rows_with_transcript: number - rows_without_transcript: number - transcript_source_counts: Record - evaluation_count: number - metrics: CallImportInsightsMetric[] -} - -// --- Metrics hierarchy + flow visualization --- - -export interface MetricSummary { - id: string - organization_id: string - name: string - description: string | null - metric_type: string - metric_category?: 'quality' | 'user_insight' | string - trigger: string - enabled: boolean - is_default: boolean - metric_origin: string - supported_surfaces: string[] - enabled_surfaces: string[] - custom_data_type: string | null - custom_config: Record | null - tags: string[] | null - capture_rationale: boolean - parent_metric_id: string | null - selection_mode: MetricSelectionMode | null - allow_discovery?: boolean - /** - * When true, this metric is a "transcript-compare judge": at - * call-import evaluation time the worker feeds BOTH the production - * transcript and the diarised transcript to the LLM as a labeled - * pair, and the run's transcript_source toggle is ignored for this - * metric. Mutually exclusive with parent_metric_id and selection_mode - * — comparison metrics stay standalone. - */ - compare_transcripts?: boolean - children?: MetricSummary[] - created_at: string - updated_at: string - created_by: string | null -} - -export interface MetricChildDraft { - name: string - description?: string | null - enabled?: boolean - capture_rationale?: boolean | null - tags?: string[] | null -} - -export interface MetricCreateWithChildrenPayload { - name: string - description?: string | null - selection_mode: MetricSelectionMode - enabled?: boolean - supported_surfaces?: string[] - enabled_surfaces?: string[] - tags?: string[] | null - allow_discovery?: boolean - children: MetricChildDraft[] -} - -export interface MetricFlowNode { - id: string - label: string - count: number - is_terminal: boolean - is_discovered?: boolean -} - -export interface MetricFlowEdge { - source: string - target: string - count: number -} - -export interface MetricFlowResponse { - parent_metric_id: string - parent_metric_name: string - selection_mode: MetricSelectionMode | null - nodes: MetricFlowNode[] - edges: MetricFlowEdge[] - total_rows: number - rows_with_sequence: number -} - -export interface DiscoveredLabel { - key: string - name: string - description?: string | null - sample_rationale?: string | null - /** - * Up to 3 distinct LLM rationales captured for this candidate - * across rows. The Discovered Labels promote flow surfaces the - * first 2 as an ``Examples:`` block on the new sub-metric's - * rubric so the user starts with concrete cases in the prompt. - */ - examples?: string[] - count: number -} - -export interface DiscoveredLabelsResponse { - parent_metric_id: string - items: DiscoveredLabel[] -} - -/** - * One LLM-discovered candidate TOP-LEVEL metric aggregated across all - * rows of an evaluation. Mirrors :class:`DiscoveredLabel` but adds a - * ``suggested_type`` field — the LLM's guess at the best shape for - * the new metric — that the promote modal can pre-fill the type radio - * with. - */ -export interface DiscoveredMetric { - key: string - name: string - description?: string | null - suggested_type: 'boolean' | 'rating' | 'category' - sample_rationale?: string | null - examples?: string[] - count: number -} - -export interface DiscoveredMetricsResponse { - evaluation_id: string - items: DiscoveredMetric[] -} - -export interface ObservabilityCallAgent { - id: string - agent_id?: string | null - name: string -} - -export interface ObservabilityCallData { - startedAt?: string - started_at?: string - endedAt?: string - ended_at?: string - from_phone_number?: string - to_phone_number?: string - endedReason?: string - recording_s3_key?: string - recording_url?: string - duration_seconds?: number - agent_name?: string - _agent_ref?: string | number - direction?: string - messages?: Array<{ role: string; content: string; start_time?: number; end_time?: number }> - live_transcript?: Array<{ role: string; content: string; timestamp?: string; start_time?: number }> - metadata?: Record - call_short_id?: string -} - -export interface ObservabilityCall { - id: string - call_short_id: string - status?: string | null - call_event?: string | null - is_live?: boolean - direction?: string | null - source?: string | null - provider_platform?: string | null - provider_call_id?: string | null - agent_id?: string | null - agent?: ObservabilityCallAgent | null - created_at?: string | null - updated_at?: string | null - call_data?: ObservabilityCallData | null - live_transcript?: Array<{ role: string; content: string; timestamp?: string }> -} +// API Types matching the backend schemas + +export type { LLMGenerationConfig } from '../config/llmGenerationParams' +import type { LLMGenerationConfig } from '../config/llmGenerationParams' + +export enum EvaluationType { + ASR = 'asr', + TTS = 'tts', +} + +export enum EvaluationStatus { + PENDING = 'pending', + PROCESSING = 'processing', + COMPLETED = 'completed', + FAILED = 'failed', + CANCELLED = 'cancelled', +} + +export interface AudioFile { + id: string + filename: string + format: string + file_size: number + duration?: number | null + sample_rate?: number | null + channels?: number | null + uploaded_at: string +} + +export interface Evaluation { + id: string + audio_id: string + reference_text?: string | null + evaluation_type: EvaluationType + model_name?: string | null + status: EvaluationStatus + metrics_requested?: string[] | null + created_at: string + started_at?: string | null + completed_at?: string | null + error_message?: string | null +} + +export interface DashboardSummary { + evaluations: { + total: number + completed: number + pending: number + failed: number + } + resources: { + agents: number + personas: number + scenarios: number + integrations: number + voice_bundles: number + } + setup_progress: { + has_integration: boolean + has_voice_bundle: boolean + has_agent: boolean + has_evaluation: boolean + } + metrics: { + total: number + enabled: number + } + call_imports: { + total: number + } + call_import_evaluations: { + total: number + completed: number + running: number + failed: number + } + recent_evaluations: Evaluation[] +} + +export interface ModelConfigEntry { + provider: string + model_type: string + description?: string + featured?: boolean + featured_rank?: number + highlights?: string[] +} + +export interface EvaluationCreate { + audio_id: string + reference_text?: string | null + evaluation_type: EvaluationType + model_name?: string | null + metrics?: string[] +} + +export interface EvaluationResult { + evaluation_id: string + status: EvaluationStatus + transcript?: string | null + metrics: Record + processing_time?: number | null + model_used?: string | null + created_at: string +} + +export interface BatchEvaluationResult { + processed_files: number + failed_files: number + aggregated_metrics?: Record | null + individual_results: EvaluationResult[] +} + +/** Voice agent evaluator run (evaluator_results table). */ +export type EvaluatorResultStatus = + | 'queued' + | 'call_initiating' + | 'call_connecting' + | 'call_in_progress' + | 'call_ended' + | 'transcribing' + | 'evaluating' + | 'fetching_details' + | 'completed' + | 'failed' + +export interface EvaluatorResultMetricScore { + value: unknown + type: string + metric_name: string + parent_metric_id?: string | null +} + +export interface EvaluatorResultRow { + id: string + result_id: string + name: string | null + evaluator_id: string | null + agent_id?: string | null + persona_id?: string | null + scenario_id?: string | null + suite_id?: string | null + timestamp: string + duration_seconds: number | null + status: EvaluatorResultStatus + metric_scores: Record | null + error_message: string | null + agent?: { id: string; name: string } | null + scenario?: { id: string; name: string } | null +} + +export interface EvaluatorResultListResponse { + items: EvaluatorResultRow[] + total: number +} + +export interface EvaluatorResultCounts { + total: number + completed: number + failed: number + in_progress: number + last_run_at?: string | null +} + +export interface EvaluatorResultsScenarioSummary { + scenario_id: string + scenario_name: string + counts: EvaluatorResultCounts +} + +export interface EvaluatorResultsSuiteSummary { + suite_id: string + suite_name?: string | null + agent_id: string + persona_id?: string | null + counts: EvaluatorResultCounts + scenarios?: EvaluatorResultsScenarioSummary[] | null +} + +export interface EvaluatorResultsAgentSummary { + agent_id: string + agent_name: string + counts: EvaluatorResultCounts + suites?: EvaluatorResultsSuiteSummary[] | null +} + +export interface EvaluatorResultsOverviewResponse { + workspace_counts: EvaluatorResultCounts + agents: EvaluatorResultsAgentSummary[] + unassigned: { + counts: EvaluatorResultCounts + recent_result_ids: string[] + } +} + +export interface ListEvaluatorResultsParams { + skip?: number + limit?: number + evaluatorId?: string + agentId?: string + suiteId?: string + scenarioId?: string + status?: 'completed' | 'failed' | 'in_progress' + unassignedOnly?: boolean + playground?: boolean + testAgentsOnly?: boolean +} + +export interface APIKey { + id: string + key: string + name?: string | null + is_active: boolean + created_at: string + last_used?: string | null + message?: string +} + +export interface MessageResponse { + message: string +} + +// IAM & User Types +export enum Role { + READER = 'reader', + WRITER = 'writer', + ADMIN = 'admin', +} + +export enum InvitationStatus { + PENDING = 'pending', + ACCEPTED = 'accepted', + DECLINED = 'declined', + EXPIRED = 'expired', +} + +export interface User { + id: string + email: string + name?: string | null + is_active: boolean + created_at: string +} + +export interface OrganizationMember { + id: string + user_id: string + organization_id: string + role: Role + joined_at: string + user: User +} + +export interface Invitation { + id: string + organization_id: string + email: string + role: Role + status: InvitationStatus + expires_at: string + created_at: string + organization_name?: string | null +} + +export interface InvitationCreate { + email: string + role: Role +} + +export interface RoleUpdate { + role: Role +} + +export interface Profile { + id: string + email: string + name?: string | null + first_name?: string | null + last_name?: string | null + created_at: string + organizations: Array<{ + id: string + name: string + role: string + joined_at: string + }> +} + +export interface UserUpdate { + name?: string | null + first_name?: string | null + last_name?: string | null + email?: string | null +} + +export interface UserPreferences { + theme?: string + notifications_enabled?: boolean + email_notifications?: boolean + default_language?: string + [key: string]: any +} + +export interface UserPreferencesUpdate { + theme?: string + notifications_enabled?: boolean + email_notifications?: boolean + default_language?: string + [key: string]: any +} + +// Integration Types +export enum IntegrationPlatform { + RETELL = 'retell', + VAPI = 'vapi', + CARTESIA = 'cartesia', + ELEVENLABS = 'elevenlabs', + DEEPGRAM = 'deepgram', + MURF = 'murf', + SARVAM = 'sarvam', + VOICEMAKER = 'voicemaker', + SMALLEST = 'smallest', +} + +export enum TelephonyProvider { + PLIVO = 'plivo', + EXOTEL = 'exotel', + VOBIZ = 'vobiz', +} + +export type CredentialRoutingMode = 'inherit' | 'gateway' | 'direct' +export type GatewayInterfaceMode = 'inherit' | 'litellm_shim' | 'native_openai' + +export type EffectiveCredentialRouting = + | 'inherit' + | 'direct' + | 'gateway' + | 'bifrost' + | 'litellm_proxy' + +export interface Integration { + id: string + organization_id: string + platform: IntegrationPlatform + name?: string | null + public_key?: string | null + is_active: boolean + /** True if this row is the default credential for (org, platform). */ + is_default?: boolean + routing_mode?: CredentialRoutingMode + effective_routing?: EffectiveCredentialRouting + created_at: string + updated_at: string + last_tested_at?: string | null +} + +export interface IntegrationCreate { + platform: IntegrationPlatform + api_key: string + public_key?: string + name?: string | null + routing_mode?: CredentialRoutingMode + /** Mark the new credential as the default for (org, platform). */ + is_default?: boolean +} + +// VoiceBundle Types +export enum ModelProvider { + OPENAI = 'openai', + ANTHROPIC = 'anthropic', + GOOGLE = 'google', + XAI = 'xai', + FIREWORKS = 'fireworks', + COHERE = 'cohere', + MISTRAL = 'mistral', + META = 'meta', + TOGETHER = 'together', + PERPLEXITY = 'perplexity', + AZURE = 'azure', + AWS = 'aws', + DEEPGRAM = 'deepgram', + CARTESIA = 'cartesia', + ELEVENLABS = 'elevenlabs', + MURF = 'murf', + CUSTOM = 'custom', + SARVAM = 'sarvam', + VOICEMAKER = 'voicemaker', + SMALLEST = 'smallest', +} + +// AI Provider Types +export interface AIProvider { + id: string + provider: ModelProvider + api_key?: string | null + name?: string | null + endpoint_url?: string | null + is_active: boolean + /** True if this row is the default credential for (org, provider). */ + is_default?: boolean + routing_mode?: CredentialRoutingMode + gateway_model?: string | null + gateway_interface?: GatewayInterfaceMode + gateway_base_url?: string | null + gateway_auth_header?: string | null + gateway_auth_secret_env?: string | null + has_gateway_auth_secret?: boolean + gateway_extra_headers?: Record | null + /** True when provider secrets are resolved by the Bifrost gateway. */ + gateway_managed?: boolean + effective_routing?: EffectiveCredentialRouting + effective_gateway_interface?: 'litellm_shim' | 'native_openai' + created_at: string + updated_at: string + last_tested_at?: string | null +} + +export interface AIProviderCreate { + provider: ModelProvider + api_key?: string | null + name?: string | null + endpoint_url?: string | null + routing_mode?: CredentialRoutingMode + gateway_model?: string | null + gateway_interface?: GatewayInterfaceMode + gateway_base_url?: string | null + gateway_auth_header?: string | null + gateway_auth_secret_env?: string | null + gateway_auth_secret?: string | null + gateway_extra_headers?: Record | null + /** Mark the new credential as the default for (org, provider). */ + is_default?: boolean +} + +export interface AIProviderUpdate { + api_key?: string | null + name?: string | null + endpoint_url?: string | null + is_active?: boolean + routing_mode?: CredentialRoutingMode + gateway_model?: string | null + gateway_interface?: GatewayInterfaceMode + gateway_base_url?: string | null + gateway_auth_header?: string | null + gateway_auth_secret_env?: string | null + gateway_auth_secret?: string | null + clear_gateway_auth_secret?: boolean + gateway_extra_headers?: Record | null +} + +export enum VoiceBundleType { + STT_LLM_TTS = 'stt_llm_tts', + S2S = 's2s', +} + +export interface VoiceBundle { + id: string + name: string + description?: string | null + bundle_type: VoiceBundleType + stt_provider?: ModelProvider | null + stt_model?: string | null + /** + * Optional explicit AIProvider/Integration row id for STT. When null the + * runtime resolver picks the default credential for stt_provider. + */ + stt_credential_id?: string | null + llm_provider?: ModelProvider | null + llm_model?: string | null + llm_temperature?: number | null + llm_max_tokens?: number | null + llm_config?: Record | null + llm_credential_id?: string | null + tts_provider?: ModelProvider | null + tts_model?: string | null + tts_voice?: string | null + tts_config?: Record | null + tts_credential_id?: string | null + s2s_provider?: ModelProvider | null + s2s_model?: string | null + s2s_config?: Record | null + s2s_credential_id?: string | null + extra_metadata?: Record | null + is_active: boolean + created_at: string + updated_at: string + created_by?: string | null +} + +export interface VoiceBundleCreate { + name: string + description?: string | null + bundle_type?: VoiceBundleType + stt_provider?: ModelProvider | null + stt_model?: string | null + stt_credential_id?: string | null + llm_provider?: ModelProvider | null + llm_model?: string | null + llm_temperature?: number | null + llm_max_tokens?: number | null + llm_config?: Record | null + llm_credential_id?: string | null + tts_provider?: ModelProvider | null + tts_model?: string | null + tts_voice?: string | null + tts_config?: Record | null + tts_credential_id?: string | null + s2s_provider?: ModelProvider | null + s2s_model?: string | null + s2s_config?: Record | null + s2s_credential_id?: string | null + extra_metadata?: Record | null +} + +// Test Agent Types +export interface AgentPhoneAssignmentConflict { + agent_id: string + agent_name: string + phone_number: string +} + +export interface AgentPhoneAssignmentCheckResponse { + available: boolean + phone_number?: string | null + conflict?: AgentPhoneAssignmentConflict | null +} + +export interface TestAgent { + id: string + agent_id?: string | null + name: string + phone_number?: string | null + telephony_phone_number_id?: string | null + language: string + description: string | null + prompt_variables?: Record | null + silence_hangup_secs?: number + call_type: string + call_medium: string + voice_bundle_id?: string | null + voice_ai_integration_id?: string | null + voice_ai_agent_id?: string | null + provider_prompt?: string | null + provider_prompt_synced_at?: string | null + created_at: string + updated_at: string +} + +// Test Agent Conversation Types +export interface TestAgentConversation { + id: string + organization_id: string + agent_id: string + persona_id: string + scenario_id: string + voice_bundle_id: string + status: string + live_transcription?: Array<{ + speaker: string + text: string + timestamp: number + audio_segment_key?: string + }> | null + conversation_audio_key?: string | null + full_transcript?: string | null + started_at: string + ended_at?: string | null + duration_seconds?: number | null + conversation_metadata?: Record | null + created_at: string + updated_at: string + created_by?: string | null +} + +export interface TestAgentConversationCreate { + agent_id: string + persona_id: string + scenario_id: string + voice_bundle_id: string + conversation_metadata?: Record | null +} + +export interface TestAgentConversationUpdate { + status?: string | null + live_transcription?: Array> | null + full_transcript?: string | null + conversation_metadata?: Record | null +} + +export interface VoiceBundleUpdate { + name?: string + description?: string | null + stt_provider?: ModelProvider + stt_model?: string + stt_credential_id?: string | null + llm_provider?: ModelProvider + llm_model?: string + llm_temperature?: number | null + llm_max_tokens?: number | null + llm_config?: Record | null + llm_credential_id?: string | null + tts_provider?: ModelProvider + tts_model?: string + tts_voice?: string | null + tts_config?: Record | null + tts_credential_id?: string | null + s2s_provider?: ModelProvider | null + s2s_model?: string | null + s2s_config?: Record | null + s2s_credential_id?: string | null + extra_metadata?: Record | null + is_active?: boolean +} + +// Data Sources Types +export interface S3ConnectionTest { + bucket_name: string + region?: string + access_key_id: string + secret_access_key: string + endpoint_url?: string | null +} + +export interface S3ConnectionTestResponse { + success: boolean + message: string + bucket_name?: string | null +} + +export interface S3FileInfo { + key: string + filename: string + size: number + last_modified: string +} + +export interface S3FolderInfo { + name: string + path: string +} + +export interface S3ListFilesResponse { + files: S3FileInfo[] + total: number + prefix?: string | null +} + +export interface S3BrowseResponse { + folders: S3FolderInfo[] + files: S3FileInfo[] + current_path: string + organization_id: string +} + +export interface S3Status { + enabled: boolean + provider?: 's3' | 'gcs' | string + error?: string | null +} + +// Alert Types +export enum AlertMetricType { + NUMBER_OF_CALLS = 'number_of_calls', + CALL_DURATION = 'call_duration', + ERROR_RATE = 'error_rate', + SUCCESS_RATE = 'success_rate', + LATENCY = 'latency', + CUSTOM = 'custom', +} + +export enum AlertAggregation { + SUM = 'sum', + AVG = 'avg', + COUNT = 'count', + MIN = 'min', + MAX = 'max', +} + +export enum AlertOperator { + GREATER_THAN = '>', + LESS_THAN = '<', + GREATER_THAN_OR_EQUAL = '>=', + LESS_THAN_OR_EQUAL = '<=', + EQUAL = '=', + NOT_EQUAL = '!=', +} + +export enum AlertNotifyFrequency { + IMMEDIATE = 'immediate', + HOURLY = 'hourly', + DAILY = 'daily', + WEEKLY = 'weekly', +} + +export enum AlertStatus { + ACTIVE = 'active', + PAUSED = 'paused', + DISABLED = 'disabled', +} + +export enum AlertHistoryStatus { + TRIGGERED = 'triggered', + NOTIFIED = 'notified', + ACKNOWLEDGED = 'acknowledged', + RESOLVED = 'resolved', +} + +export interface Alert { + id: string + organization_id: string + name: string + description?: string | null + metric_type: AlertMetricType + aggregation: AlertAggregation + operator: AlertOperator + threshold_value: number + time_window_minutes: number + agent_ids?: string[] | null + notify_frequency: AlertNotifyFrequency + notify_emails?: string[] | null + notify_webhooks?: string[] | null + status: AlertStatus + created_at: string + updated_at: string + created_by?: string | null +} + +export interface AlertCreate { + name: string + description?: string | null + metric_type?: AlertMetricType + aggregation?: AlertAggregation + operator?: AlertOperator + threshold_value: number + time_window_minutes?: number + agent_ids?: string[] | null + notify_frequency?: AlertNotifyFrequency + notify_emails?: string[] + notify_webhooks?: string[] +} + +export interface AlertUpdate { + name?: string + description?: string | null + metric_type?: AlertMetricType + aggregation?: AlertAggregation + operator?: AlertOperator + threshold_value?: number + time_window_minutes?: number + agent_ids?: string[] | null + notify_frequency?: AlertNotifyFrequency + notify_emails?: string[] + notify_webhooks?: string[] + status?: AlertStatus +} + +export interface AlertHistoryItem { + id: string + organization_id: string + alert_id: string + triggered_at: string + triggered_value: number + threshold_value: number + status: AlertHistoryStatus + notified_at?: string | null + notification_details?: Record | null + acknowledged_at?: string | null + acknowledged_by?: string | null + resolved_at?: string | null + resolved_by?: string | null + resolution_notes?: string | null + context_data?: Record | null + created_at: string + updated_at: string + alert?: Alert +} + + +// Cron Job Types +export enum CronJobStatus { + ACTIVE = 'active', + PAUSED = 'paused', + COMPLETED = 'completed', +} + +export interface CronJob { + id: string + organization_id: string + name: string + cron_expression: string + timezone: string + max_runs: number + current_runs: number + evaluator_ids: string[] + status: CronJobStatus + next_run_at?: string | null + last_run_at?: string | null + created_at: string + updated_at: string + created_by?: string | null +} + +export interface CronJobCreate { + name: string + cron_expression: string + timezone: string + max_runs: number + evaluator_ids: string[] +} + +export interface CronJobUpdate { + name?: string + cron_expression?: string + timezone?: string + max_runs?: number + evaluator_ids?: string[] + status?: CronJobStatus +} + +// --- Call Imports --- + +/** + * Lifecycle for a call-import batch. + * + * - ``uploaded`` : file landed in S3, no mapping yet. + * - ``mapped`` : user picked a schema + sheet + column mapping; no + * rows materialised yet, no worker enqueued. + * - ``processing`` : rows materialised + workers enqueued. + * - ``pending`` : transient state used by the legacy one-shot + * ``POST /upload`` endpoint just before transitioning + * to ``processing``. + */ +export type CallImportStatus = + | 'pending' + | 'uploaded' + | 'mapped' + | 'processing' + | 'completed' + | 'partial' + | 'failed' + | 'deleting' + +export type CallImportRowStatus = + | 'pending' + | 'processing' + | 'completed' + | 'failed' + +/** Where the value in `transcript` came from. */ +export type CallImportTranscriptSource = + | 'csv' + | 'transcribed' + | 'edited' + | null +/** Lifecycle status for the post-hoc transcription workflow itself. */ +export type CallImportTranscriptStatus = + | 'idle' + | 'pending' + | 'running' + | 'completed' + | 'failed' + | null + +/** + * Which transcript an evaluation run scored against. + * - `production`: the CSV-supplied value on `CallImportRow.transcript`. + * - `diarised`: the worker-produced value on `CallImportRow.diarised_transcript`. + */ +export type CallImportEvaluationTranscriptSource = 'production' | 'diarised' + +/** + * One contiguous turn inside ``CallImportRow.diarised_segments``. + * + * The diarisation worker rewrites each pyannote ``Speaker N`` label + * into ``agent`` / ``user`` (first speaker = agent heuristic). Anything + * beyond two distinct speakers keeps a generic ``speaker_N`` label so + * multi-party recordings still render every voice. + */ +export interface CallImportDiarisedSegment { + speaker: string + text: string + start: number + end: number + /** Original pyannote label (``Speaker 1`` / ``Speaker 2`` / ...). */ + raw_speaker: string +} + +export interface CallImportRow { + id: string + row_index: number + /** Mandatory identifier per row. Renamed from ``external_call_id``. */ + conversation_id: string + recording_url: string | null + recording_date: string | null + /** Production transcript ΓÇö the value supplied via the CSV upload. */ + transcript: string | null + /** Provenance of the stored production transcript (csv = CSV upload, edited = manual edit). */ + transcript_source: CallImportTranscriptSource + /** Legacy: provider recorded by the original transcription worker before the split. */ + transcript_provider: string | null + transcript_model: string | null + transcript_status: CallImportTranscriptStatus + transcript_error: string | null + transcribed_at: string | null + /** Diarised transcript ΓÇö produced by the post-hoc diarisation worker. */ + diarised_transcript: string | null + /** Provider used by the diarisation worker (e.g. "deepgram"). */ + diarised_transcript_provider: string | null + diarised_transcript_model: string | null + diarised_transcript_status: CallImportTranscriptStatus + diarised_transcript_error: string | null + diarised_at: string | null + /** + * Structured speaker turns produced by the diarisation worker. Each + * entry is a single contiguous turn shaped as + * `{ speaker: 'agent' | 'user' | 'speaker_N', text, start, end, + * raw_speaker }`. ``diarised_transcript`` is a rendered + * `: ` view of this list with + * ``diarised_speaker_swap`` applied. ``null`` on legacy rows that + * were diarised before structured turns were persisted (or when the + * STT provider didn't surface segments). + */ + diarised_segments: CallImportDiarisedSegment[] | null + /** + * When ``true`` the ``agent`` <-> ``user`` mapping inside + * ``diarised_segments`` is inverted in the rendered transcript / + * CSV export. The worker writes the canonical mapping using a + * "first speaker is the agent" heuristic; reviewers can flip the + * toggle from the row detail panel without re-running diarisation. + */ + diarised_speaker_swap: boolean + /** + * LLM that turned the STT plain-text output into structured + * ``diarised_segments``. NULL on legacy rows (pre-LLM-diariser). + */ + diarised_llm_provider: string | null + diarised_llm_model: string | null + /** + * Exact prompt the LLM diariser ran with. Useful for the modal to + * pre-fill its textarea when the operator wants to iterate on a + * previously-diarised row. + */ + diarised_prompt: string | null + /** + * Diarisation pipeline that produced this row's turns. + * - `stt_llm` (default) ΓÇö two-stage STT then LLM diariser. + * - `llm_only` ΓÇö single-stage multimodal LLM (audio in). + * Read-only; written by the worker on each diarisation. + */ + transcribe_mode?: 'stt_llm' | 'llm_only' + /** + * Per-row preservation of the mapped source cells. Values land here + * as whatever type the schema parameter coerced them to ΓÇö + * strings (text / url / conversation_id / recording_url / + * recording_date / transcript / datetime), numbers, booleans, or + * ``null`` for blanks. Always + * coerce with ``String(value)`` before string operations. + */ + raw_columns: Record | null + status: CallImportRowStatus + recording_s3_key: string | null + recording_content_type: string | null + recording_size_bytes: number | null + error_message: string | null + attempts: number + created_at: string + updated_at: string +} + +export interface CallImportTag { + id: string + name: string + color: string | null + created_at: string + updated_at: string +} + +/** + * Parameter type tag on a Call Import schema parameter. + * + * - ``conversation_id``: mandatory identifier (one per schema). + * - ``recording_url``: optional; feeds ``CallImportRow.recording_url``. + * - ``recording_date``: date-only call recording date used for reports. + * - ``transcript``: feeds ``CallImportRow.transcript``. + * - ``text`` / ``number`` / ``boolean`` / ``datetime`` / ``url``: + * generic typed fields preserved per row in ``raw_columns`` and + * surfaced in the evaluation export under the parameter's name. + */ +export type CallImportSchemaParameterType = + | 'conversation_id' + | 'recording_url' + | 'recording_date' + | 'transcript' + | 'text' + | 'number' + | 'boolean' + | 'datetime' + | 'url' + +export interface CallImportSchemaParameter { + id?: string + name: string + type: CallImportSchemaParameterType + description: string | null + is_required: boolean + ordering?: number +} + +export interface CallImportSchema { + id: string + organization_id: string + workspace_id: string + name: string + description: string | null + parameters: CallImportSchemaParameter[] + /** How many CallImport batches reference this schema. */ + usage_count: number + created_at: string + updated_at: string +} + +export interface CallImportSchemaListResponse { + items: CallImportSchema[] + total: number +} + +export interface CallImportSchemaCreate { + name: string + description?: string | null + parameters: Array> +} + +export interface CallImportSchemaUpdate { + name?: string + description?: string | null + parameters?: Array> +} + +/** + * In-org Workspace - the active workspace scopes call imports and + * metrics in the UI. The org's Default workspace is auto-seeded by + * migration 033 and cannot be deleted. + */ +export interface Workspace { + id: string + organization_id: string + name: string + slug: string + is_default: boolean + is_active: boolean + created_at: string + updated_at: string + role_id?: string | null + role_name?: string | null + capabilities?: string[] +} + +export interface WorkspaceRole { + id: string + organization_id: string + name: string + description?: string | null + capabilities: string[] + is_system: boolean + created_at: string + updated_at: string +} + +export interface WorkspaceMember { + id: string + workspace_id: string + user_id: string + role_id: string + role_name: string + user_email: string + user_name?: string | null + added_by_user_id?: string | null + created_at: string +} + +export interface CapabilityInfo { + key: string + label: string +} + +export interface CapabilityDomain { + key: string + label: string + capabilities: CapabilityInfo[] +} + +export interface WorkspaceRoleCreate { + name: string + description?: string | null + capabilities: string[] +} + +export interface WorkspaceRoleUpdate { + name?: string + description?: string | null + capabilities?: string[] +} + +export interface CallImportSourceRowSkip { + source_row: number + reason: string + message: string +} + +export interface CallImport { + id: string + organization_id: string + /** Workspace this import belongs to. */ + workspace_id: string + /** + * Telephony provider key. ``null`` until the IMPORT stage in the + * staged flow (which is the first step that knows the provider). + * Always populated on post-import batches and on legacy one-shot + * uploads. + */ + provider: string | null + telephony_integration_id: string | null + original_filename: string | null + /** + * For Excel uploads, which worksheet this batch came from. ``null`` + * for CSV uploads (CSV files have no sheet concept) and for any + * imports created before multi-sheet support landed. + */ + sheet_name: string | null + /** Optional free-text dataset label (high-level segregation filter). */ + dataset: string | null + /** Tags currently attached to this import. Empty array if untagged. */ + tags: CallImportTag[] + /** + * Reusable Input Parameter schema the batch was uploaded against. + * NULL on legacy batches uploaded before the schema-driven flow shipped. + */ + schema_id: string | null + /** + * Schema-driven mapping: ``{parameter_name: csv_header}``. Empty on + * legacy batches; check ``column_mapping`` / ``extra_columns`` / + * ``custom_column_mapping`` instead for those. + */ + parameter_mapping: Record + /** Legacy free-form mapping kept for batches uploaded before schemas. */ + column_mapping: Record + /** Legacy extra-column list kept for backwards-compat. */ + extra_columns: string[] + /** Legacy uploader-named columns kept for backwards-compat. */ + custom_column_mapping: Record + /** + * Source headers the uploader explicitly skipped, captured at the + * MAP stage. Empty for legacy one-shot uploads where the value was + * ephemeral. + */ + skipped_columns: string[] + /** + * Source rows skipped at parse time (missing/invalid conversation ID or URL). + */ + source_row_skips?: CallImportSourceRowSkip[] + /** S3 key for the staged source file. ``null`` on legacy batches. */ + source_s3_key: string | null + /** ``'csv'`` / ``'xlsx'`` for staged files, or ``'audio'`` for manual uploads. */ + source_format: string | null + source_size_bytes: number | null + source_content_type: string | null + /** + * Snapshot of the file's sheets + headers captured at UPLOAD time so + * the MAP UI can render without re-fetching the source from S3. + * ``null`` on legacy batches. + */ + available_sheets: CallImportPreviewSheet[] | null + total_rows: number + completed_rows: number + failed_rows: number + status: CallImportStatus + error_message: string | null + created_at: string + updated_at: string + created_by_email?: string | null + last_updated_by_email?: string | null +} + +export interface CallImportDetail extends CallImport { + rows: CallImportRow[] + /** + * Total row count *after* applying the optional ``q`` search filter. + * ``null`` when no filter is active ΓÇö paginate against ``total_rows`` + * in that case. + */ + filtered_total_rows: number | null + /** + * Batch-wide aggregates of ``CallImportRow.diarised_transcript_status``. + * The ``idle`` bucket (rows never touched by the transcribe/diarise + * worker) is implicit: ``total_rows - (pending + running + completed + * + failed)``. Lets the UI render a transcribe-and-diarise progress + * bar without paginating through every row. + */ + diarised_pending_rows: number + diarised_running_rows: number + diarised_completed_rows: number + diarised_failed_rows: number +} + +export interface CallImportListResponse { + items: CallImport[] + total: number + page: number + page_size: number +} + +export interface CallImportUploadResponse { + id: string + total_rows: number + status: CallImportStatus + dataset: string | null + tags: CallImportTag[] + message: string +} + +/** One worksheet (or one CSV file synthesized as a single sheet). */ +export interface CallImportPreviewSheet { + /** Sheet name for xlsx; filename for csv. */ + name: string + /** Column headers from the first non-empty row. */ + headers: string[] + /** Approximate count of data rows (excludes the header row). */ + row_count: number +} + +/** + * Sheets / headers extracted server-side from an uploaded CSV or Excel + * workbook. Drives the modal's column-mapping UI without forcing the + * frontend to parse the file itself. + */ +export interface CallImportPreviewResponse { + /** ``'csv'`` or ``'xlsx'``. */ + format: 'csv' | 'xlsx' + sheets: CallImportPreviewSheet[] +} + +export type MetricSelectionMode = 'single_choice' | 'multi_label' + +export interface CallImportMetricSummary { + id: string + name: string + metric_type: string | null + description: string | null + parent_metric_id?: string | null + selection_mode?: MetricSelectionMode | null + /** Only meaningful on multi_label parents; gates the Discovered + * Labels panel on the Flow tab. Defaults to false. */ + allow_discovery?: boolean +} + +/** Per-metric LLM override (provider+model+optional credential + generation params). */ +export interface CallImportEvaluationLLMOverride { + provider?: string | null + model?: string | null + credential_id?: string | null + llm_config?: LLMGenerationConfig | null +} + +export interface CallImportEvaluation { + id: string + call_import_id: string + organization_id: string + /** User-supplied label for the run; null when not named. */ + name: string | null + selected_metric_ids: string[] + /** parent_id -> [child_id, ...] snapshot captured at run time. */ + selected_metric_groups?: Record | null + metrics: CallImportMetricSummary[] + status: 'pending' | 'running' | 'completed' | 'partial' | 'failed' + total_rows: number + completed_rows: number + failed_rows: number + error_message: string | null + /** Run-level LLM provider chosen by the user (null = legacy default). */ + llm_provider: string | null + llm_model: string | null + llm_credential_id: string | null + llm_config?: LLMGenerationConfig | null + metric_llm_overrides: Record | null + stt_provider: string | null + stt_model: string | null + stt_credential_id: string | null + /** + * Run-level LLM diariser config used when the worker auto-diarises + * rows that are missing a diarised transcript. + */ + diarisation_llm_provider?: string | null + diarisation_llm_model?: string | null + diarisation_llm_credential_id?: string | null + diarisation_prompt?: string | null + /** + * Diarisation pipeline shape this run was created with. + * - `stt_llm` (default) ΓÇö STT then an LLM diariser over the text. + * - `llm_only` ΓÇö audio fed directly to a multimodal diariser LLM. + * Surfaced so the retry / re-run UI can preselect the right mode. + */ + transcribe_mode?: 'stt_llm' | 'llm_only' + /** + * Which transcript column this run scored against. + * Defaults to `production` on legacy runs. + */ + transcript_source: CallImportEvaluationTranscriptSource + /** + * Other evaluation ids created in the same Run Evaluation request. + * Populated only on the POST response when the user ticked both + * Production and Diarised. Empty array on all other reads. + */ + sibling_evaluation_ids: string[] + started_at: string | null + finished_at: string | null + created_at: string + updated_at: string + created_by_email?: string | null + last_updated_by_email?: string | null + /** + * Cached LLM-generated TLDR rendered above the Visualizations tab. + * Populated lazily via ``POST /evaluations/{id}/insights``; null on + * runs the user has not summarised yet. + */ + tldr_summary?: EvaluationTldrSummary | null + user_insights?: EvaluationUserInsightsState | null + metric_clusters?: EvaluationMetricClustersState | null + /** + * True when the user opted into top-level metric discovery on the + * Run Evaluation modal. Gates the Discovered metrics panel on the + * evaluation detail Flow tab. + */ + discover_new_metrics?: boolean + /** + * Set while a bulk background operation (abort, force-fail, retry) is + * still running. The UI disables other mutating actions until cleared. + */ + bulk_operation?: 'abort' | 'force_fail_pending' | 'retry' | null +} + +/** + * LLM-generated narrative + bullet patterns for a single evaluation + * run. Cached on the evaluation row so re-opening the Visualizations + * tab doesn't auto-burn LLM tokens. ``is_stale`` is computed by the + * backend at read time when ``completed_rows`` has grown since the + * summary was generated. + */ +export interface EvaluationTldrSummary { + narrative: string + patterns: string[] + metric_insights?: Record + generated_at: string + generated_at_completed_rows: number + provider?: string | null + model?: string | null + is_stale: boolean +} + +export interface UserInsightCategory { + label: string + count: number + share_pct: number +} + +export interface UserInsightEvidenceTurn { + speaker: string + text: string +} + +export interface UserInsightEvidence { + conversation_id?: string | null + quote: string + turns?: UserInsightEvidenceTurn[] +} + +export interface EvaluationUserInsightItem { + id: string + title: string + categories: UserInsightCategory[] + observation: string + evidence: UserInsightEvidence +} + +export interface EvaluationUserInsightsState { + status: 'idle' | 'running' | 'completed' | 'failed' + insights: EvaluationUserInsightItem[] + overview?: string | null + generated_at?: string | null + generated_at_completed_rows: number + progress?: { completed_llm_calls: number; total_llm_calls: number } | null + provider?: string | null + model?: string | null + llm_calls_used: number + max_llm_calls?: number | null + error_message?: string | null + is_stale: boolean +} + +export type MetricClusterGapLabel = + | 'LOGIC_GAP' + | 'UNDERSPEC' + | 'EXISTS_NO_TRIGGER' + | 'MISSING' + +export interface MetricSubCluster { + label: string + count: number + share_pct: number +} + +export interface MetricClusterEvidenceTurn { + speaker: string + text: string +} + +export interface MetricClusterEvidence { + conversation_id?: string | null + evaluation_row_id?: string | null + quote: string + turns?: MetricClusterEvidenceTurn[] +} + +export interface MetricCluster { + id: string + label: string + gap_label: MetricClusterGapLabel + level: number + count: number + share_pct: number + sub_clusters: MetricSubCluster[] + observation: string + failure_reason?: string + evidence: MetricClusterEvidence + is_discovered: boolean +} + +export interface MetricClusterGroup { + metric_id: string + metric_name: string + flagged_count: number + failure_reason?: string + clusters: MetricCluster[] +} + +export interface DiscoveredProblemCluster { + id: string + label: string + gap_label: MetricClusterGapLabel + count: number + share_pct: number + observation: string + failure_reason?: string + evidence: MetricClusterEvidence +} + +export interface RcaRepeatedPatternRow { + metric_id: string + metric_name: string + top_rca_patterns: string + evidence_share_pct: number + evidence_calls: number + evidence_cluster_count?: number + failure_reason: string +} + +export interface RcaMetricHotspotRow { + metric_id: string + metric_name: string + description: string + metric_rate_pct: number + flagged_calls: number +} + +export interface RcaPromptAreaRow { + label: string + share_pct: number + gap_label: MetricClusterGapLabel +} + +export interface MetricClustersRcaSummary { + total_clusters: number + total_clustered_instances: number + total_flagged_instances?: number + analysed_calls: number + repeated_patterns: RcaRepeatedPatternRow[] + metric_hotspots: RcaMetricHotspotRow[] + prompt_areas: RcaPromptAreaRow[] +} + +export interface MetricFailurePolicy { + metric_id: string + failure_values: string[] + failure_child_names?: string[] + numeric_rule?: { op: 'lt' | 'lte' | 'gt' | 'gte'; threshold: number } | null +} + +export interface MetricFailurePolicyValueCount { + label: string + count: number +} + +export interface MetricFailurePolicyMetricPreview { + metric_id: string + metric_name: string + metric_type?: string | null + selection_mode?: string | null + is_multi_label_parent: boolean + value_counts: MetricFailurePolicyValueCount[] + child_names: string[] + row_count_by_value: Record + suggested_policy: MetricFailurePolicy + effective_policy: MetricFailurePolicy +} + +export interface MetricFailurePoliciesResponse { + previews: MetricFailurePolicyMetricPreview[] + policies: Record + source: 'inferred' | 'user' + updated_at?: string | null +} + +export interface MetricClusterEligibleRow { + evaluation_row_id: string + conversation_id?: string | null + row_index?: number | null + flagged_metric_names: string[] +} + +export interface MetricClusterEligibleRowsResponse { + items: MetricClusterEligibleRow[] + total: number +} + +export interface EvaluationMetricClustersState { + status: 'idle' | 'running' | 'completed' | 'failed' | 'cancelled' + groups: MetricClusterGroup[] + discovered_problems: DiscoveredProblemCluster[] + overview?: string | null + generated_at?: string | null + generated_at_completed_rows: number + progress?: { completed_llm_calls: number; total_llm_calls: number } | null + provider?: string | null + model?: string | null + llm_calls_used: number + max_llm_calls?: number | null + error_message?: string | null + is_stale: boolean + selected_evaluation_row_ids?: string[] + failure_policies?: Record + failure_policies_source?: 'inferred' | 'user' + failure_policies_updated_at?: string | null + rca_summary?: MetricClustersRcaSummary | null +} + +export interface AgentFlowNode { + id: string + label: string + node_type: 'start' | 'decision' | 'action' | 'terminal' + position_x?: number | null + position_y?: number | null + prompt_excerpt?: string | null + start_offset?: number | null + end_offset?: number | null +} + +export interface AgentFlowEdge { + source: string + target: string + condition?: string | null +} + +export interface AgentFlowGraph { + nodes: AgentFlowNode[] + edges: AgentFlowEdge[] + generated_at?: string | null + provider?: string | null + model?: string | null + layout_saved_at?: string | null + prompt_content_hash?: string | null + mapping_error?: string | null + generation_error?: string | null +} + +export interface ImportedAgent { + id: string + organization_id: string + name: string + description: string | null + content: string + tags: string[] | null + current_version: number + agent_flowchart?: AgentFlowGraph | null + agent_flowchart_status?: string | null + created_at: string + updated_at: string + created_by: string | null +} + +export interface ImportedAgentDetail extends ImportedAgent { + versions: PromptPartialVersion[] +} + +export interface MetricPartialChild { + name: string + description: string + example: string +} + +export interface MetricPartialContent { + schema_version: 1 + metric_kind: 'single' | 'category' + description: string + children?: MetricPartialChild[] +} + +export interface MetricPartial { + id: string + organization_id: string + name: string + description: string | null + content: string + tags: string[] | null + current_version: number + created_at: string + updated_at: string + created_by: string | null +} + +export interface MetricPartialDetail extends MetricPartial { + versions: PromptPartialVersion[] +} + +export interface PromptPartialVersion { + id: string + prompt_partial_id: string + version: number + content: string + change_summary: string | null + created_at: string + created_by: string | null +} + +export interface PromptImprovementSuggestion { + id: string + metric_id: string + metric_name: string + cluster_id: string + cluster_label: string + gap_label: MetricClusterGapLabel + share_pct: number + priority: 'high' | 'medium' | 'low' + change_type?: 'edit' | 'add' + target_section: string + anchor_excerpt?: string + current_gap: string + suggested_text: string + rationale: string + flow_node_id?: string + flow_node_label?: string +} + +export interface EvaluationPromptImprovementsState { + status: 'idle' | 'running' | 'completed' | 'failed' + imported_agent_id?: string | null + imported_agent_name?: string | null + suggestions: PromptImprovementSuggestion[] + overview?: string | null + generated_at?: string | null + generated_at_completed_rows: number + provider?: string | null + model?: string | null + error_message?: string | null + is_stale: boolean +} + +export interface MetricPeriodDelta { + label: string + detail: string + why?: string | null +} + +export interface CallImportEvaluationListResponse { + items: CallImportEvaluation[] + total: number +} + +export interface CallImportEvaluationBaselineCandidate { + evaluation_id: string + name: string + dataset: string + period_label: string | null + period_start: string | null + period_end: string | null + period_display: string + completed_rows: number + created_at: string + is_default: boolean +} + +export interface CallImportEvaluationBaselineCandidatesResponse { + items: CallImportEvaluationBaselineCandidate[] + default_evaluation_id: string | null +} + +export interface CallImportEvaluationPdfReport { + id: string + filename: string + preview_url?: string | null + download_url?: string | null + created_at: string + created_by?: string | null + report_type: string + vendor_name: string + config_summary?: string | null + storage_available?: boolean + cache_hit?: boolean +} + +export interface CallImportEvaluationPdfReportListItem { + id: string + filename?: string | null + vendor_name: string + report_type: string + created_by?: string | null + created_at: string + config_summary?: string | null + cache_fingerprint?: string | null +} + +export interface CallImportEvaluationPdfReportListResponse { + items: CallImportEvaluationPdfReportListItem[] +} + +export interface CallImportEvaluationRow { + id: string + evaluation_id: string + call_import_row_id: string + row_index: number | null + /** Mandatory identifier from the source batch (renamed from ``external_call_id``). */ + conversation_id: string | null + transcript: string | null + raw_columns: Record | null + recording_url: string | null + recording_date: string | null + /** + * S3 object key for the downloaded recording. Prefer this over + * ``recording_url`` for playback ΓÇö we resolve it to a presigned URL + * so audio plays from our storage instead of the (often expired) + * provider URL. + */ + recording_s3_key: string | null + diarised_transcript_status?: string | null + diarised_transcript_error?: string | null + status: 'pending' | 'running' | 'completed' | 'failed' | 'skipped' + metric_scores: Record + error_message: string | null + started_at: string | null + finished_at: string | null + created_at: string + updated_at: string +} + +export interface CallImportEvaluationRowListResponse { + items: CallImportEvaluationRow[] + total: number + page: number + page_size: number +} + +// --- Retry (re-enqueue failed rows on an existing evaluation run) --- + +export interface CallImportEvaluationRetryRequest { + /** + * Restrict the retry to a specific subset of evaluation rows. + * When omitted, every row with status='failed' in this run is + * re-enqueued. + */ + eval_row_ids?: string[] + + /** + * Optional LLM overrides. When provided, persisted onto the run so + * future retries default to the new config. ``llm_provider`` and + * ``llm_model`` must be sent together ΓÇö the backend 400s on + * half-configured input. + */ + llm_provider?: string + llm_model?: string + llm_credential_id?: string | null + + /** + * Optional STT overrides (only meaningful when the run scores the + * diarised transcript). Same paired-field rule as LLM. + */ + stt_provider?: string + stt_model?: string + stt_credential_id?: string | null + + /** + * When true, wipe the diarised transcript on every retried row so + * the (possibly new) STT runs from scratch. Only takes effect for + * diarised runs that have STT config. + */ + transcribe_overwrite?: boolean +} + +export interface CallImportEvaluationRetrySkippedItem { + eval_row_id: string + /** + * Why this row was not re-enqueued. Known values: + * - 'unknown' (id not in this run) + * - 'in_progress' (status is pending/running) + * - 'completed' (already successful) + * - 'source_row_missing' + */ + reason: 'unknown' | 'in_progress' | 'completed' | 'source_row_missing' +} + +export interface CallImportEvaluationRetryResponse { + requeued: number + /** + * Of those, how many were chained through a diarisation task first + * because the diarised transcript was missing. + */ + transcribe_requeued: number + skipped: CallImportEvaluationRetrySkippedItem[] +} + +export interface CallImportEvaluationBulkActionResponse { + accepted: boolean + target_count: number + evaluation_id: string +} + +// --- Diarization / transcription --- + +export interface CallImportTranscribeRequest { + /** + * Diarisation pipeline shape. + * - `stt_llm` (default) ΓÇö STT produces plain text, then an LLM + * diariser splits it into agent/user turns. STT fields required. + * - `llm_only` ΓÇö skip STT entirely and feed the audio bytes + * directly to a multimodal `diarization_llm_*` model along with + * `diarization_prompt`. STT fields MUST be omitted in this mode. + */ + mode?: 'stt_llm' | 'llm_only' + /** Required when `mode === 'stt_llm'`; must be null/omitted in `llm_only`. */ + stt_provider?: string | null + /** Required when `mode === 'stt_llm'`; must be null/omitted in `llm_only`. */ + stt_model?: string | null + credential_id?: string | null + language?: string | null + only_missing?: boolean + overwrite_existing?: boolean + row_ids?: string[] + /** + * LLM diariser. In `stt_llm` mode it splits the STT plain-text into + * agent/user turns; in `llm_only` mode it directly receives the + * audio along with `diarization_prompt`. Always required. + */ + diarization_llm_provider: string + diarization_llm_model: string + diarization_llm_credential_id?: string | null + /** + * Operator-supplied system prompt for the diariser LLM. NULL/empty + * means "fall back to the canonical default" (see + * ``getDiarisationDefaultPrompt``). + */ + diarization_prompt?: string | null +} + +export interface CallImportTranscribeResponse { + queued: number + skipped_rows: number + skipped_reason_counts: Record + accepted?: boolean +} + +export interface CallImportRowBulkDeleteResponse { + deleted: number + status?: 'completed' | 'accepted' +} + +export interface CallImportRetryFailedRowsResponse { + requeued: number + enqueue_failed: number + skipped: number +} + +export interface CallImportDiarisationPromptDefaultResponse { + prompt: string +} + +// --- Aggregation / visualization payloads --- + +export interface CallImportMetricHistogramBucket { + x0: number + x1: number + count: number +} + +export interface CallImportMetricValueCount { + label: string + count: number +} + +/** + * One unordered pair-count cell from a multi-label parent's + * co-occurrence matrix. ``a`` and ``b`` are child label names with + * ``a < b`` lexicographically; ``count`` is the number of rows on + * which both labels fired together. + */ +export interface CallImportMetricLabelPair { + a: string + b: string + count: number +} + +export interface CallImportMetricAggregate { + metric_id: string + metric_name: string + metric_type: string | null + metric_category?: 'quality' | 'user_insight' | string + /** + * True when the metric is a multi-label classifier parent. + * ``value_counts`` then lists per-child label tallies and one row + * may contribute to several labels, so the chart layout has to + * ignore the pie toggle (slices wouldn't sum to 100%) and the + * n-badge represents rows scored, not label occurrences. + */ + is_multi_label_parent?: boolean + count: number + skipped_count: number + error_count: number + mean: number | null + median: number | null + p25: number | null + p75: number | null + p95: number | null + min: number | null + max: number | null + stddev: number | null + histogram_buckets: CallImportMetricHistogramBucket[] + value_counts: CallImportMetricValueCount[] + /** + * Pairwise label intersections for multi-label parent metrics. + * Empty for everything else. The Visualizations tab reconstructs + * a square symmetric matrix from these unordered pairs to render + * the co-occurrence heatmap chart type. + */ + co_occurrence?: CallImportMetricLabelPair[] +} + +export interface CallImportEvaluationAggregateResponse { + evaluation_id: string + total_rows: number + completed_rows: number + failed_rows: number + metrics: CallImportMetricAggregate[] + period_deltas?: Record + baseline_evaluation_id?: string | null + failure_policies_source?: 'inferred' | 'user' | null +} + +export interface EvaluatorResultsAggregateResponse { + scope: string + suite_id?: string | null + agent_id?: string | null + scenario_id?: string | null + total_rows: number + completed_rows: number + failed_rows: number + metrics: CallImportMetricAggregate[] +} + +export interface CallImportInsightsRunPoint { + evaluation_id: string + name: string | null + created_at: string + mean: number | null + completed_rows: number +} + +export interface CallImportInsightsMetric { + metric_id: string + metric_name: string + metric_type: string | null + latest: CallImportMetricAggregate | null + trend: CallImportInsightsRunPoint[] +} + +export interface CallImportInsightsResponse { + call_import_id: string + total_rows: number + rows_with_transcript: number + rows_without_transcript: number + transcript_source_counts: Record + evaluation_count: number + metrics: CallImportInsightsMetric[] +} + +// --- Metrics hierarchy + flow visualization --- + +export interface MetricSummary { + id: string + organization_id: string + name: string + description: string | null + metric_type: string + metric_category?: 'quality' | 'user_insight' | string + trigger: string + enabled: boolean + is_default: boolean + metric_origin: string + supported_surfaces: string[] + enabled_surfaces: string[] + custom_data_type: string | null + custom_config: Record | null + tags: string[] | null + capture_rationale: boolean + parent_metric_id: string | null + selection_mode: MetricSelectionMode | null + allow_discovery?: boolean + /** + * When true, this metric is a "transcript-compare judge": at + * call-import evaluation time the worker feeds BOTH the production + * transcript and the diarised transcript to the LLM as a labeled + * pair, and the run's transcript_source toggle is ignored for this + * metric. Mutually exclusive with parent_metric_id and selection_mode + * ΓÇö comparison metrics stay standalone. + */ + compare_transcripts?: boolean + children?: MetricSummary[] + created_at: string + updated_at: string + created_by: string | null +} + +export interface MetricChildDraft { + name: string + description?: string | null + enabled?: boolean + capture_rationale?: boolean | null + tags?: string[] | null +} + +export interface MetricCreateWithChildrenPayload { + name: string + description?: string | null + selection_mode: MetricSelectionMode + enabled?: boolean + supported_surfaces?: string[] + enabled_surfaces?: string[] + tags?: string[] | null + allow_discovery?: boolean + children: MetricChildDraft[] +} + +export interface MetricFlowNode { + id: string + label: string + count: number + is_terminal: boolean + is_discovered?: boolean +} + +export interface MetricFlowEdge { + source: string + target: string + count: number +} + +export interface MetricFlowResponse { + parent_metric_id: string + parent_metric_name: string + selection_mode: MetricSelectionMode | null + nodes: MetricFlowNode[] + edges: MetricFlowEdge[] + total_rows: number + rows_with_sequence: number +} + +export interface DiscoveredLabel { + key: string + name: string + description?: string | null + sample_rationale?: string | null + /** + * Up to 3 distinct LLM rationales captured for this candidate + * across rows. The Discovered Labels promote flow surfaces the + * first 2 as an ``Examples:`` block on the new sub-metric's + * rubric so the user starts with concrete cases in the prompt. + */ + examples?: string[] + count: number +} + +export interface DiscoveredLabelsResponse { + parent_metric_id: string + items: DiscoveredLabel[] +} + +/** + * One LLM-discovered candidate TOP-LEVEL metric aggregated across all + * rows of an evaluation. Mirrors :class:`DiscoveredLabel` but adds a + * ``suggested_type`` field ΓÇö the LLM's guess at the best shape for + * the new metric ΓÇö that the promote modal can pre-fill the type radio + * with. + */ +export interface DiscoveredMetric { + key: string + name: string + description?: string | null + suggested_type: 'boolean' | 'rating' | 'category' + sample_rationale?: string | null + examples?: string[] + count: number +} + +export interface DiscoveredMetricsResponse { + evaluation_id: string + items: DiscoveredMetric[] +} + +export interface ObservabilityCallAgent { + id: string + agent_id?: string | null + name: string +} + +export interface ObservabilityCallData { + startedAt?: string + started_at?: string + endedAt?: string + ended_at?: string + from_phone_number?: string + to_phone_number?: string + endedReason?: string + recording_s3_key?: string + recording_url?: string + duration_seconds?: number + agent_name?: string + _agent_ref?: string | number + direction?: string + messages?: Array<{ role: string; content: string; start_time?: number; end_time?: number }> + live_transcript?: Array<{ role: string; content: string; timestamp?: string; start_time?: number }> + metadata?: Record + call_short_id?: string +} + +export interface ObservabilityCall { + id: string + call_short_id: string + status?: string | null + call_event?: string | null + is_live?: boolean + direction?: string | null + source?: string | null + provider_platform?: string | null + provider_call_id?: string | null + agent_id?: string | null + agent?: ObservabilityCallAgent | null + created_at?: string | null + updated_at?: string | null + call_data?: ObservabilityCallData | null + live_transcript?: Array<{ role: string; content: string; timestamp?: string }> + display_name?: string | null +} diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 5e60c88f..ed3e3ff3 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -30,5 +30,8 @@ export default defineConfig({ // Use relative paths for assets when building for production base: './', }, + test: { + environment: 'node', + }, }) diff --git a/scripts/create_platform_admin.py b/scripts/create_platform_admin.py new file mode 100644 index 00000000..c695d014 --- /dev/null +++ b/scripts/create_platform_admin.py @@ -0,0 +1,88 @@ +""" +Create a platform administrator account. + +Usage: + python -m scripts.create_platform_admin --email ops@example.com --password 'SecurePass1!' + +Loads config.yml and runs pending migrations so the admin is created in the +same catalog database the API server uses (important when DB sharding is on). +""" + +from __future__ import annotations + +import argparse +import getpass +import sys +from pathlib import Path + +from app.config import load_config_from_file +from app.core.migrations import run_migrations +from app.core.password import hash_password, validate_password_strength +from app.database import SessionLocal, init_db +from app.db_sharding.pool_manager import db_pool_manager +from app.models.database import PlatformAdmin + +CONFIG_PATH = Path(__file__).resolve().parent.parent / "config.yml" + + +def _load_runtime_config() -> None: + if CONFIG_PATH.exists(): + load_config_from_file(str(CONFIG_PATH)) + db_pool_manager.reset() + + +def main() -> int: + parser = argparse.ArgumentParser(description="Create an EfficientAI platform admin.") + parser.add_argument("--email", required=True, help="Platform admin email address.") + parser.add_argument( + "--password", + help="Password (prompted securely when omitted).", + default=None, + ) + args = parser.parse_args() + + password = args.password or getpass.getpass("Platform admin password: ") + try: + validate_password_strength(password) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 2 + + _load_runtime_config() + init_db() + run_migrations() + + db = SessionLocal() + try: + existing = db.query(PlatformAdmin).filter(PlatformAdmin.email == args.email).first() + if existing is not None: + print(f"Platform admin already exists for {args.email}.", file=sys.stderr) + return 3 + + admin = PlatformAdmin( + email=args.email, + password_hash=hash_password(password), + is_active=True, + ) + db.add(admin) + db.commit() + db.refresh(admin) + + admin_count = ( + db.query(PlatformAdmin) + .filter(PlatformAdmin.is_active == True) # noqa: E712 + .count() + ) + catalog_url = str(db_pool_manager.catalog_engine.url) + print(f"Created platform admin {args.email} (id={admin.id})") + print(f" Catalog database: {catalog_url}") + print(f" Active platform admins: {admin_count}") + print() + print("Sign in at /platform/login on the frontend.") + return 0 + finally: + db.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/excalidraw_builder.py b/scripts/excalidraw_builder.py new file mode 100644 index 00000000..f22aa387 --- /dev/null +++ b/scripts/excalidraw_builder.py @@ -0,0 +1,443 @@ +"""Helper to build well-spaced Excalidraw (.excalidraw) JSON files.""" + +from __future__ import annotations + +import json +import random +import string +from pathlib import Path +from typing import Any + + +def _id() -> str: + return "".join(random.choices(string.ascii_letters + string.digits, k=10)) + + +def _seed() -> int: + return random.randint(1, 2_000_000_000) + + +def _line_count(text: str) -> int: + return max(1, len(text.split("\n"))) + + +def _estimate_height(text: str, width: float, font_size: int, min_h: float) -> float: + lines = 0 + chars_per_line = max(12, int(width / (font_size * 0.55))) + for part in text.split("\n"): + lines += max(1, (len(part) + chars_per_line - 1) // chars_per_line) + return max(min_h, lines * font_size * 1.45 + 28) + + +class ExcalidrawBuilder: + """Build diagrams with consistent spacing and auto-sized boxes.""" + + def __init__(self, *, canvas_width: float = 1400) -> None: + self.elements: list[dict[str, Any]] = [] + self.canvas_width = canvas_width + self._max_y = 0.0 + self._max_x = 0.0 + + def _track_bounds(self, x: float, y: float, w: float, h: float) -> None: + self._max_x = max(self._max_x, x + w) + self._max_y = max(self._max_y, y + h) + + def box( + self, + x: float, + y: float, + w: float, + h: float, + label: str, + *, + bg: str = "#a5d8ff", + stroke: str = "#1e1e1e", + font_size: int = 18, + min_h: float | None = None, + ) -> tuple[str, float, float, float, float]: + """Return (id, x, y, w, h) with height auto-adjusted for label.""" + if min_h is None: + min_h = h + h = _estimate_height(label, w, font_size, min_h) + rid = _id() + tid = _id() + self._track_bounds(x, y, w, h) + self.elements.append( + { + "id": rid, + "type": "rectangle", + "x": x, + "y": y, + "width": w, + "height": h, + "angle": 0, + "strokeColor": stroke, + "backgroundColor": bg, + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": None, + "roundness": {"type": 3}, + "seed": _seed(), + "version": 1, + "versionNonce": _seed(), + "isDeleted": False, + "boundElements": [{"type": "text", "id": tid}], + "updated": 1, + "link": None, + "locked": False, + } + ) + self.elements.append( + { + "id": tid, + "type": "text", + "x": x + 12, + "y": y + 12, + "width": w - 24, + "height": h - 24, + "angle": 0, + "strokeColor": stroke, + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": None, + "roundness": None, + "seed": _seed(), + "version": 1, + "versionNonce": _seed(), + "isDeleted": False, + "boundElements": None, + "updated": 1, + "link": None, + "locked": False, + "text": label, + "fontSize": font_size, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": rid, + "originalText": label, + "lineHeight": 1.35, + } + ) + return rid, x, y, w, h + + def frame(self, x: float, y: float, w: float, h: float, label: str, *, behind: bool = False) -> None: + """Light grouping rectangle. Use behind=True to render under other elements.""" + rid = _id() + tid = _id() + self._track_bounds(x, y, w, h) + rect = { + "id": rid, + "type": "rectangle", + "x": x, + "y": y, + "width": w, + "height": h, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": None, + "roundness": {"type": 3}, + "seed": _seed(), + "version": 1, + "versionNonce": _seed(), + "isDeleted": False, + "boundElements": [{"type": "text", "id": tid}], + "updated": 1, + "link": None, + "locked": False, + } + text = { + "id": tid, + "type": "text", + "x": x + 16, + "y": y + 8, + "width": len(label) * 10, + "height": 24, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": None, + "roundness": None, + "seed": _seed(), + "version": 1, + "versionNonce": _seed(), + "isDeleted": False, + "boundElements": None, + "updated": 1, + "link": None, + "locked": False, + "text": label, + "fontSize": 16, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": rid, + "originalText": label, + "lineHeight": 1.25, + } + if behind: + self.elements.insert(0, text) + self.elements.insert(0, rect) + else: + self.elements.extend([rect, text]) + + def arrow( + self, + x1: float, + y1: float, + x2: float, + y2: float, + *, + label: str | None = None, + stroke: str = "#495057", + label_offset_y: float = -18, + ) -> None: + dx, dy = x2 - x1, y2 - y1 + self.elements.append( + { + "id": _id(), + "type": "arrow", + "x": x1, + "y": y1, + "width": dx, + "height": dy, + "angle": 0, + "strokeColor": stroke, + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": None, + "roundness": {"type": 2}, + "seed": _seed(), + "version": 1, + "versionNonce": _seed(), + "isDeleted": False, + "boundElements": None, + "updated": 1, + "link": None, + "locked": False, + "points": [[0, 0], [dx, dy]], + "lastCommittedPoint": None, + "startBinding": None, + "endBinding": None, + "startArrowhead": None, + "endArrowhead": "arrow", + } + ) + if label: + lx = (x1 + x2) / 2 - len(label) * 4 + ly = (y1 + y2) / 2 + label_offset_y + self._text_free(lx, ly, label, font_size=15, color=stroke) + + def arrow_down(self, cx: float, y_from: float, y_to: float, *, label: str | None = None) -> None: + self.arrow(cx, y_from, cx, y_to, label=label, label_offset_y=-12) + + def arrow_right(self, x_from: float, x_to: float, cy: float, *, label: str | None = None) -> None: + self.arrow(x_from, cy, x_to, cy, label=label) + + def title(self, text: str, *, y: float = 40) -> float: + x = 80 + self._text_free(x, y, text, font_size=32, color="#1a365d", width=self.canvas_width - 160) + return y + 56 + + def subtitle(self, text: str, y: float) -> float: + self._text_free(80, y, text, font_size=18, color="#495057", width=self.canvas_width - 160) + return y + 36 + + def note(self, y: float, text: str, *, bg: str = "#fff3bf", font_size: int = 16) -> float: + x = 80 + w = self.canvas_width - 160 + h = _estimate_height(text, w, font_size, 70) + self.box(x, y, w, h, text, bg=bg, font_size=font_size, min_h=h) + return y + h + 40 + + def _text_free( + self, + x: float, + y: float, + text: str, + *, + font_size: int = 16, + color: str = "#1e1e1e", + width: float = 400, + ) -> None: + h = _estimate_height(text, width, font_size, font_size + 8) + self._track_bounds(x, y, width, h) + self.elements.append( + { + "id": _id(), + "type": "text", + "x": x, + "y": y, + "width": width, + "height": h, + "angle": 0, + "strokeColor": color, + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": None, + "roundness": None, + "seed": _seed(), + "version": 1, + "versionNonce": _seed(), + "isDeleted": False, + "boundElements": None, + "updated": 1, + "link": None, + "locked": False, + "text": text, + "fontSize": font_size, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": None, + "originalText": text, + "lineHeight": 1.35, + } + ) + + def row_boxes( + self, + y: float, + items: list[tuple[str, str]], + *, + box_w: float = 220, + box_h: float = 100, + gap: float = 48, + font_size: int = 17, + center: bool = True, + ) -> list[tuple[float, float, float, float]]: + """Place boxes in a horizontal row; return list of (x,y,w,h).""" + n = len(items) + total_w = n * box_w + (n - 1) * gap + x0 = (self.canvas_width - total_w) / 2 if center else 80 + rects: list[tuple[float, float, float, float]] = [] + x = x0 + for label, color in items: + _, rx, ry, rw, rh = self.box(x, y, box_w, box_h, label, bg=color, font_size=font_size) + rects.append((rx, ry, rw, rh)) + x += box_w + gap + return rects + + def vertical_flow( + self, + start_y: float, + steps: list[tuple[str, str]], + *, + box_w: float = 520, + box_h: float = 88, + gap: float = 56, + font_size: int = 18, + ) -> list[tuple[float, float, float, float]]: + """Centered vertical pipeline with down arrows.""" + x = (self.canvas_width - box_w) / 2 + y = start_y + rects: list[tuple[float, float, float, float]] = [] + for i, (label, color) in enumerate(steps): + _, rx, ry, rw, rh = self.box(x, y, box_w, box_h, label, bg=color, font_size=font_size) + rects.append((rx, ry, rw, rh)) + if i < len(steps) - 1: + cx = x + box_w / 2 + next_y = y + rh + gap + self.arrow_down(cx, y + rh + 4, next_y - 4) + y = next_y + else: + y += rh + return rects + + def section_header(self, y: float, text: str, *, number: str | None = None) -> float: + """Large section divider for multi-diagram canvases.""" + label = f"{number} {text}" if number else text + x = 60 + w = self.canvas_width - 120 + h = 56 + self._track_bounds(x, y, w, h) + self.elements.append( + { + "id": _id(), + "type": "rectangle", + "x": x, + "y": y, + "width": w, + "height": h, + "angle": 0, + "strokeColor": "#1a365d", + "backgroundColor": "#e7f5ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": None, + "roundness": {"type": 3}, + "seed": _seed(), + "version": 1, + "versionNonce": _seed(), + "isDeleted": False, + "boundElements": None, + "updated": 1, + "link": None, + "locked": False, + } + ) + self._text_free(x + 20, y + 14, label, font_size=24, color="#1a365d", width=w - 40) + return y + h + 32 + + def save(self, path: Path, *, fit_viewport: bool = True, zoom: float | None = None) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + pad = 48 + if fit_viewport and zoom is None: + viewport_w, viewport_h = 1280, 900 + content_w = max(self._max_x + pad, self.canvas_width) + content_h = self._max_y + pad + zoom_val = min(viewport_w / content_w, viewport_h / content_h) + zoom_val = max(0.55, min(zoom_val, 1.15)) + else: + zoom_val = zoom if zoom is not None else 0.8 + payload = { + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": self.elements, + "appState": { + "gridSize": 20, + "viewBackgroundColor": "#ffffff", + "scrollX": pad, + "scrollY": pad, + "zoom": {"value": round(zoom_val, 2)}, + }, + "files": {}, + } + path.write_text(json.dumps(payload, indent=2), encoding="utf-8") diff --git a/scripts/generate_call_import_excalidraw_diagrams.py b/scripts/generate_call_import_excalidraw_diagrams.py new file mode 100644 index 00000000..ea7becca --- /dev/null +++ b/scripts/generate_call_import_excalidraw_diagrams.py @@ -0,0 +1,359 @@ +#!/usr/bin/env python3 +"""Generate a single Excalidraw file with all call-import architecture diagrams.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from excalidraw_builder import ExcalidrawBuilder + +OUT = Path(__file__).resolve().parent.parent / "docs" / "diagrams" / "excalidraw" +COMBINED = OUT / "call-import-architecture.excalidraw" + +# Palette +C_API = "#d0bfff" +C_WORKER = "#a5d8ff" +C_REDIS = "#ffc9c9" +C_CATALOG = "#b2f2bb" +C_SHARD = "#96f2d7" +C_BLOB = "#ffec99" +C_EXT = "#ffd8a8" +C_K8S = "#e7f5ff" +C_NOTE = "#fff3bf" + +W = 1500 +SECTION_GAP = 100 + + +def draw_platform_architecture(b: ExcalidrawBuilder, y: float) -> float: + y = b.subtitle("Kubernetes deployment with catalog DB, 4 data shards, Redis fair-share", y) + + cluster_y = y + y += 8 + + row1 = b.row_boxes( + y, + [ + ("API\n3 replicas", C_API), + ("worker-imports\n8–16 pods × 16 threads\nimports · diarization · eval", C_WORKER), + ("worker\naudio-metrics\nPraat / UTMOS / torch", C_WORKER), + ("Redis\nCelery broker\ninflight counters", C_REDIS), + ("Prometheus\n+ KEDA", C_K8S), + ], + box_w=240, + box_h=110, + gap=36, + font_size=16, + ) + y = max(r[1] + r[3] for r in row1) + 64 + + cat_x = (W - 360) / 2 + _, _, cy, _, ch = b.box( + cat_x, + y, + 360, + 100, + "Catalog Postgres — efficientai_catalog\nCallImport · CallImportEvaluation · metrics · shard registry", + bg=C_CATALOG, + font_size=17, + ) + y = cy + ch + 48 + + shard_rects = b.row_boxes( + y, + [ + ("data-shard-01\nCallImportRow\nEvalRow", C_SHARD), + ("data-shard-02\nCallImportRow\nEvalRow", C_SHARD), + ("data-shard-03\nCallImportRow\nEvalRow", C_SHARD), + ("data-shard-04\nCallImportRow\nEvalRow", C_SHARD), + ], + box_w=280, + box_h=110, + gap=40, + font_size=16, + ) + y = max(r[1] + r[3] for r in shard_rects) + 24 + b.frame(60, cluster_y, W - 120, y - cluster_y + 16, "Kubernetes cluster", behind=True) + y += 32 + + ext = b.row_boxes( + y, + [ + ("Blob storage\nS3 / GCS / Azure", C_BLOB), + ("STT / LLM providers", C_EXT), + ("Telephony\nExotel / Plivo", C_EXT), + ], + box_w=320, + box_h=90, + gap=60, + font_size=17, + ) + y = max(r[1] + r[3] for r in ext) + 48 + + cx_imports = row1[1][0] + row1[1][2] / 2 + b.arrow_down(cx_imports, row1[1][1] + row1[1][3], cy - 4, label="read/write rows") + b.arrow_down(cat_x + 180, cy + ch, shard_rects[0][1] - 4) + + return b.note( + y, + "Queue drain order on worker-imports: imports → diarization → eval-control → evaluations", + bg=C_K8S, + ) + + +def draw_postgres_redis_flow(b: ExcalidrawBuilder, y: float) -> float: + y = b.subtitle("Pending backlog lives in Postgres. Redis decides who may run.", y) + + steps = [ + ("1 · Write headers + rows\nPostgres catalog + data shards", C_CATALOG), + ("2 · Fair dispatcher reads pending rows\nPostgres scatter-gather (NOT Celery queue depth)", C_SHARD), + ("3 · acquire_eval_slot() / acquire_import_slot()\nRedis Lua script increments inflight counters", C_REDIS), + ("4 · Enqueue Celery task + store celery_task_id\nPostgres shard row updated", C_WORKER), + ("5 · Worker executes task\nPostgres reads/writes + STT / LLM / telephony APIs", C_WORKER), + ("6 · release_*_slot()\nRedis decrements inflight counters", C_REDIS), + ("7 · schedule_fair_dispatch()\nNext workspace round-robin turn", C_REDIS), + ] + rects = b.vertical_flow(y, steps, box_w=560, box_h=92, gap=64, font_size=17) + + y = rects[-1][1] + rects[-1][3] + 48 + return b.note( + y, + "Key insight: Celery queue can look empty while thousands of rows remain pending in Postgres.\n" + "Scale on org-wide pending rows + inflight saturation — not queue depth alone.", + bg=C_NOTE, + font_size=16, + ) + + +def draw_call_import_pipeline(b: ExcalidrawBuilder, y: float) -> float: + y = b.subtitle("Two slot types: import:* (bulk CSV) vs eval:* (transcribe + scoring chain)", y) + + stages = [ + ("Upload CSV\nAPI creates CallImport header", C_API), + ("Materialize rows\nBulk insert onto data shards", C_SHARD), + ("Bulk import (optional)\nprocess_call_import_row — uses import:* slots", C_REDIS), + ("Create evaluation\nCallImportEvaluation header on catalog", C_CATALOG), + ("Fair eval dispatch\nUses eval:* slots from here onward", C_REDIS), + ("Transcribe / diarize\nSTT + LLM diarisation queue", C_WORKER), + ("LLM + audio metrics\nFinal metric scores on shard row", C_WORKER), + ] + rects = b.vertical_flow(y, stages, box_w=560, box_h=92, gap=56, font_size=17) + + y = rects[-1][1] + rects[-1][3] + 48 + return b.note( + y, + "Eval-chain recording fetch uses eval:* slots (NOT import:*).\n" + "One eval slot is held from dispatch until the row finishes scoring.", + bg=C_NOTE, + ) + + +def draw_sharding_topology(b: ExcalidrawBuilder, y: float) -> float: + y = b.subtitle("Row routing spreads 10k imports across shards (~2.5k rows each)", y) + + cat_x = (W - 420) / 2 + _, _, cy, _, ch = b.box( + cat_x, + y, + 420, + 110, + "Catalog DB\nCallImport · CallImportEvaluation · metrics · call_import_shard_slices registry", + bg=C_CATALOG, + font_size=17, + ) + y = cy + ch + 56 + + shards = b.row_boxes( + y, + [ + ("data-shard-01", C_SHARD), + ("data-shard-02", C_SHARD), + ("data-shard-03", C_SHARD), + ("data-shard-04", C_SHARD), + ], + box_w=280, + box_h=80, + gap=44, + font_size=18, + ) + + for sx, sy, sw, sh in shards: + b.arrow_down(cat_x + 210, cy + ch, sy - 4) + + y = max(s[1] + s[3] for s in shards) + 48 + y = b.note( + y, + "Routing formula:\n" + " slice_id = row_index // 500\n" + " shard_id = SHA256(call_import_id : slice_id) mod 4", + bg=C_K8S, + font_size=17, + ) + return b.note( + y, + "Each pod holds SQLAlchemy pools: 1 catalog + 4 shards.\n" + "Rule: concurrency per pod ≤ catalog pool max (pool_size + max_overflow).", + bg=C_NOTE, + font_size=16, + ) + + +def draw_fair_dispatch(b: ExcalidrawBuilder, y: float) -> float: + y = b.subtitle("Dual 10k across two workspaces — equal fair share", y) + + ws = b.row_boxes( + y, + [ + ("Workspace A\n10,000 rows pending", C_API), + ("Workspace B\n10,000 rows pending", C_API), + ], + box_w=360, + box_h=100, + gap=120, + font_size=18, + ) + y = max(r[1] + r[3] for r in ws) + 56 + + cx = W / 2 + _, dx, dy, dw, dh = b.box( + cx - 300, + y, + 600, + 120, + "Global round-robin dispatcher\n" + "• max_workspace_turns = 999 on eval create (fill capacity)\n" + "• max_workspace_turns = 1 after each row completes (fair refill)\n" + "• batch_size should match eval_workspace_inflight_limit", + bg=C_REDIS, + font_size=16, + ) + y = dy + dh + 48 + + checks = b.row_boxes( + y, + [ + ( + "Eval slot checks\n(all must pass)\n" + "workspace → org → global → job", + C_REDIS, + ), + ( + "Effective parallelism\n" + "min(job, workspace,\n" + "global, threads)", + C_NOTE, + ), + ], + box_w=420, + box_h=130, + gap=80, + font_size=17, + ) + y = max(c[1] + c[3] for c in checks) + 48 + + b.arrow_right(ws[0][0] + ws[0][2], dx - 8, ws[0][1] + ws[0][3] / 2) + b.arrow_right(ws[1][0], dx + dw + 8, ws[1][1] + ws[1][3] / 2) + b.arrow_down(cx, ws[0][1] + ws[0][3], dy - 4) + + return b.note( + y, + "Recommended (2 workspaces, 16×16 pods):\n" + " eval_global = 256 · eval_workspace = 128 each · eval_job = 128\n" + " import_global = 96 · import_workspace = 48 each · import_batch = 48", + bg=C_K8S, + font_size=16, + ) + + +def draw_keda_pgbouncer_roadmap(b: ExcalidrawBuilder, y: float) -> float: + y = b.subtitle("Phase 1–3 now (no PgBouncer) → Phase 4–5 with PgBouncer", y) + + col_w = 560 + gap = 80 + x_left = (W - 2 * col_w - gap) / 2 + x_right = x_left + col_w + gap + + _, _, y1, _, h1 = b.box( + x_left, + y, + col_w, + 280, + "Now — Phases 1–3\n\n" + "• 8–16 pods × 16 concurrency\n" + "• eval_global 256 · job 128\n" + "• import 96 / workspace 48 (keep)\n" + "• KEDA on org-wide pending rows\n" + "• No PgBouncer\n\n" + "~256 parallel rows at max scale\n" + "Dual 10k ≈ 32 min", + bg="#d0bfff", + font_size=17, + ) + _, _, y2, _, h2 = b.box( + x_right, + y, + col_w, + 280, + "Future — Phases 4–5\n\n" + "• 16–20 pods × 28 concurrency\n" + "• eval_global 560 · workspace 280\n" + "• PgBouncer per catalog + shard\n" + "• transaction pool mode\n" + "• Small app pools (5–8)\n\n" + "Dual 10k in ~30 min target", + bg="#b2f2bb", + font_size=17, + ) + + mid_y = y + max(h1, h2) / 2 + b.arrow_right(x_left + col_w + 4, x_right - 4, mid_y, label="PgBouncer") + + y = y + max(h1, h2) + 48 + return b.note( + y, + "PgBouncer multiplexes many client connections into fewer server connections to RDS.\n" + "Required before scaling to 20 pods × 28 threads without pool_timeout errors.", + bg=C_NOTE, + ) + + +SECTIONS: list[tuple[str, str, object]] = [ + ("01", "Platform Architecture", draw_platform_architecture), + ("02", "Postgres ↔ Redis ↔ Workers", draw_postgres_redis_flow), + ("03", "Call Import Pipeline", draw_call_import_pipeline), + ("04", "Database Sharding", draw_sharding_topology), + ("05", "Fair Dispatch & Inflight Limits", draw_fair_dispatch), + ("06", "Scaling Roadmap (KEDA + PgBouncer)", draw_keda_pgbouncer_roadmap), +] + + +def build_combined() -> None: + b = ExcalidrawBuilder(canvas_width=W) + y = b.title("EfficientAI Call Import — Architecture Diagrams") + y = b.subtitle("All 6 diagrams in one canvas — scroll down to navigate sections 01–06", y) + y += 48 + + for number, title, draw_fn in SECTIONS: + y = b.section_header(y, title, number=number) + y = draw_fn(b, y) + y += SECTION_GAP + + b.save(COMBINED, fit_viewport=False, zoom=0.75) + + +def main() -> None: + build_combined() + + # Remove legacy per-diagram files if present + for legacy in OUT.glob("0*.excalidraw"): + legacy.unlink() + print(f"Removed legacy file: {legacy.name}") + + print(f"Wrote combined diagram: {COMBINED}") + + +if __name__ == "__main__": + main() diff --git a/scripts/generate_call_import_scaling_presentation.py b/scripts/generate_call_import_scaling_presentation.py new file mode 100644 index 00000000..e37a9ae7 --- /dev/null +++ b/scripts/generate_call_import_scaling_presentation.py @@ -0,0 +1,623 @@ +#!/usr/bin/env python3 +"""Generate PowerPoint: EfficientAI Call Import Architecture & Scaling Guide.""" + +from __future__ import annotations + +from pathlib import Path + +from pptx import Presentation +from pptx.dml.color import RGBColor +from pptx.enum.text import PP_ALIGN, MSO_ANCHOR +from pptx.util import Inches, Pt + +OUTPUT = Path(__file__).resolve().parent.parent / "docs" / "presentations" / "EfficientAI_Call_Import_Architecture_and_Scaling.pptx" + +# Brand-ish palette +NAVY = RGBColor(0x1A, 0x36, 0x5D) +TEAL = RGBColor(0x00, 0x96, 0x88) +SLATE = RGBColor(0x47, 0x55, 0x69) +WHITE = RGBColor(0xFF, 0xFF, 0xFF) +LIGHT_BG = RGBColor(0xF1, 0xF5, 0xF9) +ACCENT_ORANGE = RGBColor(0xEA, 0x58, 0x0C) + + +def _set_slide_bg(slide, color: RGBColor) -> None: + fill = slide.background.fill + fill.solid() + fill.fore_color.rgb = color + + +def _add_title_slide(prs: Presentation, title: str, subtitle: str) -> None: + slide = prs.slides.add_slide(prs.slide_layouts[6]) # blank + _set_slide_bg(slide, NAVY) + box = slide.shapes.add_textbox(Inches(0.6), Inches(2.0), Inches(12.0), Inches(1.5)) + tf = box.text_frame + p = tf.paragraphs[0] + p.text = title + p.font.size = Pt(40) + p.font.bold = True + p.font.color.rgb = WHITE + p.alignment = PP_ALIGN.LEFT + + sub = slide.shapes.add_textbox(Inches(0.6), Inches(3.6), Inches(12.0), Inches(1.2)) + stf = sub.text_frame + sp = stf.paragraphs[0] + sp.text = subtitle + sp.font.size = Pt(20) + sp.font.color.rgb = RGBColor(0xCB, 0xD5, 0xE1) + sp.alignment = PP_ALIGN.LEFT + + +def _add_section_slide(prs: Presentation, title: str) -> None: + slide = prs.slides.add_slide(prs.slide_layouts[6]) + _set_slide_bg(slide, TEAL) + box = slide.shapes.add_textbox(Inches(0.8), Inches(3.0), Inches(11.5), Inches(1.2)) + tf = box.text_frame + p = tf.paragraphs[0] + p.text = title + p.font.size = Pt(36) + p.font.bold = True + p.font.color.rgb = WHITE + p.alignment = PP_ALIGN.LEFT + + +def _add_content_slide( + prs: Presentation, + title: str, + bullets: list[str], + *, + subtitle: str | None = None, +) -> None: + slide = prs.slides.add_slide(prs.slide_layouts[6]) + _set_slide_bg(slide, LIGHT_BG) + + title_box = slide.shapes.add_textbox(Inches(0.5), Inches(0.35), Inches(12.3), Inches(0.7)) + tfp = title_box.text_frame.paragraphs[0] + tfp.text = title + tfp.font.size = Pt(28) + tfp.font.bold = True + tfp.font.color.rgb = NAVY + + y = 1.15 + if subtitle: + sub_box = slide.shapes.add_textbox(Inches(0.5), Inches(y), Inches(12.3), Inches(0.5)) + sfp = sub_box.text_frame.paragraphs[0] + sfp.text = subtitle + sfp.font.size = Pt(14) + sfp.font.italic = True + sfp.font.color.rgb = SLATE + y += 0.55 + + body = slide.shapes.add_textbox(Inches(0.55), Inches(y), Inches(12.2), Inches(6.5 - y)) + tf = body.text_frame + tf.word_wrap = True + tf.vertical_anchor = MSO_ANCHOR.TOP + + for i, bullet in enumerate(bullets): + p = tf.paragraphs[0] if i == 0 else tf.add_paragraph() + p.text = bullet + p.level = 0 + p.font.size = Pt(16 if len(bullet) < 120 else 14) + p.font.color.rgb = SLATE + p.space_after = Pt(8) + + +def _add_table_slide( + prs: Presentation, + title: str, + headers: list[str], + rows: list[list[str]], +) -> None: + slide = prs.slides.add_slide(prs.slide_layouts[6]) + _set_slide_bg(slide, LIGHT_BG) + + title_box = slide.shapes.add_textbox(Inches(0.5), Inches(0.35), Inches(12.3), Inches(0.7)) + tfp = title_box.text_frame.paragraphs[0] + tfp.text = title + tfp.font.size = Pt(26) + tfp.font.bold = True + tfp.font.color.rgb = NAVY + + nrows = len(rows) + 1 + ncols = len(headers) + table_shape = slide.shapes.add_table(nrows, ncols, Inches(0.4), Inches(1.1), Inches(12.5), Inches(0.35 * nrows + 0.3)) + table = table_shape.table + + col_width = Inches(12.5 / ncols) + for c in range(ncols): + table.columns[c].width = int(col_width) + + for c, header in enumerate(headers): + cell = table.cell(0, c) + cell.text = header + for p in cell.text_frame.paragraphs: + p.font.bold = True + p.font.size = Pt(11) + p.font.color.rgb = WHITE + cell.fill.solid() + cell.fill.fore_color.rgb = NAVY + + for r, row in enumerate(rows, start=1): + for c, val in enumerate(row): + cell = table.cell(r, c) + cell.text = val + for p in cell.text_frame.paragraphs: + p.font.size = Pt(10) + p.font.color.rgb = SLATE + if r % 2 == 0: + cell.fill.solid() + cell.fill.fore_color.rgb = RGBColor(0xE2, 0xE8, 0xF0) + + +def build_presentation() -> Path: + prs = Presentation() + prs.slide_width = Inches(13.333) + prs.slide_height = Inches(7.5) + + # --- Title --- + _add_title_slide( + prs, + "EfficientAI Call Import Architecture & Scaling", + "Enterprise deployment guide | Sharded Postgres | Redis fair-share | Kubernetes + KEDA", + ) + + # --- Agenda --- + _add_content_slide( + prs, + "Agenda", + [ + "Platform architecture overview", + "Call import end-to-end pipeline (import → transcribe → eval)", + "Postgres catalog + data shards vs Redis coordination", + "Fair dispatch, inflight limits, and batch sizing", + "Customer scenario: dual 10k workspace runs", + "Before vs recommended configuration numbers", + "Database connection budget (without PgBouncer)", + "Kubernetes autoscaling with Prometheus + KEDA", + "Future scope: PgBouncer / RDS Proxy", + "Monitoring, load-test exit criteria, and next steps", + ], + ) + + # --- Architecture --- + _add_section_slide(prs, "1. Platform Architecture") + + _add_content_slide( + prs, + "High-Level Architecture", + [ + "EfficientAI runs as Kubernetes services: API, worker-imports, worker (audio-metrics), Redis, and Postgres.", + "Catalog DB (efficientai_catalog): metadata — CallImport headers, CallImportEvaluation, org/workspace, metrics catalog, shard registry.", + "Data shards (efficientai_data_01 … _04): high-volume row data — CallImportRow, CallImportEvaluationRow.", + "Redis: Celery broker/result backend + fair-share inflight counters + dispatch cursors + rate-limit state.", + "Blob storage (S3/GCS/Azure): CSV uploads and call recordings.", + "External APIs: STT/LLM providers, telephony recording URLs (Exotel, Plivo, direct URLs).", + ], + subtitle="1 catalog + 4 data shards (customer topology)", + ) + + _add_content_slide( + prs, + "Worker Topology", + [ + "worker-imports (thread pool): queues imports, diarization, eval-control, evaluations", + " → I/O bound: recording fetch, STT, LLM diarisation, LLM metric scoring, fair dispatch", + " → Typical K8s: 8–16 replicas × 16 concurrency (recommended)", + "worker (prefork): queues celery + audio-metrics", + " → CPU/audio bound: Praat, UTMOS, torch-based qualitative voice metrics", + " → Separate pool avoids OMP/torch deadlocks with high Celery concurrency", + "Queue drain order (worker-imports): imports → diarization → eval-control → evaluations", + ], + ) + + # --- Pipeline --- + _add_section_slide(prs, "2. Call Import Pipeline") + + _add_content_slide( + prs, + "End-to-End Pipeline Stages", + [ + "1. Upload CSV → API creates CallImport header (catalog) + stores CSV in blob storage", + "2. Materialize rows → bulk_insert_mappings_on_shards() → CallImportRow on data shards", + "3. Bulk import (optional phase) → fair_import_dispatch → process_call_import_row (fetch recording → S3)", + "4. Create evaluation → CallImportEvaluation header (catalog) + materialize CallImportEvaluationRow on shards", + "5. Fair eval dispatch → per row: import (if needed) → transcribe → audio metrics → LLM eval", + "6. Rollup → parent counters and evaluation status updated; insights tasks may follow", + ], + subtitle="Unified eval pipeline: one Redis eval slot can cover import + transcribe + eval for a row", + ) + + _add_content_slide( + prs, + "Celery Queues & Key Tasks", + [ + "imports — process_call_import_row, dispatch_fair_import_rows, bulk materialize/delete", + "diarization — transcribe_call_import_row, dispatch_fair_diarization_rows", + "eval-control — materialize/cancel/retry evaluation (operator actions, not head-of-line blocked)", + "evaluations — dispatch_fair_eval_rows, evaluate_call_import_row, insights generation", + "audio-metrics — evaluate_call_import_row_audio (separate worker service)", + "Task time limits: transcribe soft 12 min; LLM eval soft 8 min; global Celery hard limit 30 min", + ], + ) + + _add_content_slide( + prs, + "Per-Row Dispatch Decision Tree", + [ + "_try_dispatch_single_row() (eval_dispatch.py) picks the next stage for each pending eval row:", + " • No S3 recording yet → enqueue process_call_import_row on imports (uses eval slot in eval chain)", + " • Needs diarised transcript → enqueue transcribe_call_import_row on diarization", + " • Audio-only metrics configured → enqueue evaluate_call_import_row_audio on audio-metrics", + " • Otherwise → enqueue evaluate_call_import_row on evaluations (LLM/comparison metrics)", + "Slot held from dispatch until row completes → finish_eval_work_and_redispatch() releases slot", + "Bulk CSV import (pre-eval) uses separate import:* Redis slots via fair_import_dispatch", + ], + ) + + # --- Postgres vs Redis --- + _add_section_slide(prs, "3. Postgres vs Redis") + + _add_content_slide( + prs, + "Division of Responsibility", + [ + "Postgres = source of truth for durable state (rows, statuses, scores, headers, registry)", + "Redis = ephemeral coordination (inflight caps, fair-scheduling cursors, dedupe locks, progress hashes)", + "Celery broker (Redis): task messages — but bulk pending work lives in Postgres, not the queue", + "Key insight: queue depth alone is a poor autoscaling signal — fair dispatch only enqueues when a slot is free", + ], + ) + + _add_content_slide( + prs, + "What Lives in Postgres", + [ + "Catalog DB:", + " • CallImport, CallImportEvaluation (headers, config, status)", + " • call_import_shard_slices registry (rebalance overrides)", + " • Organizations, workspaces, metrics, AI provider credentials", + "Data shards (per shard):", + " • CallImportRow — recording_url, recording_s3_key, transcripts, import status", + " • CallImportEvaluationRow — eval status, metric_scores, celery_task_id", + "Pending work query: status=pending AND celery_task_id IS NULL (scatter-gather across shards)", + ], + ) + + _add_content_slide( + prs, + "What Lives in Redis", + [ + "Eval inflight counters: eval:inflight:global | :org:{id} | :workspace:{id} | :job:{eval_id}", + "Import inflight counters: import:inflight:global | :org:{id} | :workspace:{id}", + "Slot task maps: eval:slot:task:{celery_task_id}, import:slot:task:{celery_task_id}", + "Fair dispatch cursors: eval:fair:rr_cursor, eval:fair:rr_cursor:ws:{workspace_id} (and import equivalents)", + "Dispatch locks/dedupe: eval:fair:dispatch_lock, eval:fair:dispatch_dedupe (15s backoff at capacity)", + "Retry metadata: eval:restricted:row:{id}, eval:transcribe_overwrite:{evaluation_id}", + "Telephony rate limits: telephony:import:credits:{fingerprint} (default 1000/min per credential)", + ], + ) + + _add_content_slide( + prs, + "Data Flow: Postgres ↔ Redis ↔ Workers", + [ + "① API writes headers + rows to Postgres (catalog + shards via router)", + "② API/worker schedules dispatch_fair_* Celery task (message → Redis broker)", + "③ Dispatcher reads pending rows from Postgres (scatter-gather on shards + catalog joins)", + "④ For each row: acquire_*_slot() atomically increments Redis inflight counters (Lua script)", + "⑤ If slot acquired → enqueue row task to Celery; store celery_task_id on shard row (Postgres)", + "⑥ Worker executes task: reads/writes Postgres (catalog + one shard), calls external APIs", + "⑦ Task completes → release_*_slot() decrements Redis counters → schedule next fair dispatch turn", + "At capacity: dispatch stops enqueueing; pending rows remain in Postgres (queue may look empty)", + ], + subtitle="Pending backlog is in Postgres; Redis tracks who is allowed to run right now", + ) + + # --- Sharding --- + _add_section_slide(prs, "4. Database Sharding") + + _add_content_slide( + prs, + "Shard Routing", + [ + "Enabled via database.sharding.enabled: true in config.yml", + "slice_id = row_index // row_chunk_size (default row_chunk_size = 500)", + "shard_id = SHA256(call_import_id : slice_id) mod N (N = number of shards)", + "Registry table call_import_shard_slices can override routing during rebalance", + "10k row import → ~20 slices (500 rows each) → hash-distributed across 4 shards (~2.5k rows/shard)", + "Router: app/db_sharding/router.py (ShardRouter) | Pools: app/db_sharding/pool_manager.py", + ], + ) + + _add_content_slide( + prs, + "Connection Pools (Per Process)", + [ + "Each worker/API process holds SQLAlchemy pools for catalog + every shard", + "With 4 shards, pool_manager reduces per-shard pool but enforces floor: 8+8 = 16 connections/shard/process", + "Catalog pool uses full pool_size + max_overflow (not divided)", + "Each active task typically holds: 1 catalog session + 1 shard session (seconds to minutes)", + "Fair dispatch may open ShardSessionCache: up to 4 shard sessions + 1 catalog per dispatch pass", + "Rule: concurrency per pod ≤ catalog pool max — or threads block on pool timeout", + ], + ) + + # --- Fair dispatch --- + _add_section_slide(prs, "5. Fair Dispatch & Limits") + + _add_content_slide( + prs, + "Fair Dispatch Mechanics", + [ + "Two-level round-robin:", + " • Global: rotate across workspaces that have pending rows (eval:fair:rr_cursor)", + " • Per-workspace: rotate across evaluations (eval:fair:rr_cursor:ws:{workspace_id})", + "max_workspace_turns:", + " • 999 on eval create / catch-up → fill capacity across workspaces quickly", + " • 1 after each row completes → fair refill, one workspace turn at a time", + "eval_fair_dispatch_batch_size: max rows attempted per workspace turn (default 75)", + " → Should align with eval_workspace_inflight_limit to avoid stair-step ramp", + ], + ) + + _add_content_slide( + prs, + "Inflight Limit Hierarchy", + [ + "Eval slot acquisition checks (all must pass):", + " workspace_limit → org_limit → global_limit → job_limit (per evaluation id)", + "Import slot acquisition (bulk CSV only): workspace → org → global", + "Effective parallel rows = min(job, workspace, org, global, worker_threads)", + "Customer issue identified: eval_job_inflight_limit defaults to 75 — caps large single evaluations", + "Even with 512 global and 28 concurrency, effective parallelism was ~75 rows for 5k/10k runs", + ], + ) + + # --- Customer scenario --- + _add_section_slide(prs, "6. Customer Scenario & Metrics") + + _add_content_slide( + prs, + "Observed Customer Baseline", + [ + "Topology: 1 catalog Postgres + 4 data shard Postgres instances", + "Deployment: Kubernetes with Prometheus + KEDA autoscaling", + "Workload: 5,000 calls per workspace — full pipeline (import + transcribe + eval) in ~30 minutes", + "Target: 10,000 calls in 30 minutes (single workspace) or dual 10k across two workspaces", + "Original config: 8 pods × 28 concurrency, eval_global=512, eval_workspace=150, import_global=96", + "Hidden bottleneck: eval_job_inflight_limit = 75 (default) — not explicitly raised", + ], + ) + + _add_table_slide( + prs, + "Before vs Recommended Configuration", + ["Setting", "Before (customer)", "Now (recommended)"], + [ + ["KEDA min / max replicas", "8 / 16", "8 / 16"], + ["worker_imports_concurrency", "28", "16"], + ["Thread capacity @ max pods", "448", "256"], + ["database pool_size / max_overflow", "10 / 15", "6 / 10"], + ["Catalog pool max / pod", "25", "16 (= concurrency)"], + ["eval_global_inflight_limit", "512", "256"], + ["eval_workspace_inflight_limit", "150", "128"], + ["eval_job_inflight_limit", "75 (default)", "128"], + ["eval_fair_dispatch_batch_size", "75 (default)", "128"], + ["import_global_inflight_limit", "96", "96 (keep)"], + ["import_workspace_inflight_limit", "48", "48 (keep)"], + ["import_fair_dispatch_batch_size", "75 (default)", "48"], + ], + ) + + _add_table_slide( + prs, + "Effective Parallelism & Throughput (@ ~24 s/row)", + ["Scenario", "Before (effective)", "Recommended @ max (16 pods)"], + [ + ["Single 10k eval @ 8 pods", "~75 parallel (~30 min)", "~128 parallel (~31 min)"], + ["Single 10k eval @ 16 pods", "~75 parallel (~30 min)", "~256 parallel (~16 min)"], + ["Dual 10k (2 workspaces) @ 16 pods", "~150 parallel (2×75 job)", "~256 parallel (2×128 ws)"], + ["Bulk import per workspace", "48 parallel fetches", "48 parallel (unchanged)"], + ], + ) + + _add_content_slide( + prs, + "Why Recommended Numbers Change", + [ + "Raise eval_job 75 → 128: removes hidden cap that prevented scaling past ~75 rows/eval", + "Lower concurrency 28 → 16: catalog pool (25) < 28 threads caused pool timeout / DB throttling", + "Align eval batch 75 → 128: one workspace turn can fill quota (no stair-step with ws limit 128)", + "Keep import 96/48: bulk import phase stays fast for dual 10k; only fix import batch 75 → 48", + "eval_global 512 → 256: match realistic max capacity (16 × 16) without over-dispatching DB", + "Fair share for 2 workspaces: eval_workspace = eval_global ÷ 2 = 128", + ], + ) + + _add_table_slide( + prs, + "Fair-Share Limit Formulas", + ["Limit", "Formula", "Example (2 heavy workspaces, max 16×16)"], + [ + ["eval_global", "max_replicas × concurrency", "16 × 16 = 256"], + ["eval_workspace", "eval_global ÷ N workspaces", "256 ÷ 2 = 128"], + ["eval_job", "≥ eval_workspace", "128"], + ["eval_fair_dispatch_batch_size", "= eval_workspace", "128"], + ["import_global", "keep or import_ws × N", "48 × 2 = 96"], + ["import_workspace", "import_global ÷ N", "96 ÷ 2 = 48"], + ["import_fair_dispatch_batch_size", "= import_workspace", "48"], + ], + ) + + # --- DB connections --- + _add_section_slide(prs, "7. Database Connection Budget") + + _add_content_slide( + prs, + "Connection Math (No PgBouncer)", + [ + "Assume: 16 worker-imports + 3 API pods = 19 processes at max scale", + "Catalog connections: 19 × (pool_size + max_overflow) = 19 × 16 = ~304", + "Per shard connections: 19 × 16 (shard pool floor) = ~304 per shard RDS", + "Minimum RDS max_connections: catalog ≥ 450, each shard ≥ 450 (with admin headroom)", + "Load-test exit criteria (docs/operations/call-import-sharding-load-test.md):", + " • Shard CPU ≤ 75% • Catalog CPU ≤ 50% • No pool_timeout / connection exhaustion", + ], + ) + + _add_content_slide( + prs, + "DB Throttling Risks at Scale", + [ + "Per-pod pool exhaustion: concurrency > catalog pool → SQLAlchemy pool timeout", + "RDS max_connections: 19 processes × 16/shard × 4 shards = high aggregate without multiplexing", + "Catalog CPU hotspot: every row reads eval config/metrics from catalog during dispatch + scoring", + "Hot shard: uneven hash distribution (usually OK for 10k imports with 20 slices across 4 shards)", + "Write IOPS on shards: concurrent status + metric_scores JSON updates under high inflight", + "Mitigation now: match concurrency to pool, cap max replicas, monitor dispatch diagnostics", + ], + ) + + # --- KEDA --- + _add_section_slide(prs, "8. Kubernetes Autoscaling (KEDA)") + + _add_content_slide( + prs, + "KEDA Strategy", + [ + "minReplicaCount: 8 (never scale below — customer requirement)", + "maxReplicaCount: 16 (20 only with PgBouncer or larger RDS max_connections)", + "Do NOT scale on Celery queue depth alone — fair dispatch keeps queues small while Postgres backlog grows", + "Primary signals (Prometheus):", + " • sum(efficientai_eval_rows_pending) — org-wide Postgres backlog", + " • eval_inflight_global / eval_global_limit > 0.85 AND pending > 0", + " • sum(celery_queue_length{queue=~\"imports|diarization|evaluations\"}) — burst detector", + "Scale-down: slow (5–10 min stabilization); terminationGracePeriodSeconds: 900 (15 min tasks)", + ], + ) + + _add_content_slide( + prs, + "Observability Stack", + [ + "Prometheus scrapes: FastAPI /metrics, celery-exporter, redis-exporter, postgres-exporter", + "celery-exporter (docker-compose.observability.yml): queue lengths, task metrics", + "Operator endpoint: GET /api/v1/call-imports/dispatch-diagnostics (admin)", + " → per-workspace inflight, pending_dispatch_rows, job inflight, RR cursors, at_capacity flags", + "Grafana dashboards: compare workspace A vs B inflight during dual 10k runs", + "Alerts: global_at_capacity + high pending; catalog/shard CPU; pool timeout in worker logs", + ], + ) + + # --- PgBouncer --- + _add_section_slide(prs, "9. Future Scope: PgBouncer") + + _add_content_slide( + prs, + "Why PgBouncer (Next Phase)", + [ + "Current constraint: each K8s pod opens pool_size + max_overflow connections per catalog + each shard", + "Without multiplexing: scaling to 20 pods × 28 threads requires 500+ catalog and 300+ per shard connections", + "PgBouncer sits between app pods and RDS — many client connections, fewer server connections", + "Enables: higher concurrency per pod, more replicas, higher inflight limits (560 global / 280 per workspace)", + "Target outcome: dual 10k in ~30 min at max scale without pool_timeout throttling", + ], + ) + + _add_content_slide( + prs, + "PgBouncer Topology (Recommended)", + [ + "Deploy one PgBouncer pool per logical database:", + " • pgbouncer-catalog → efficientai_catalog RDS", + " • pgbouncer-shard-01 … pgbouncer-shard-04 → each data shard RDS", + "Alternative: single PgBouncer instance with multiple database entries (simpler ops, shared process)", + "App DATABASE_URL / shard URLs point to PgBouncer service DNS, not RDS directly", + "K8s: PgBouncer as StatefulSet or Helm chart; sidecar pattern generally NOT recommended here", + ], + ) + + _add_table_slide( + prs, + "PgBouncer Settings (Starting Point)", + ["Parameter", "Catalog pool", "Each data shard pool"], + [ + ["pool_mode", "transaction", "transaction"], + ["default_pool_size", "80–120", "50–80"], + ["max_client_conn", "2000", "2000"], + ["server_idle_timeout", "600", "600"], + ["App pool_size (per pod)", "5–8", "5–8"], + ["App max_overflow", "8–12", "8–12"], + ["App concurrency (with PgBouncer)", "24–28", "24–28"], + ], + ) + + _add_content_slide( + prs, + "PgBouncer + Scaling Profile (Future)", + [ + "After PgBouncer rollout, target enterprise profile:", + " • 16–20 worker-imports replicas × 28 concurrency = 448–560 thread capacity", + " • eval_global = 560, eval_workspace = 280 (÷ 2 workspaces), eval_job = 280", + " • eval_fair_dispatch_batch_size = 280", + " • import_global = 128–160, import_workspace = 64–80", + "Expected: 20k rows (dual 10k) in ~30 min @ ~24 s/row with full pod scale", + "Prerequisite: load-test scenario D (25k rows) pass — shard CPU ≤ 75%, catalog ≤ 50%", + ], + ) + + _add_content_slide( + prs, + "PgBouncer Caveats for SQLAlchemy", + [ + "Use transaction pooling — compatible with short ORM transactions in workers", + "Avoid session-level features across transactions: TEMP tables, advisory locks, SET per session", + "SQLAlchemy: pool_pre_ping=True (already enabled), keep app pools small — let PgBouncer multiplex", + "Do NOT set app pool_size equal to concurrency without PgBouncer; with PgBouncer, small app pools are correct", + "Migrate staging first: run call-import-sharding-load-test scenarios A–D before production cutover", + "Rollback plan: keep direct RDS URLs in config; switch DNS/URL to revert", + ], + ) + + # --- Next steps --- + _add_section_slide(prs, "10. Next Steps") + + _add_content_slide( + prs, + "Implementation Roadmap", + [ + "Phase 1 (now, no PgBouncer): Apply recommended config; set eval_job=128; concurrency=16; keep import 96/48", + "Phase 2: Expose pending-row + inflight Prometheus metrics; tune KEDA on org-wide backlog", + "Phase 3: Run load-test scenarios A–D on staging; validate dispatch diagnostics during dual 10k", + "Phase 4: Deploy PgBouncer per catalog + shard; retune to 560/280 enterprise inflight profile", + "Phase 5: Re-run dual 10k sign-off; document RDS instance sizing and connection budgets", + ], + ) + + _add_content_slide( + prs, + "Key Takeaways", + [ + "Postgres holds durable row state; Redis enforces fair concurrency — they work together, not interchangeably", + "Sharding spreads row I/O; catalog remains a shared metadata hub — size and pool it carefully", + "eval_job_inflight_limit was the hidden bottleneck — not global limit or pod count", + "Dual 10k across workspaces: set workspace limits = global ÷ 2 for both eval AND import", + "Queue-length autoscaling alone fails — scale on Postgres pending rows + inflight saturation", + "PgBouncer unlocks the next tier (560 inflight, 20 pods, dual 10k in ~30 min) safely", + ], + ) + + _add_title_slide( + prs, + "Questions?", + "EfficientAI Call Import Architecture | docs/operations/call-import-sharding-load-test.md", + ) + + OUTPUT.parent.mkdir(parents=True, exist_ok=True) + prs.save(str(OUTPUT)) + return OUTPUT + + +if __name__ == "__main__": + path = build_presentation() + print(f"Wrote {path}") diff --git a/tests/conftest.py b/tests/conftest.py index 6d64dd1c..4fa7220a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,1100 +1,1104 @@ -"""Shared pytest fixtures for backend tests.""" - -import os -import sys -import types -from contextlib import asynccontextmanager -from pathlib import Path -from uuid import uuid4 - -import pytest -from fastapi import FastAPI -from fastapi.testclient import TestClient -from sqlalchemy import create_engine, event, text -from sqlalchemy.engine import make_url -from sqlalchemy.orm import sessionmaker -from sqlalchemy.pool import StaticPool - -# Some local environments provide ALLOWED_AUDIO_FORMATS as a non-JSON string, -# which breaks pydantic-settings parsing during module import in tests. -os.environ["ALLOWED_AUDIO_FORMATS"] = '["wav","mp3","flac","m4a"]' -# Ensure storage service singletons can initialize in test environments. -os.environ["UPLOAD_DIR"] = "/tmp/efficientai-test-uploads" -# Local dev often sets SERVICE_MODE=media and config.yml media URLs; keep API tests on full app mode. -os.environ["SERVICE_MODE"] = "api" - -_REPO_ROOT = Path(__file__).resolve().parents[1] -_SRC_ROOT = _REPO_ROOT / "src" -for _path in (str(_REPO_ROOT), str(_SRC_ROOT)): - if _path not in sys.path: - sys.path.insert(0, _path) - -_TASKS_PACKAGE_DIR = str( - Path(__file__).resolve().parents[1] / "app" / "workers" / "tasks" -) - - -@pytest.fixture(autouse=True) -def isolate_service_mode_for_app_factory(monkeypatch): - """Prevent local SERVICE_MODE / media URL config from breaking create_app tests.""" - from app.config import settings - - monkeypatch.setenv("SERVICE_MODE", "api") - monkeypatch.setattr(settings, "SERVICE_MODE", "api", raising=False) - monkeypatch.setattr(settings, "MEDIA_WS_BASE_URL", "", raising=False) - yield - - -@pytest.fixture(autouse=True) -def disable_db_sharding_for_tests(monkeypatch, request): - """Tests use one SQLAlchemy session; ignore production shard routing.""" - if request.node.get_closest_marker("integration"): - yield - return - - from app.config import settings - from app.db_sharding.pool_manager import db_pool_manager - - monkeypatch.setattr(settings, "DB_SHARDING_ENABLED", False) - db_pool_manager.reset() - yield - - -@pytest.fixture(autouse=True) -def ensure_workers_tasks_package(): - """Keep ``app.workers.tasks`` importable without eager Celery imports.""" - import importlib - - workers_pkg = importlib.import_module("app.workers") - tasks_pkg = sys.modules.get("app.workers.tasks") - if tasks_pkg is None: - tasks_pkg = types.ModuleType("app.workers.tasks") - sys.modules["app.workers.tasks"] = tasks_pkg - tasks_pkg.__path__ = [_TASKS_PACKAGE_DIR] - workers_pkg.tasks = tasks_pkg - - helpers_pkg = sys.modules.get("app.workers.tasks.helpers") - if helpers_pkg is None: - helpers_pkg = types.ModuleType("app.workers.tasks.helpers") - sys.modules["app.workers.tasks.helpers"] = helpers_pkg - helpers_pkg.__path__ = [ - str(Path(__file__).resolve().parents[1] / "app" / "workers" / "tasks" / "helpers") - ] - - -@pytest.fixture -def org_id(): - """Stable org UUID for auth-related tests.""" - return uuid4() - - -@pytest.fixture -def seed_org(db_session, org_id): - from app.models.database import Organization - - org = db_session.query(Organization).filter(Organization.id == org_id).first() - if org is None: - org = Organization(id=org_id, name="Test Org") - db_session.add(org) - db_session.commit() - return org - - -@pytest.fixture -def default_workspace(db_session, org_id, seed_org): - from app.models.database import Workspace - - ws = ( - db_session.query(Workspace) - .filter( - Workspace.organization_id == org_id, - Workspace.is_default.is_(True), - ) - .first() - ) - if ws is None: - ws = Workspace( - organization_id=org_id, - name="Default", - slug="default", - is_default=True, - ) - db_session.add(ws) - db_session.commit() - db_session.refresh(ws) - return ws - - -@pytest.fixture -def api_key(): - """Stable API key for authenticated test clients.""" - return "test_api_key_123" - - -def _xdist_worker_id(worker_id: str) -> str | None: - """Return the xdist worker suffix (``gw0``), or None for serial runs.""" - if worker_id in ("master", "main"): - return None - return worker_id - - -def _worker_database_url(base_url: str, worker_id: str) -> str: - """Give each xdist worker its own Postgres database to avoid DDL races.""" - suffix = _xdist_worker_id(worker_id) - if suffix is None: - return base_url - parsed = make_url(base_url) - db_name = parsed.database or "efficientai_test" - return parsed.set(database=f"{db_name}_{suffix}").render_as_string( - hide_password=False - ) - - -def _ensure_postgres_database(admin_url: str, database_name: str) -> None: - """Create ``database_name`` if missing (connects via the admin database).""" - admin = make_url(admin_url).set(database="postgres") - bootstrap = create_engine(admin, isolation_level="AUTOCOMMIT") - try: - with bootstrap.connect() as conn: - exists = conn.execute( - text("SELECT 1 FROM pg_database WHERE datname = :name"), - {"name": database_name}, - ).scalar() - if not exists: - conn.execute(text(f'CREATE DATABASE "{database_name}"')) - finally: - bootstrap.dispose() - - -def _postgres_engine_kwargs(*, parallel: bool) -> dict: - if parallel: - # Many xdist workers × large pools exhaust Postgres connection limits. - return {"pool_pre_ping": True, "pool_size": 1, "max_overflow": 2} - return {"pool_pre_ping": True, "pool_size": 5, "max_overflow": 10} - - -def _bind_runtime_database_url(database_url: str) -> None: - """Keep SessionLocal/db_pool_manager on the same DB as the test engine.""" - os.environ["TEST_DATABASE_URL"] = database_url - os.environ["DATABASE_URL"] = database_url - from app.config import settings - - settings.DATABASE_URL = database_url - - -@pytest.fixture(scope="session") -def test_engine(worker_id): - """ - Database engine used for tests. - Defaults to in-memory SQLite for local speed, but can use a real database - when TEST_DATABASE_URL is provided (for CI/Postgres validation). - Schema is created once per test session and torn down at the end. - - With pytest-xdist, each worker gets its own Postgres database - (``…_gw0``, ``…_gw1``, …) so parallel ``create_all()`` calls do not race - on shared ENUM types. - """ - from app.database import Base - - import app.models.database # noqa: F401 - - base_database_url = os.getenv("TEST_DATABASE_URL", "").strip() - parallel_postgres = bool(base_database_url and _xdist_worker_id(worker_id)) - - if base_database_url: - database_url = _worker_database_url(base_database_url, worker_id) - parsed = make_url(database_url) - if parsed.drivername.startswith("postgresql"): - _ensure_postgres_database(base_database_url, parsed.database) - _bind_runtime_database_url(database_url) - engine = create_engine( - database_url, - **_postgres_engine_kwargs(parallel=parallel_postgres), - ) - drop_schema_on_teardown = not parallel_postgres - else: - engine = create_engine( - "sqlite+pysqlite:///:memory:", - connect_args={"check_same_thread": False}, - poolclass=StaticPool, - ) - drop_schema_on_teardown = True - - Base.metadata.create_all(bind=engine) - try: - yield engine - finally: - if drop_schema_on_teardown: - Base.metadata.drop_all(bind=engine) - engine.dispose() - - -@pytest.fixture -def db_session(test_engine): - """Transaction-scoped SQLAlchemy session; rolls back after each test.""" - connection = test_engine.connect() - transaction = connection.begin() - TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=connection) - session = TestingSessionLocal() - nested = connection.begin_nested() - - @event.listens_for(session, "after_transaction_end") - def _restart_savepoint(sess, trans): # noqa: ARG001 - nonlocal nested - if trans.nested and not trans._parent.nested: - nested = connection.begin_nested() - - try: - yield session - finally: - session.close() - transaction.rollback() - connection.close() - - -_SESSION_API_APP = None -_SESSION_STUBS_READY = False - - -def _install_static_stubs(): - if "python_multipart" not in sys.modules: - fake_python_multipart = types.ModuleType("python_multipart") - fake_python_multipart.__version__ = "0.0.20" - sys.modules["python_multipart"] = fake_python_multipart - - if "multipart" not in sys.modules: - fake_multipart = types.ModuleType("multipart") - fake_multipart.__version__ = "0.0.20" - fake_multipart_submodule = types.ModuleType("multipart.multipart") - fake_multipart_submodule.parse_options_header = lambda *_args, **_kwargs: ("", {}) - sys.modules["multipart"] = fake_multipart - sys.modules["multipart.multipart"] = fake_multipart_submodule - - if "boto3" not in sys.modules: - fake_boto3 = types.ModuleType("boto3") - fake_boto3.client = lambda *_args, **_kwargs: object() - sys.modules["boto3"] = fake_boto3 - - if "botocore.exceptions" not in sys.modules: - fake_botocore = types.ModuleType("botocore") - fake_exceptions = types.ModuleType("botocore.exceptions") - - class _ClientError(Exception): - pass - - class _NoCredentialsError(Exception): - pass - - fake_exceptions.ClientError = _ClientError - fake_exceptions.NoCredentialsError = _NoCredentialsError - fake_botocore.exceptions = fake_exceptions - sys.modules["botocore"] = fake_botocore - sys.modules["botocore.exceptions"] = fake_exceptions - - if "croniter" not in sys.modules: - fake_croniter_module = types.ModuleType("croniter") - - class _FakeCroniter: - def __init__(self, _expression, start_time=None): - self._start_time = start_time - - def get_next(self, _type): - from datetime import timedelta - - if self._start_time is None: - raise ValueError("start_time is required") - return self._start_time + timedelta(minutes=5) - - fake_croniter_module.croniter = _FakeCroniter - sys.modules["croniter"] = fake_croniter_module - - if "pytz" not in sys.modules: - from datetime import timezone as _timezone - - fake_pytz_module = types.ModuleType("pytz") - - class _UnknownTimeZoneError(Exception): - pass - - def _timezone_factory(name): - if not name: - raise _UnknownTimeZoneError("Unknown timezone") - return _timezone.utc - - fake_pytz_module.timezone = _timezone_factory - fake_pytz_module.UTC = _timezone.utc - fake_pytz_module.UnknownTimeZoneError = _UnknownTimeZoneError - sys.modules["pytz"] = fake_pytz_module - - if "app.services.audio" not in sys.modules: - fake_audio_pkg = types.ModuleType("app.services.audio") - fake_audio_pkg.__path__ = [] - fake_audio_service_module = types.ModuleType("app.services.audio.audio_service") - fake_voice_quality_module = types.ModuleType("app.services.audio.voice_quality_service") - - class _FakeAudioService: - def extract_metadata(self, _file_path): - return {"duration": None, "sample_rate": None, "channels": None} - - fake_voice_quality_module.AUDIO_METRICS = [] - fake_voice_quality_module.is_audio_metric = lambda *_args, **_kwargs: False - fake_voice_quality_module.calculate_audio_metrics = lambda *_args, **_kwargs: {} - fake_audio_service_module.AudioService = _FakeAudioService - fake_audio_pkg.audio_service = fake_audio_service_module - fake_audio_pkg.voice_quality_service = fake_voice_quality_module - sys.modules["app.services.audio"] = fake_audio_pkg - sys.modules["app.services.audio.audio_service"] = fake_audio_service_module - sys.modules["app.services.audio.voice_quality_service"] = fake_voice_quality_module - - if "app.services.ai" not in sys.modules: - fake_ai_pkg = types.ModuleType("app.services.ai") - # Point the stubbed package at the real on-disk directory so - # genuinely-needed submodules (like ``llm_resolver``) can still - # be imported from disk even when the rest of the package is - # replaced by light stubs above. - import os as _os - - _real_ai_dir = _os.path.join( - _os.path.dirname(_os.path.dirname(_os.path.abspath(__file__))), - "app", - "services", - "ai", - ) - fake_ai_pkg.__path__ = [_real_ai_dir] - fake_model_config_module = types.ModuleType("app.services.ai.model_config_service") - fake_llm_module = types.ModuleType("app.services.ai.llm_service") - fake_transcription_module = types.ModuleType("app.services.ai.transcription_service") - - class _FakeModelConfigService: - def get_all_models(self): - return {} - - def get_model_config(self, *_args, **_kwargs): - return None - - def get_models_by_provider(self, *_args, **_kwargs): - return [] - - def get_model_options_by_provider(self, *_args, **_kwargs): - return {"stt": [], "llm": [], "tts": [], "s2s": []} - - def get_tts_voices_by_provider(self, *_args, **_kwargs): - return {} - - def get_models_by_type(self, *_args, **_kwargs): - return [] - - def get_voices_for_model(self, *_args, **_kwargs): - return [] - - class _FakeLLMService: - def generate_response(self, *_args, **_kwargs): - return {"text": '{"objective_achieved": false, "overall_score": 0.0}', "usage": {}} - - class _FakeTranscriptionService: - def transcribe(self, *_args, **_kwargs): - return {"transcript": "test transcript", "processing_time": 0.1} - - fake_model_config_module.model_config_service = _FakeModelConfigService() - fake_llm_module.llm_service = _FakeLLMService() - fake_llm_module._resolve_azure_endpoint_from_provider = lambda *_args, **_kwargs: None - fake_transcription_module.transcription_service = _FakeTranscriptionService() - fake_ai_pkg.model_config_service = fake_model_config_module - fake_ai_pkg.llm_service = fake_llm_module - fake_ai_pkg.transcription_service = fake_transcription_module - sys.modules["app.services.ai"] = fake_ai_pkg - sys.modules["app.services.ai.model_config_service"] = fake_model_config_module - sys.modules["app.services.ai.llm_service"] = fake_llm_module - sys.modules["app.services.ai.transcription_service"] = fake_transcription_module - - if "app.services.testing.test_agent_service" not in sys.modules: - fake_testing_pkg = types.ModuleType("app.services.testing") - fake_testing_pkg.__path__ = [] - fake_test_agent_service_module = types.ModuleType("app.services.testing.test_agent_service") - - class _FakeTestAgentService: - def create_conversation(self, *args, **kwargs): # pragma: no cover - overridden in tests - raise ValueError("Not implemented in base test stub") - - def start_conversation(self, *args, **kwargs): # pragma: no cover - overridden in tests - raise ValueError("Not implemented in base test stub") - - def process_audio_chunk(self, *_args, **_kwargs): - return {"transcription": "ok", "metadata": {}, "error": None} - - def end_conversation(self, *args, **kwargs): # pragma: no cover - overridden in tests - raise ValueError("Not implemented in base test stub") - - fake_test_agent_service_module.test_agent_service = _FakeTestAgentService() - fake_testing_pkg.test_agent_service = fake_test_agent_service_module - sys.modules["app.services.testing"] = fake_testing_pkg - sys.modules["app.services.testing.test_agent_service"] = fake_test_agent_service_module - - if "app.services.voice_providers" not in sys.modules: - fake_voice_providers_module = types.ModuleType("app.services.voice_providers") - - class _FakeVoiceProvider: - def __init__(self, *args, **kwargs): - pass - - def create_web_call(self, **_kwargs): - return {"call_id": "fake-call-id"} - - def update_agent_prompt(self, **_kwargs): - return {"ok": True} - - fake_voice_providers_module.get_voice_provider = lambda *_args, **_kwargs: _FakeVoiceProvider - fake_voice_providers_module.sync_provider_prompt = lambda *_args, **_kwargs: {"synced": False} - sys.modules["app.services.voice_providers"] = fake_voice_providers_module - - if "app.services.voice_agent.bot_fast_api" not in sys.modules: - voice_agent_dir = str( - Path(__file__).resolve().parents[1] / "app" / "services" / "voice_agent" - ) - fake_voice_agent_pkg = types.ModuleType("app.services.voice_agent") - fake_voice_agent_pkg.__path__ = [voice_agent_dir] - fake_bot_fast_api_module = types.ModuleType("app.services.voice_agent.bot_fast_api") - fake_voice_bundle_module = types.ModuleType("app.services.voice_agent.voice_bundle") - fake_bot_fast_api_module.run_bot = lambda *_args, **_kwargs: None - fake_voice_bundle_module.run_voice_bundle_fastapi = lambda *_args, **_kwargs: None - sys.modules["app.services.voice_agent"] = fake_voice_agent_pkg - sys.modules["app.services.voice_agent.bot_fast_api"] = fake_bot_fast_api_module - sys.modules["app.services.voice_agent.voice_bundle"] = fake_voice_bundle_module - - if "app.services.reporting.voice_playground_report_service" not in sys.modules: - fake_reporting_pkg = types.ModuleType("app.services.reporting") - fake_reporting_pkg.__path__ = [ - str(Path(__file__).resolve().parents[1] / "app" / "services" / "reporting") - ] - fake_report_service_module = types.ModuleType("app.services.reporting.voice_playground_report_service") - - class _FakeVoicePlaygroundReportService: - def get_threshold_defaults(self, *_args, **_kwargs): - return {} - - def update_threshold_defaults(self, *_args, **_kwargs): - return {} - - fake_report_service_module.voice_playground_report_service = _FakeVoicePlaygroundReportService() - sys.modules["app.services.reporting"] = fake_reporting_pkg - sys.modules["app.services.reporting.voice_playground_report_service"] = fake_report_service_module - - fake_workers_tasks_pkg = sys.modules.get("app.workers.tasks") - if fake_workers_tasks_pkg is None: - fake_workers_tasks_pkg = types.ModuleType("app.workers.tasks") - sys.modules["app.workers.tasks"] = fake_workers_tasks_pkg - fake_workers_tasks_pkg.__path__ = [_TASKS_PACKAGE_DIR] - - # ``app.workers.tasks`` is stubbed with an empty ``__path__`` so Celery - # task modules are not eagerly imported, but several API routes and tests - # still need the real ``helpers`` subpackage (e.g. the diariser default - # prompt endpoint). Register it explicitly so - # ``app.workers.tasks.helpers.llm_diarisation`` resolves normally. - if "app.workers.tasks.helpers" not in sys.modules: - helpers_pkg = types.ModuleType("app.workers.tasks.helpers") - helpers_pkg.__path__ = [ - str( - Path(__file__).resolve().parents[1] - / "app" - / "workers" - / "tasks" - / "helpers" - ) - ] - sys.modules["app.workers.tasks.helpers"] = helpers_pkg - - fake_run_prompt_opt_module = sys.modules.get("app.workers.tasks.run_prompt_optimization") - if fake_run_prompt_opt_module is None: - fake_run_prompt_opt_module = types.ModuleType("app.workers.tasks.run_prompt_optimization") - sys.modules["app.workers.tasks.run_prompt_optimization"] = fake_run_prompt_opt_module - - class _FakePromptOptTask: - def delay(self, *_args, **_kwargs): - class _TaskResult: - id = "fake-prompt-opt-task-id" - - return _TaskResult() - - # Ensure these symbols always exist for API tests, regardless of import order. - fake_run_prompt_opt_module.run_prompt_optimization_task = _FakePromptOptTask() - fake_workers_tasks_pkg.process_evaluation_task = _FakePromptOptTask() - fake_workers_tasks_pkg.process_evaluator_result_task = _FakePromptOptTask() - fake_workers_tasks_pkg.run_evaluator_task = _FakePromptOptTask() - fake_workers_tasks_pkg.generate_tts_comparison_task = _FakePromptOptTask() - fake_workers_tasks_pkg.evaluate_tts_comparison_task = _FakePromptOptTask() - fake_workers_tasks_pkg.generate_tts_report_pdf_task = _FakePromptOptTask() - fake_workers_tasks_pkg.run_prompt_optimization_task = _FakePromptOptTask() - # Required by app.workers.celery_app's eager import block - missing - # these makes any test that imports a route file fail before the - # fixture can install dependency overrides. - fake_workers_tasks_pkg.process_call_import_row_task = _FakePromptOptTask() - fake_workers_tasks_pkg.evaluate_call_import_row_task = _FakePromptOptTask() - fake_workers_tasks_pkg.transcribe_call_import_row_task = _FakePromptOptTask() - fake_workers_tasks_pkg.run_judge_alignment_task = _FakePromptOptTask() - - class _FakeCeleryApp: - """Minimal Celery stand-in for route / worker imports in API tests.""" - - control = types.SimpleNamespace(revoke=lambda *_args, **_kwargs: None) - - def task(self, *_args, **_kwargs): - def _decorator(fn): - fn.delay = lambda *_a, **_kw: types.SimpleNamespace(id="fake-task") - fn.apply_async = lambda *_a, **_kw: types.SimpleNamespace( - id="fake-task" - ) - fn.run = fn - return fn - - return _decorator - - fake_config_module = types.ModuleType("app.workers.config") - fake_config_module.celery_app = _FakeCeleryApp() - sys.modules["app.workers.config"] = fake_config_module - - fake_celery_app_module = types.ModuleType("app.workers.celery_app") - fake_celery_app_module.celery_app = fake_config_module.celery_app - fake_celery_app_module.process_evaluation_task = _FakePromptOptTask() - fake_celery_app_module.process_evaluator_result_task = _FakePromptOptTask() - fake_celery_app_module.run_evaluator_task = _FakePromptOptTask() - fake_celery_app_module.generate_tts_comparison_task = _FakePromptOptTask() - fake_celery_app_module.evaluate_tts_comparison_task = _FakePromptOptTask() - fake_celery_app_module.generate_tts_report_pdf_task = _FakePromptOptTask() - fake_celery_app_module.run_prompt_optimization_task = _FakePromptOptTask() - fake_celery_app_module.process_call_import_row_task = _FakePromptOptTask() - fake_celery_app_module.run_judge_alignment_task = _FakePromptOptTask() - sys.modules["app.workers.celery_app"] = fake_celery_app_module - import importlib - - workers_pkg = importlib.import_module("app.workers") - workers_pkg.celery_app = fake_celery_app_module - -def _wire_bulk_ops_stubs(db_session): - # Bulk call-import tasks: materialize runs synchronously in API tests; - # diarize/delete are no-ops (return immediately). - fake_bulk_ops_module = sys.modules.get("app.workers.tasks.call_import_bulk_ops") - if fake_bulk_ops_module is None: - fake_bulk_ops_module = types.ModuleType("app.workers.tasks.call_import_bulk_ops") - sys.modules["app.workers.tasks.call_import_bulk_ops"] = fake_bulk_ops_module - - def _sync_materialize_delay(evaluation_id, *, transcribe_overwrite=False): - from uuid import UUID - - from app.services.call_imports.bulk_ops import materialize_and_enqueue_evaluation - - materialize_and_enqueue_evaluation( - db_session, - UUID(evaluation_id), - transcribe_overwrite=transcribe_overwrite, - ) - return types.SimpleNamespace(id="fake-sync-bulk-task") - - class _NoopBulkTask: - @staticmethod - def delay(*_args, **_kwargs): - return types.SimpleNamespace(id="fake-sync-bulk-task") - - fake_bulk_ops_module.materialize_call_import_evaluation_task = types.SimpleNamespace( - delay=_sync_materialize_delay - ) - - def _sync_mapped_materialize_delay( - call_import_id, - organization_id, - workspace_id, - evaluation_id, - *, - transcribe_overwrite=False, - ): - from uuid import UUID - - from app.services.call_imports.bulk_ops import ( - execute_call_import_materialization, - materialize_and_enqueue_evaluation, - ) - - mat_result = execute_call_import_materialization( - db_session, - UUID(call_import_id), - UUID(organization_id), - UUID(workspace_id), - schedule_import_dispatch=False, - ) - if mat_result.get("status") != "failed": - materialize_and_enqueue_evaluation( - db_session, - UUID(evaluation_id), - transcribe_overwrite=transcribe_overwrite, - ) - return types.SimpleNamespace(id="fake-sync-mapped-bulk-task") - - fake_bulk_ops_module.materialize_mapped_call_import_evaluation_task = ( - types.SimpleNamespace(delay=_sync_mapped_materialize_delay) - ) - - def _sync_import_materialize_delay(call_import_id, organization_id, workspace_id): - from uuid import UUID - - from app.services.call_imports.bulk_ops import execute_call_import_materialization - - execute_call_import_materialization( - db_session, - UUID(call_import_id), - UUID(organization_id), - UUID(workspace_id), - ) - return types.SimpleNamespace(id="fake-sync-bulk-task") - - fake_bulk_ops_module.materialize_call_import_rows_task = types.SimpleNamespace( - delay=_sync_import_materialize_delay - ) - fake_bulk_ops_module.bulk_diarize_call_import_task = _NoopBulkTask() - fake_bulk_ops_module.bulk_delete_call_import_rows_task = _NoopBulkTask() - fake_bulk_ops_module.delete_call_import_task = _NoopBulkTask() - - def _sync_retry_delay(evaluation_id, payload_dict): - from uuid import UUID - - eval_row_ids_raw = payload_dict.get("eval_row_ids") - metric_ids_raw = payload_dict.get("metric_ids") - from app.services.call_imports.bulk_ops import execute_evaluation_retry - - execute_evaluation_retry( - db_session, - UUID(evaluation_id), - eval_row_ids=( - [UUID(rid) for rid in eval_row_ids_raw] - if eval_row_ids_raw - else None - ), - metric_ids=( - [UUID(mid) for mid in metric_ids_raw] if metric_ids_raw else None - ), - include_completed=bool(payload_dict.get("include_completed", False)), - transcribe_overwrite=bool(payload_dict.get("transcribe_overwrite", False)), - ) - return types.SimpleNamespace(id="fake-sync-retry-task") - - fake_bulk_ops_module.retry_call_import_evaluation_task = types.SimpleNamespace( - delay=_sync_retry_delay - ) - - def _sync_cancel_delay(evaluation_id, *, mode): - from uuid import UUID - - from app.services.call_imports.bulk_ops import execute_evaluation_cancel - - execute_evaluation_cancel(db_session, UUID(evaluation_id), mode=mode) - return types.SimpleNamespace(id="fake-sync-cancel-task") - - fake_bulk_ops_module.cancel_call_import_evaluation_task = types.SimpleNamespace( - delay=_sync_cancel_delay - ) - -def _install_concurrency_stubs(): - fake_fair_dispatch_module = sys.modules.get("app.workers.concurrency.fair_dispatch") - if fake_fair_dispatch_module is None: - fake_fair_dispatch_module = types.ModuleType( - "app.workers.concurrency.fair_dispatch" - ) - sys.modules["app.workers.concurrency.fair_dispatch"] = ( - fake_fair_dispatch_module - ) - if not hasattr(fake_fair_dispatch_module, "schedule_fair_dispatch"): - fake_fair_dispatch_module.schedule_fair_dispatch = lambda *_a, **_kw: None - if not hasattr(fake_fair_dispatch_module, "store_row_restricted_metrics"): - fake_fair_dispatch_module.store_row_restricted_metrics = lambda *_a, **_kw: None - if not hasattr(fake_fair_dispatch_module, "store_evaluation_transcribe_overwrite"): - fake_fair_dispatch_module.store_evaluation_transcribe_overwrite = ( - lambda *_a, **_kw: None - ) - if not hasattr(fake_fair_dispatch_module, "read_fair_dispatch_state"): - fake_fair_dispatch_module.read_fair_dispatch_state = lambda: { - "global_rr_cursor": 0, - "dispatch_dedupe_active": False, - "dispatch_queue": "celery", - "at_capacity_backoff_seconds": 15, - } - if not hasattr(fake_fair_dispatch_module, "read_workspace_eval_rr_cursor"): - fake_fair_dispatch_module.read_workspace_eval_rr_cursor = lambda _ws_id: 0 - if not hasattr(fake_fair_dispatch_module, "finish_eval_work_and_redispatch"): - fake_fair_dispatch_module.finish_eval_work_and_redispatch = ( - lambda *_a, **_kw: None - ) - - def _ensure_concurrency_submodule(name: str) -> types.ModuleType: - full_name = f"app.workers.concurrency.{name}" - module = sys.modules.get(full_name) - if module is None: - module = types.ModuleType(full_name) - sys.modules[full_name] = module - return module - - for submodule, attrs in { - "fair_diarization_dispatch": ( - "finish_diarization_work_and_redispatch", - "schedule_fair_diarization_dispatch", - ), - "fair_import_dispatch": ( - "finish_import_work_and_redispatch", - "schedule_fair_import_dispatch", - ), - "diarization_dispatch": ( - "build_diarization_params_from_request", - "store_row_diarization_params", - ), - "limits": ( - "acquire_eval_slot", - "release_eval_slot_for_celery_task", - "slot_registered_for_task", - ), - }.items(): - mod = _ensure_concurrency_submodule(submodule) - for attr in attrs: - if not hasattr(mod, attr): - setattr(mod, attr, lambda *_a, **_kw: None) - - limits_mod = _ensure_concurrency_submodule("limits") - for attr in ( - "read_global_inflight", - "read_org_inflight", - "read_workspace_inflight", - "read_job_inflight", - ): - if not hasattr(limits_mod, attr): - setattr(limits_mod, attr, lambda *_a, **_kw: 0) - - fake_eval_dispatch_module = sys.modules.get("app.workers.concurrency.eval_dispatch") - if fake_eval_dispatch_module is None: - fake_eval_dispatch_module = types.ModuleType( - "app.workers.concurrency.eval_dispatch" - ) - sys.modules["app.workers.concurrency.eval_dispatch"] = fake_eval_dispatch_module - for queue_name in ("DIARIZATION_QUEUE", "EVALUATIONS_QUEUE", "IMPORTS_QUEUE"): - if not hasattr(fake_eval_dispatch_module, queue_name): - setattr( - fake_eval_dispatch_module, - queue_name, - queue_name.replace("_QUEUE", "").lower(), - ) - if not hasattr(fake_eval_dispatch_module, "schedule_evaluation_dispatch"): - fake_eval_dispatch_module.schedule_evaluation_dispatch = lambda *_a, **_kw: None - - -def _build_session_api_app(): - import app.dependencies as app_dependencies - from app.api.v1.routes import ( - aiproviders, - agents, - alerts, - audio, - auth, - llm_gateway, - call_import_evaluations, - call_import_schemas, - call_import_tags, - call_imports, - chat, - conversation_evaluations, - cron_jobs, - data_sources, - evaluations, - evaluator_results, - evaluators, - evaluator_suites, - iam, - integrations, - manual_evaluations, - metrics, - model_config, - observability, - personas, - playground, - profile, - prompt_optimization, - prompt_partials, - results, - scenarios, - settings, - telephony, - test_agents, - voice_agent, - voice_playground, - voicebundles, - vobiz_telephony, - workspaces, - workspace_iam, - ) - - app = FastAPI() - app.include_router(auth.router, prefix="/api/v1") - app.include_router(evaluations.router, prefix="/api/v1") - app.include_router(results.router, prefix="/api/v1") - app.include_router(agents.router, prefix="/api/v1") - app.include_router(evaluators.router, prefix="/api/v1") - app.include_router(evaluator_suites.router, prefix="/api/v1") - app.include_router(personas.router, prefix="/api/v1") - app.include_router(scenarios.router, prefix="/api/v1") - app.include_router(settings.router, prefix="/api/v1") - app.include_router(iam.router, prefix="/api/v1") - app.include_router(audio.router, prefix="/api/v1") - app.include_router(integrations.router, prefix="/api/v1") - app.include_router(aiproviders.router, prefix="/api/v1") - app.include_router(llm_gateway.router, prefix="/api/v1") - app.include_router(metrics.router, prefix="/api/v1") - app.include_router(evaluator_results.router, prefix="/api/v1") - app.include_router(voicebundles.router, prefix="/api/v1") - app.include_router(test_agents.router, prefix="/api/v1") - app.include_router(manual_evaluations.router, prefix="/api/v1") - app.include_router(conversation_evaluations.router, prefix="/api/v1") - app.include_router(alerts.router, prefix="/api/v1") - app.include_router(model_config.router, prefix="/api/v1") - app.include_router(data_sources.router, prefix="/api/v1") - app.include_router(chat.router, prefix="/api/v1") - app.include_router(prompt_partials.router, prefix="/api/v1") - app.include_router(cron_jobs.router, prefix="/api/v1") - app.include_router(profile.router, prefix="/api/v1") - app.include_router(observability.router, prefix="/api/v1") - app.include_router(playground.router, prefix="/api/v1") - app.include_router(prompt_optimization.router, prefix="/api/v1") - app.include_router(voice_agent.router, prefix="/api/v1") - app.include_router(voice_playground.router, prefix="/api/v1") - app.include_router(telephony.router, prefix="/api/v1") - app.include_router(vobiz_telephony.router, prefix="/api/v1") - app.include_router(call_imports.router, prefix="/api/v1") - app.include_router(call_import_schemas.router, prefix="/api/v1") - app.include_router(call_import_tags.router, prefix="/api/v1") - app.include_router(call_import_evaluations.router, prefix="/api/v1") - app.include_router(workspaces.router, prefix="/api/v1") - app.include_router(workspace_iam.router, prefix="/api/v1") - # Enterprise route dependencies call app.dependencies.is_feature_enabled at runtime. - # Force-enable it for API tests so tests remain focused on route behavior. - app_dependencies.is_feature_enabled = lambda *_args, **_kwargs: True - - @asynccontextmanager - async def _noop_lifespan(_: object): - yield - - app.router.lifespan_context = _noop_lifespan - return app - -@pytest.fixture -def client(db_session, api_key, org_id): - """ - FastAPI client with DB/auth dependency overrides and no startup lifespan. - This avoids running migrations in test bootstrap. - """ - global _SESSION_API_APP, _SESSION_STUBS_READY - - if not _SESSION_STUBS_READY: - _install_static_stubs() - _install_concurrency_stubs() - _SESSION_API_APP = _build_session_api_app() - _SESSION_STUBS_READY = True - - _wire_bulk_ops_stubs(db_session) - - from app.database import get_db - from app.dependencies import ( - get_api_key, - get_organization_id, - get_workspace_context, - get_workspace_id, - require_enterprise_feature, - WorkspaceContext, - ) - from app.core.auth.capabilities import ALL_CAPABILITIES - from app.models.database import Organization, Workspace - from app.services.workspace_rbac import backfill_org_workspace_memberships, seed_system_workspace_roles - - app = _SESSION_API_APP - - def _override_workspace_context() -> WorkspaceContext: - return WorkspaceContext( - workspace_id=default_workspace.id, - organization_id=org_id, - capabilities=frozenset(ALL_CAPABILITIES), - is_org_admin=True, - ) - # The TestClient flow doesn't run migration 033, so we manually - # ensure the test org has a Default workspace before any route - # that depends on ``get_workspace_id`` runs. This mirrors what the - # real migration would have produced. - def _ensure_default_workspace() -> Workspace: - org = ( - db_session.query(Organization) - .filter(Organization.id == org_id) - .first() - ) - if org is None: - org = Organization(id=org_id, name="Test Organization") - db_session.add(org) - db_session.flush() - ws = ( - db_session.query(Workspace) - .filter( - Workspace.organization_id == org_id, - Workspace.is_default.is_(True), - ) - .first() - ) - if ws is None: - ws = Workspace( - organization_id=org_id, - name="Default", - slug="default", - is_default=True, - ) - db_session.add(ws) - db_session.commit() - seed_system_workspace_roles(db_session, organization_id=org_id) - return ws - - default_workspace = _ensure_default_workspace() - backfill_org_workspace_memberships(db_session, organization_id=org_id) - - def _override_get_db(): - yield db_session - - app.dependency_overrides[get_db] = _override_get_db - app.dependency_overrides[get_api_key] = lambda: api_key - app.dependency_overrides[get_organization_id] = lambda: org_id - app.dependency_overrides[get_workspace_id] = lambda: default_workspace.id - app.dependency_overrides[get_workspace_context] = _override_workspace_context - app.dependency_overrides[require_enterprise_feature] = lambda: None - - with TestClient(app) as test_client: - yield test_client - - app.dependency_overrides.clear() - - -@pytest.fixture -def telephony_client(db_session): - """Telephony edge TestClient (Vobiz carrier webhooks + media WebSocket routes only).""" - from contextlib import asynccontextmanager - - from fastapi import FastAPI - from fastapi.testclient import TestClient - - from app.api.v1.routes import vobiz_telephony - from app.database import get_db - - app = FastAPI() - - @asynccontextmanager - async def _noop_lifespan(_: object): - yield - - app.router.lifespan_context = _noop_lifespan - app.include_router(vobiz_telephony.webhook_router, prefix="/api/v1") - app.include_router(vobiz_telephony.ws_router, prefix="/api/v1") - - def _override_get_db(): - yield db_session - - app.dependency_overrides[get_db] = _override_get_db - - with TestClient(app) as test_client: - yield test_client - - app.dependency_overrides.clear() - - -@pytest.fixture -def authenticated_client(client, api_key, db_session, org_id): - """Client pre-populated with auth header and a real API key in the DB.""" - from app.models.database import APIKey, Organization, OrganizationMember, RoleEnum, User - - existing_org = db_session.query(Organization).filter(Organization.id == org_id).first() - if existing_org is None: - db_session.add(Organization(id=org_id, name="Test Organization")) - db_session.flush() - - existing_key = ( - db_session.query(APIKey) - .filter(APIKey.key == api_key, APIKey.organization_id == org_id) - .first() - ) - if existing_key is None: - user = User( - id=uuid4(), - email="owner@example.com", - name="Org Owner", - is_active=True, - ) - db_session.add(user) - db_session.flush() - db_session.add( - OrganizationMember( - organization_id=org_id, - user_id=user.id, - role=RoleEnum.ADMIN.value, - ) - ) - db_session.add( - APIKey( - id=uuid4(), - key=api_key, - name="Test API Key", - organization_id=org_id, - user_id=user.id, - is_active=True, - ) - ) - db_session.commit() - - client.headers.update({"X-API-Key": api_key}) - return client - - -@pytest.fixture -def payload_factory(): - """Factory helpers for common API payload shapes.""" - - def _agent_payload(**overrides): - payload = { - "name": "Test Agent", - "phone_number": "+1234567890", - "language": "en", - "description": "This is a test agent description with enough words to pass validation.", - "call_type": "outbound", - "call_medium": "phone_call", - "voice_ai_integration_id": str(uuid4()), - "voice_ai_agent_id": "agent_123", - } - payload.update(overrides) - return payload - - def _persona_payload(**overrides): - payload = { - "name": "Test Persona", - "gender": "neutral", - "is_custom": False, - } - payload.update(overrides) - return payload - - def _scenario_payload(**overrides): - payload = { - "name": "Test Scenario", - "description": "Simple test scenario for backend API tests.", - } - payload.update(overrides) - return payload - - def _evaluation_payload(**overrides): - payload = { - "audio_id": str(uuid4()), - "evaluation_type": "asr", - "metrics": ["wer", "latency"], - } - payload.update(overrides) - return payload - - return { - "agent": _agent_payload, - "persona": _persona_payload, - "scenario": _scenario_payload, - "evaluation": _evaluation_payload, - } +"""Shared pytest fixtures for backend tests.""" + +import os +import sys +import types +from contextlib import asynccontextmanager +from pathlib import Path +from uuid import uuid4 + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from sqlalchemy import create_engine, event, text +from sqlalchemy.engine import make_url +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +# Some local environments provide ALLOWED_AUDIO_FORMATS as a non-JSON string, +# which breaks pydantic-settings parsing during module import in tests. +os.environ["ALLOWED_AUDIO_FORMATS"] = '["wav","mp3","flac","m4a"]' +# Ensure storage service singletons can initialize in test environments. +os.environ["UPLOAD_DIR"] = "/tmp/efficientai-test-uploads" +# Local dev often sets SERVICE_MODE=media and config.yml media URLs; keep API tests on full app mode. +os.environ["SERVICE_MODE"] = "api" + +_REPO_ROOT = Path(__file__).resolve().parents[1] +_SRC_ROOT = _REPO_ROOT / "src" +for _path in (str(_REPO_ROOT), str(_SRC_ROOT)): + if _path not in sys.path: + sys.path.insert(0, _path) + +_TASKS_PACKAGE_DIR = str( + Path(__file__).resolve().parents[1] / "app" / "workers" / "tasks" +) + + +@pytest.fixture(autouse=True) +def isolate_service_mode_for_app_factory(monkeypatch): + """Prevent local SERVICE_MODE / media URL config from breaking create_app tests.""" + from app.config import settings + + monkeypatch.setenv("SERVICE_MODE", "api") + monkeypatch.setattr(settings, "SERVICE_MODE", "api", raising=False) + monkeypatch.setattr(settings, "MEDIA_WS_BASE_URL", "", raising=False) + yield + + +@pytest.fixture(autouse=True) +def disable_db_sharding_for_tests(monkeypatch, request): + """Tests use one SQLAlchemy session; ignore production shard routing.""" + if request.node.get_closest_marker("integration"): + yield + return + + from app.config import settings + from app.db_sharding.pool_manager import db_pool_manager + + monkeypatch.setattr(settings, "DB_SHARDING_ENABLED", False) + db_pool_manager.reset() + yield + + +@pytest.fixture(autouse=True) +def ensure_workers_tasks_package(): + """Keep ``app.workers.tasks`` importable without eager Celery imports.""" + import importlib + + workers_pkg = importlib.import_module("app.workers") + tasks_pkg = sys.modules.get("app.workers.tasks") + if tasks_pkg is None: + tasks_pkg = types.ModuleType("app.workers.tasks") + sys.modules["app.workers.tasks"] = tasks_pkg + tasks_pkg.__path__ = [_TASKS_PACKAGE_DIR] + workers_pkg.tasks = tasks_pkg + + helpers_pkg = sys.modules.get("app.workers.tasks.helpers") + if helpers_pkg is None: + helpers_pkg = types.ModuleType("app.workers.tasks.helpers") + sys.modules["app.workers.tasks.helpers"] = helpers_pkg + helpers_pkg.__path__ = [ + str(Path(__file__).resolve().parents[1] / "app" / "workers" / "tasks" / "helpers") + ] + + +@pytest.fixture +def org_id(): + """Stable org UUID for auth-related tests.""" + return uuid4() + + +@pytest.fixture +def seed_org(db_session, org_id): + from app.models.database import Organization + + org = db_session.query(Organization).filter(Organization.id == org_id).first() + if org is None: + org = Organization(id=org_id, name="Test Org") + db_session.add(org) + db_session.commit() + return org + + +@pytest.fixture +def default_workspace(db_session, org_id, seed_org): + from app.models.database import Workspace + + ws = ( + db_session.query(Workspace) + .filter( + Workspace.organization_id == org_id, + Workspace.is_default.is_(True), + ) + .first() + ) + if ws is None: + ws = Workspace( + organization_id=org_id, + name="Default", + slug="default", + is_default=True, + ) + db_session.add(ws) + db_session.commit() + db_session.refresh(ws) + return ws + + +@pytest.fixture +def api_key(): + """Stable API key for authenticated test clients.""" + return "test_api_key_123" + + +def _xdist_worker_id(worker_id: str) -> str | None: + """Return the xdist worker suffix (``gw0``), or None for serial runs.""" + if worker_id in ("master", "main"): + return None + return worker_id + + +def _worker_database_url(base_url: str, worker_id: str) -> str: + """Give each xdist worker its own Postgres database to avoid DDL races.""" + suffix = _xdist_worker_id(worker_id) + if suffix is None: + return base_url + parsed = make_url(base_url) + db_name = parsed.database or "efficientai_test" + return parsed.set(database=f"{db_name}_{suffix}").render_as_string( + hide_password=False + ) + + +def _ensure_postgres_database(admin_url: str, database_name: str) -> None: + """Create ``database_name`` if missing (connects via the admin database).""" + admin = make_url(admin_url).set(database="postgres") + bootstrap = create_engine(admin, isolation_level="AUTOCOMMIT") + try: + with bootstrap.connect() as conn: + exists = conn.execute( + text("SELECT 1 FROM pg_database WHERE datname = :name"), + {"name": database_name}, + ).scalar() + if not exists: + conn.execute(text(f'CREATE DATABASE "{database_name}"')) + finally: + bootstrap.dispose() + + +def _postgres_engine_kwargs(*, parallel: bool) -> dict: + if parallel: + # Many xdist workers × large pools exhaust Postgres connection limits. + return {"pool_pre_ping": True, "pool_size": 1, "max_overflow": 2} + return {"pool_pre_ping": True, "pool_size": 5, "max_overflow": 10} + + +def _bind_runtime_database_url(database_url: str) -> None: + """Keep SessionLocal/db_pool_manager on the same DB as the test engine.""" + os.environ["TEST_DATABASE_URL"] = database_url + os.environ["DATABASE_URL"] = database_url + from app.config import settings + + settings.DATABASE_URL = database_url + + +@pytest.fixture(scope="session") +def test_engine(worker_id): + """ + Database engine used for tests. + Defaults to in-memory SQLite for local speed, but can use a real database + when TEST_DATABASE_URL is provided (for CI/Postgres validation). + Schema is created once per test session and torn down at the end. + + With pytest-xdist, each worker gets its own Postgres database + (``…_gw0``, ``…_gw1``, …) so parallel ``create_all()`` calls do not race + on shared ENUM types. + """ + from app.database import Base + + import app.models.database # noqa: F401 + + base_database_url = os.getenv("TEST_DATABASE_URL", "").strip() + parallel_postgres = bool(base_database_url and _xdist_worker_id(worker_id)) + + if base_database_url: + database_url = _worker_database_url(base_database_url, worker_id) + parsed = make_url(database_url) + if parsed.drivername.startswith("postgresql"): + _ensure_postgres_database(base_database_url, parsed.database) + _bind_runtime_database_url(database_url) + engine = create_engine( + database_url, + **_postgres_engine_kwargs(parallel=parallel_postgres), + ) + drop_schema_on_teardown = not parallel_postgres + else: + engine = create_engine( + "sqlite+pysqlite:///:memory:", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + drop_schema_on_teardown = True + + Base.metadata.create_all(bind=engine) + try: + yield engine + finally: + if drop_schema_on_teardown: + Base.metadata.drop_all(bind=engine) + engine.dispose() + + +@pytest.fixture +def db_session(test_engine): + """Transaction-scoped SQLAlchemy session; rolls back after each test.""" + connection = test_engine.connect() + transaction = connection.begin() + TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=connection) + session = TestingSessionLocal() + nested = connection.begin_nested() + + @event.listens_for(session, "after_transaction_end") + def _restart_savepoint(sess, trans): # noqa: ARG001 + nonlocal nested + if trans.nested and not trans._parent.nested: + nested = connection.begin_nested() + + try: + yield session + finally: + session.close() + transaction.rollback() + connection.close() + + +_SESSION_API_APP = None +_SESSION_STUBS_READY = False + + +def _install_static_stubs(): + if "python_multipart" not in sys.modules: + fake_python_multipart = types.ModuleType("python_multipart") + fake_python_multipart.__version__ = "0.0.20" + sys.modules["python_multipart"] = fake_python_multipart + + if "multipart" not in sys.modules: + fake_multipart = types.ModuleType("multipart") + fake_multipart.__version__ = "0.0.20" + fake_multipart_submodule = types.ModuleType("multipart.multipart") + fake_multipart_submodule.parse_options_header = lambda *_args, **_kwargs: ("", {}) + sys.modules["multipart"] = fake_multipart + sys.modules["multipart.multipart"] = fake_multipart_submodule + + if "boto3" not in sys.modules: + fake_boto3 = types.ModuleType("boto3") + fake_boto3.client = lambda *_args, **_kwargs: object() + sys.modules["boto3"] = fake_boto3 + + if "botocore.exceptions" not in sys.modules: + fake_botocore = types.ModuleType("botocore") + fake_exceptions = types.ModuleType("botocore.exceptions") + + class _ClientError(Exception): + pass + + class _NoCredentialsError(Exception): + pass + + fake_exceptions.ClientError = _ClientError + fake_exceptions.NoCredentialsError = _NoCredentialsError + fake_botocore.exceptions = fake_exceptions + sys.modules["botocore"] = fake_botocore + sys.modules["botocore.exceptions"] = fake_exceptions + + if "croniter" not in sys.modules: + fake_croniter_module = types.ModuleType("croniter") + + class _FakeCroniter: + def __init__(self, _expression, start_time=None): + self._start_time = start_time + + def get_next(self, _type): + from datetime import timedelta + + if self._start_time is None: + raise ValueError("start_time is required") + return self._start_time + timedelta(minutes=5) + + fake_croniter_module.croniter = _FakeCroniter + sys.modules["croniter"] = fake_croniter_module + + if "pytz" not in sys.modules: + from datetime import timezone as _timezone + + fake_pytz_module = types.ModuleType("pytz") + + class _UnknownTimeZoneError(Exception): + pass + + def _timezone_factory(name): + if not name: + raise _UnknownTimeZoneError("Unknown timezone") + return _timezone.utc + + fake_pytz_module.timezone = _timezone_factory + fake_pytz_module.UTC = _timezone.utc + fake_pytz_module.UnknownTimeZoneError = _UnknownTimeZoneError + sys.modules["pytz"] = fake_pytz_module + + if "app.services.audio" not in sys.modules: + fake_audio_pkg = types.ModuleType("app.services.audio") + fake_audio_pkg.__path__ = [] + fake_audio_service_module = types.ModuleType("app.services.audio.audio_service") + fake_voice_quality_module = types.ModuleType("app.services.audio.voice_quality_service") + + class _FakeAudioService: + def extract_metadata(self, _file_path): + return {"duration": None, "sample_rate": None, "channels": None} + + fake_voice_quality_module.AUDIO_METRICS = [] + fake_voice_quality_module.is_audio_metric = lambda *_args, **_kwargs: False + fake_voice_quality_module.calculate_audio_metrics = lambda *_args, **_kwargs: {} + fake_audio_service_module.AudioService = _FakeAudioService + fake_audio_pkg.audio_service = fake_audio_service_module + fake_audio_pkg.voice_quality_service = fake_voice_quality_module + sys.modules["app.services.audio"] = fake_audio_pkg + sys.modules["app.services.audio.audio_service"] = fake_audio_service_module + sys.modules["app.services.audio.voice_quality_service"] = fake_voice_quality_module + + if "app.services.ai" not in sys.modules: + fake_ai_pkg = types.ModuleType("app.services.ai") + # Point the stubbed package at the real on-disk directory so + # genuinely-needed submodules (like ``llm_resolver``) can still + # be imported from disk even when the rest of the package is + # replaced by light stubs above. + import os as _os + + _real_ai_dir = _os.path.join( + _os.path.dirname(_os.path.dirname(_os.path.abspath(__file__))), + "app", + "services", + "ai", + ) + fake_ai_pkg.__path__ = [_real_ai_dir] + fake_model_config_module = types.ModuleType("app.services.ai.model_config_service") + fake_llm_module = types.ModuleType("app.services.ai.llm_service") + fake_transcription_module = types.ModuleType("app.services.ai.transcription_service") + + class _FakeModelConfigService: + def get_all_models(self): + return {} + + def get_model_config(self, *_args, **_kwargs): + return None + + def get_models_by_provider(self, *_args, **_kwargs): + return [] + + def get_model_options_by_provider(self, *_args, **_kwargs): + return {"stt": [], "llm": [], "tts": [], "s2s": []} + + def get_tts_voices_by_provider(self, *_args, **_kwargs): + return {} + + def get_models_by_type(self, *_args, **_kwargs): + return [] + + def get_voices_for_model(self, *_args, **_kwargs): + return [] + + class _FakeLLMService: + def generate_response(self, *_args, **_kwargs): + return {"text": '{"objective_achieved": false, "overall_score": 0.0}', "usage": {}} + + class _FakeTranscriptionService: + def transcribe(self, *_args, **_kwargs): + return {"transcript": "test transcript", "processing_time": 0.1} + + fake_model_config_module.model_config_service = _FakeModelConfigService() + fake_llm_module.llm_service = _FakeLLMService() + fake_llm_module._resolve_azure_endpoint_from_provider = lambda *_args, **_kwargs: None + fake_transcription_module.transcription_service = _FakeTranscriptionService() + fake_ai_pkg.model_config_service = fake_model_config_module + fake_ai_pkg.llm_service = fake_llm_module + fake_ai_pkg.transcription_service = fake_transcription_module + sys.modules["app.services.ai"] = fake_ai_pkg + sys.modules["app.services.ai.model_config_service"] = fake_model_config_module + sys.modules["app.services.ai.llm_service"] = fake_llm_module + sys.modules["app.services.ai.transcription_service"] = fake_transcription_module + + if "app.services.testing.test_agent_service" not in sys.modules: + fake_testing_pkg = types.ModuleType("app.services.testing") + fake_testing_pkg.__path__ = [] + fake_test_agent_service_module = types.ModuleType("app.services.testing.test_agent_service") + + class _FakeTestAgentService: + def create_conversation(self, *args, **kwargs): # pragma: no cover - overridden in tests + raise ValueError("Not implemented in base test stub") + + def start_conversation(self, *args, **kwargs): # pragma: no cover - overridden in tests + raise ValueError("Not implemented in base test stub") + + def process_audio_chunk(self, *_args, **_kwargs): + return {"transcription": "ok", "metadata": {}, "error": None} + + def end_conversation(self, *args, **kwargs): # pragma: no cover - overridden in tests + raise ValueError("Not implemented in base test stub") + + fake_test_agent_service_module.test_agent_service = _FakeTestAgentService() + fake_testing_pkg.test_agent_service = fake_test_agent_service_module + sys.modules["app.services.testing"] = fake_testing_pkg + sys.modules["app.services.testing.test_agent_service"] = fake_test_agent_service_module + + if "app.services.voice_providers" not in sys.modules: + fake_voice_providers_module = types.ModuleType("app.services.voice_providers") + + class _FakeVoiceProvider: + def __init__(self, *args, **kwargs): + pass + + def create_web_call(self, **_kwargs): + return {"call_id": "fake-call-id"} + + def update_agent_prompt(self, **_kwargs): + return {"ok": True} + + fake_voice_providers_module.get_voice_provider = lambda *_args, **_kwargs: _FakeVoiceProvider + fake_voice_providers_module.sync_provider_prompt = lambda *_args, **_kwargs: {"synced": False} + sys.modules["app.services.voice_providers"] = fake_voice_providers_module + + if "app.services.voice_agent.bot_fast_api" not in sys.modules: + voice_agent_dir = str( + Path(__file__).resolve().parents[1] / "app" / "services" / "voice_agent" + ) + fake_voice_agent_pkg = types.ModuleType("app.services.voice_agent") + fake_voice_agent_pkg.__path__ = [voice_agent_dir] + fake_bot_fast_api_module = types.ModuleType("app.services.voice_agent.bot_fast_api") + fake_voice_bundle_module = types.ModuleType("app.services.voice_agent.voice_bundle") + fake_bot_fast_api_module.run_bot = lambda *_args, **_kwargs: None + fake_voice_bundle_module.run_voice_bundle_fastapi = lambda *_args, **_kwargs: None + sys.modules["app.services.voice_agent"] = fake_voice_agent_pkg + sys.modules["app.services.voice_agent.bot_fast_api"] = fake_bot_fast_api_module + sys.modules["app.services.voice_agent.voice_bundle"] = fake_voice_bundle_module + + if "app.services.reporting.voice_playground_report_service" not in sys.modules: + fake_reporting_pkg = types.ModuleType("app.services.reporting") + fake_reporting_pkg.__path__ = [ + str(Path(__file__).resolve().parents[1] / "app" / "services" / "reporting") + ] + fake_report_service_module = types.ModuleType("app.services.reporting.voice_playground_report_service") + + class _FakeVoicePlaygroundReportService: + def get_threshold_defaults(self, *_args, **_kwargs): + return {} + + def update_threshold_defaults(self, *_args, **_kwargs): + return {} + + fake_report_service_module.voice_playground_report_service = _FakeVoicePlaygroundReportService() + sys.modules["app.services.reporting"] = fake_reporting_pkg + sys.modules["app.services.reporting.voice_playground_report_service"] = fake_report_service_module + + fake_workers_tasks_pkg = sys.modules.get("app.workers.tasks") + if fake_workers_tasks_pkg is None: + fake_workers_tasks_pkg = types.ModuleType("app.workers.tasks") + sys.modules["app.workers.tasks"] = fake_workers_tasks_pkg + fake_workers_tasks_pkg.__path__ = [_TASKS_PACKAGE_DIR] + + # ``app.workers.tasks`` is stubbed with an empty ``__path__`` so Celery + # task modules are not eagerly imported, but several API routes and tests + # still need the real ``helpers`` subpackage (e.g. the diariser default + # prompt endpoint). Register it explicitly so + # ``app.workers.tasks.helpers.llm_diarisation`` resolves normally. + if "app.workers.tasks.helpers" not in sys.modules: + helpers_pkg = types.ModuleType("app.workers.tasks.helpers") + helpers_pkg.__path__ = [ + str( + Path(__file__).resolve().parents[1] + / "app" + / "workers" + / "tasks" + / "helpers" + ) + ] + sys.modules["app.workers.tasks.helpers"] = helpers_pkg + + fake_run_prompt_opt_module = sys.modules.get("app.workers.tasks.run_prompt_optimization") + if fake_run_prompt_opt_module is None: + fake_run_prompt_opt_module = types.ModuleType("app.workers.tasks.run_prompt_optimization") + sys.modules["app.workers.tasks.run_prompt_optimization"] = fake_run_prompt_opt_module + + class _FakePromptOptTask: + def delay(self, *_args, **_kwargs): + class _TaskResult: + id = "fake-prompt-opt-task-id" + + return _TaskResult() + + # Ensure these symbols always exist for API tests, regardless of import order. + fake_run_prompt_opt_module.run_prompt_optimization_task = _FakePromptOptTask() + fake_workers_tasks_pkg.process_evaluation_task = _FakePromptOptTask() + fake_workers_tasks_pkg.process_evaluator_result_task = _FakePromptOptTask() + fake_workers_tasks_pkg.run_evaluator_task = _FakePromptOptTask() + fake_workers_tasks_pkg.generate_tts_comparison_task = _FakePromptOptTask() + fake_workers_tasks_pkg.evaluate_tts_comparison_task = _FakePromptOptTask() + fake_workers_tasks_pkg.generate_tts_report_pdf_task = _FakePromptOptTask() + fake_workers_tasks_pkg.run_prompt_optimization_task = _FakePromptOptTask() + # Required by app.workers.celery_app's eager import block - missing + # these makes any test that imports a route file fail before the + # fixture can install dependency overrides. + fake_workers_tasks_pkg.process_call_import_row_task = _FakePromptOptTask() + fake_workers_tasks_pkg.evaluate_call_import_row_task = _FakePromptOptTask() + fake_workers_tasks_pkg.transcribe_call_import_row_task = _FakePromptOptTask() + fake_workers_tasks_pkg.run_judge_alignment_task = _FakePromptOptTask() + + class _FakeCeleryApp: + """Minimal Celery stand-in for route / worker imports in API tests.""" + + control = types.SimpleNamespace(revoke=lambda *_args, **_kwargs: None) + + def task(self, *_args, **_kwargs): + def _decorator(fn): + fn.delay = lambda *_a, **_kw: types.SimpleNamespace(id="fake-task") + fn.apply_async = lambda *_a, **_kw: types.SimpleNamespace( + id="fake-task" + ) + fn.run = fn + return fn + + return _decorator + + fake_config_module = types.ModuleType("app.workers.config") + fake_config_module.celery_app = _FakeCeleryApp() + sys.modules["app.workers.config"] = fake_config_module + + fake_celery_app_module = types.ModuleType("app.workers.celery_app") + fake_celery_app_module.celery_app = fake_config_module.celery_app + fake_celery_app_module.process_evaluation_task = _FakePromptOptTask() + fake_celery_app_module.process_evaluator_result_task = _FakePromptOptTask() + fake_celery_app_module.run_evaluator_task = _FakePromptOptTask() + fake_celery_app_module.generate_tts_comparison_task = _FakePromptOptTask() + fake_celery_app_module.evaluate_tts_comparison_task = _FakePromptOptTask() + fake_celery_app_module.generate_tts_report_pdf_task = _FakePromptOptTask() + fake_celery_app_module.run_prompt_optimization_task = _FakePromptOptTask() + fake_celery_app_module.process_call_import_row_task = _FakePromptOptTask() + fake_celery_app_module.run_judge_alignment_task = _FakePromptOptTask() + sys.modules["app.workers.celery_app"] = fake_celery_app_module + import importlib + + workers_pkg = importlib.import_module("app.workers") + workers_pkg.celery_app = fake_celery_app_module + +def _wire_bulk_ops_stubs(db_session): + # Bulk call-import tasks: materialize runs synchronously in API tests; + # diarize/delete are no-ops (return immediately). + fake_bulk_ops_module = sys.modules.get("app.workers.tasks.call_import_bulk_ops") + if fake_bulk_ops_module is None: + fake_bulk_ops_module = types.ModuleType("app.workers.tasks.call_import_bulk_ops") + sys.modules["app.workers.tasks.call_import_bulk_ops"] = fake_bulk_ops_module + + def _sync_materialize_delay(evaluation_id, *, transcribe_overwrite=False): + from uuid import UUID + + from app.services.call_imports.bulk_ops import materialize_and_enqueue_evaluation + + materialize_and_enqueue_evaluation( + db_session, + UUID(evaluation_id), + transcribe_overwrite=transcribe_overwrite, + ) + return types.SimpleNamespace(id="fake-sync-bulk-task") + + class _NoopBulkTask: + @staticmethod + def delay(*_args, **_kwargs): + return types.SimpleNamespace(id="fake-sync-bulk-task") + + fake_bulk_ops_module.materialize_call_import_evaluation_task = types.SimpleNamespace( + delay=_sync_materialize_delay + ) + + def _sync_mapped_materialize_delay( + call_import_id, + organization_id, + workspace_id, + evaluation_id, + *, + transcribe_overwrite=False, + ): + from uuid import UUID + + from app.services.call_imports.bulk_ops import ( + execute_call_import_materialization, + materialize_and_enqueue_evaluation, + ) + + mat_result = execute_call_import_materialization( + db_session, + UUID(call_import_id), + UUID(organization_id), + UUID(workspace_id), + schedule_import_dispatch=False, + ) + if mat_result.get("status") != "failed": + materialize_and_enqueue_evaluation( + db_session, + UUID(evaluation_id), + transcribe_overwrite=transcribe_overwrite, + ) + return types.SimpleNamespace(id="fake-sync-mapped-bulk-task") + + fake_bulk_ops_module.materialize_mapped_call_import_evaluation_task = ( + types.SimpleNamespace(delay=_sync_mapped_materialize_delay) + ) + + def _sync_import_materialize_delay(call_import_id, organization_id, workspace_id): + from uuid import UUID + + from app.services.call_imports.bulk_ops import execute_call_import_materialization + + execute_call_import_materialization( + db_session, + UUID(call_import_id), + UUID(organization_id), + UUID(workspace_id), + ) + return types.SimpleNamespace(id="fake-sync-bulk-task") + + fake_bulk_ops_module.materialize_call_import_rows_task = types.SimpleNamespace( + delay=_sync_import_materialize_delay + ) + fake_bulk_ops_module.bulk_diarize_call_import_task = _NoopBulkTask() + fake_bulk_ops_module.bulk_delete_call_import_rows_task = _NoopBulkTask() + fake_bulk_ops_module.delete_call_import_task = _NoopBulkTask() + + def _sync_retry_delay(evaluation_id, payload_dict): + from uuid import UUID + + eval_row_ids_raw = payload_dict.get("eval_row_ids") + metric_ids_raw = payload_dict.get("metric_ids") + from app.services.call_imports.bulk_ops import execute_evaluation_retry + + execute_evaluation_retry( + db_session, + UUID(evaluation_id), + eval_row_ids=( + [UUID(rid) for rid in eval_row_ids_raw] + if eval_row_ids_raw + else None + ), + metric_ids=( + [UUID(mid) for mid in metric_ids_raw] if metric_ids_raw else None + ), + include_completed=bool(payload_dict.get("include_completed", False)), + transcribe_overwrite=bool(payload_dict.get("transcribe_overwrite", False)), + ) + return types.SimpleNamespace(id="fake-sync-retry-task") + + fake_bulk_ops_module.retry_call_import_evaluation_task = types.SimpleNamespace( + delay=_sync_retry_delay + ) + + def _sync_cancel_delay(evaluation_id, *, mode): + from uuid import UUID + + from app.services.call_imports.bulk_ops import execute_evaluation_cancel + + execute_evaluation_cancel(db_session, UUID(evaluation_id), mode=mode) + return types.SimpleNamespace(id="fake-sync-cancel-task") + + fake_bulk_ops_module.cancel_call_import_evaluation_task = types.SimpleNamespace( + delay=_sync_cancel_delay + ) + +def _install_concurrency_stubs(): + fake_fair_dispatch_module = sys.modules.get("app.workers.concurrency.fair_dispatch") + if fake_fair_dispatch_module is None: + fake_fair_dispatch_module = types.ModuleType( + "app.workers.concurrency.fair_dispatch" + ) + sys.modules["app.workers.concurrency.fair_dispatch"] = ( + fake_fair_dispatch_module + ) + if not hasattr(fake_fair_dispatch_module, "schedule_fair_dispatch"): + fake_fair_dispatch_module.schedule_fair_dispatch = lambda *_a, **_kw: None + if not hasattr(fake_fair_dispatch_module, "store_row_restricted_metrics"): + fake_fair_dispatch_module.store_row_restricted_metrics = lambda *_a, **_kw: None + if not hasattr(fake_fair_dispatch_module, "store_evaluation_transcribe_overwrite"): + fake_fair_dispatch_module.store_evaluation_transcribe_overwrite = ( + lambda *_a, **_kw: None + ) + if not hasattr(fake_fair_dispatch_module, "read_fair_dispatch_state"): + fake_fair_dispatch_module.read_fair_dispatch_state = lambda: { + "global_rr_cursor": 0, + "dispatch_dedupe_active": False, + "dispatch_queue": "celery", + "at_capacity_backoff_seconds": 15, + } + if not hasattr(fake_fair_dispatch_module, "read_workspace_eval_rr_cursor"): + fake_fair_dispatch_module.read_workspace_eval_rr_cursor = lambda _ws_id: 0 + if not hasattr(fake_fair_dispatch_module, "finish_eval_work_and_redispatch"): + fake_fair_dispatch_module.finish_eval_work_and_redispatch = ( + lambda *_a, **_kw: None + ) + + def _ensure_concurrency_submodule(name: str) -> types.ModuleType: + full_name = f"app.workers.concurrency.{name}" + module = sys.modules.get(full_name) + if module is None: + module = types.ModuleType(full_name) + sys.modules[full_name] = module + return module + + for submodule, attrs in { + "fair_diarization_dispatch": ( + "finish_diarization_work_and_redispatch", + "schedule_fair_diarization_dispatch", + ), + "fair_import_dispatch": ( + "finish_import_work_and_redispatch", + "schedule_fair_import_dispatch", + ), + "diarization_dispatch": ( + "build_diarization_params_from_request", + "store_row_diarization_params", + ), + "limits": ( + "acquire_eval_slot", + "release_eval_slot_for_celery_task", + "slot_registered_for_task", + ), + }.items(): + mod = _ensure_concurrency_submodule(submodule) + for attr in attrs: + if not hasattr(mod, attr): + setattr(mod, attr, lambda *_a, **_kw: None) + + limits_mod = _ensure_concurrency_submodule("limits") + for attr in ( + "read_global_inflight", + "read_org_inflight", + "read_workspace_inflight", + "read_job_inflight", + ): + if not hasattr(limits_mod, attr): + setattr(limits_mod, attr, lambda *_a, **_kw: 0) + + fake_eval_dispatch_module = sys.modules.get("app.workers.concurrency.eval_dispatch") + if fake_eval_dispatch_module is None: + fake_eval_dispatch_module = types.ModuleType( + "app.workers.concurrency.eval_dispatch" + ) + sys.modules["app.workers.concurrency.eval_dispatch"] = fake_eval_dispatch_module + for queue_name in ("DIARIZATION_QUEUE", "EVALUATIONS_QUEUE", "IMPORTS_QUEUE"): + if not hasattr(fake_eval_dispatch_module, queue_name): + setattr( + fake_eval_dispatch_module, + queue_name, + queue_name.replace("_QUEUE", "").lower(), + ) + if not hasattr(fake_eval_dispatch_module, "schedule_evaluation_dispatch"): + fake_eval_dispatch_module.schedule_evaluation_dispatch = lambda *_a, **_kw: None + + +def _build_session_api_app(): + import app.dependencies as app_dependencies + from app.api.v1.routes import ( + aiproviders, + agents, + alerts, + audio, + auth, + llm_gateway, + call_import_evaluations, + call_import_schemas, + call_import_tags, + call_imports, + chat, + conversation_evaluations, + cron_jobs, + data_sources, + evaluations, + evaluator_results, + evaluators, + evaluator_suites, + iam, + integrations, + manual_evaluations, + metrics, + model_config, + observability, + personas, + playground, + profile, + prompt_optimization, + prompt_partials, + results, + scenarios, + settings, + telephony, + test_agents, + voice_agent, + voice_playground, + voicebundles, + vobiz_telephony, + workspaces, + workspace_iam, + platform_admin, + metric_studio, + ) + + app = FastAPI() + app.include_router(auth.router, prefix="/api/v1") + app.include_router(evaluations.router, prefix="/api/v1") + app.include_router(results.router, prefix="/api/v1") + app.include_router(agents.router, prefix="/api/v1") + app.include_router(evaluators.router, prefix="/api/v1") + app.include_router(evaluator_suites.router, prefix="/api/v1") + app.include_router(personas.router, prefix="/api/v1") + app.include_router(scenarios.router, prefix="/api/v1") + app.include_router(settings.router, prefix="/api/v1") + app.include_router(iam.router, prefix="/api/v1") + app.include_router(audio.router, prefix="/api/v1") + app.include_router(integrations.router, prefix="/api/v1") + app.include_router(aiproviders.router, prefix="/api/v1") + app.include_router(llm_gateway.router, prefix="/api/v1") + app.include_router(metrics.router, prefix="/api/v1") + app.include_router(evaluator_results.router, prefix="/api/v1") + app.include_router(voicebundles.router, prefix="/api/v1") + app.include_router(test_agents.router, prefix="/api/v1") + app.include_router(manual_evaluations.router, prefix="/api/v1") + app.include_router(conversation_evaluations.router, prefix="/api/v1") + app.include_router(alerts.router, prefix="/api/v1") + app.include_router(model_config.router, prefix="/api/v1") + app.include_router(data_sources.router, prefix="/api/v1") + app.include_router(chat.router, prefix="/api/v1") + app.include_router(prompt_partials.router, prefix="/api/v1") + app.include_router(cron_jobs.router, prefix="/api/v1") + app.include_router(profile.router, prefix="/api/v1") + app.include_router(observability.router, prefix="/api/v1") + app.include_router(playground.router, prefix="/api/v1") + app.include_router(prompt_optimization.router, prefix="/api/v1") + app.include_router(voice_agent.router, prefix="/api/v1") + app.include_router(voice_playground.router, prefix="/api/v1") + app.include_router(telephony.router, prefix="/api/v1") + app.include_router(vobiz_telephony.router, prefix="/api/v1") + app.include_router(call_imports.router, prefix="/api/v1") + app.include_router(call_import_schemas.router, prefix="/api/v1") + app.include_router(call_import_tags.router, prefix="/api/v1") + app.include_router(call_import_evaluations.router, prefix="/api/v1") + app.include_router(workspaces.router, prefix="/api/v1") + app.include_router(workspace_iam.router, prefix="/api/v1") + app.include_router(platform_admin.router, prefix="/api/v1") + app.include_router(metric_studio.router, prefix="/api/v1") + # Enterprise route dependencies call app.dependencies.is_feature_enabled at runtime. + # Force-enable it for API tests so tests remain focused on route behavior. + app_dependencies.is_feature_enabled = lambda *_args, **_kwargs: True + + @asynccontextmanager + async def _noop_lifespan(_: object): + yield + + app.router.lifespan_context = _noop_lifespan + return app + +@pytest.fixture +def client(db_session, api_key, org_id): + """ + FastAPI client with DB/auth dependency overrides and no startup lifespan. + This avoids running migrations in test bootstrap. + """ + global _SESSION_API_APP, _SESSION_STUBS_READY + + if not _SESSION_STUBS_READY: + _install_static_stubs() + _install_concurrency_stubs() + _SESSION_API_APP = _build_session_api_app() + _SESSION_STUBS_READY = True + + _wire_bulk_ops_stubs(db_session) + + from app.database import get_db + from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_context, + get_workspace_id, + require_enterprise_feature, + WorkspaceContext, + ) + from app.core.auth.capabilities import ALL_CAPABILITIES + from app.models.database import Organization, Workspace + from app.services.workspace_rbac import backfill_org_workspace_memberships, seed_system_workspace_roles + + app = _SESSION_API_APP + + def _override_workspace_context() -> WorkspaceContext: + return WorkspaceContext( + workspace_id=default_workspace.id, + organization_id=org_id, + capabilities=frozenset(ALL_CAPABILITIES), + is_org_admin=True, + ) + # The TestClient flow doesn't run migration 033, so we manually + # ensure the test org has a Default workspace before any route + # that depends on ``get_workspace_id`` runs. This mirrors what the + # real migration would have produced. + def _ensure_default_workspace() -> Workspace: + org = ( + db_session.query(Organization) + .filter(Organization.id == org_id) + .first() + ) + if org is None: + org = Organization(id=org_id, name="Test Organization") + db_session.add(org) + db_session.flush() + ws = ( + db_session.query(Workspace) + .filter( + Workspace.organization_id == org_id, + Workspace.is_default.is_(True), + ) + .first() + ) + if ws is None: + ws = Workspace( + organization_id=org_id, + name="Default", + slug="default", + is_default=True, + ) + db_session.add(ws) + db_session.commit() + seed_system_workspace_roles(db_session, organization_id=org_id) + return ws + + default_workspace = _ensure_default_workspace() + backfill_org_workspace_memberships(db_session, organization_id=org_id) + + def _override_get_db(): + yield db_session + + app.dependency_overrides[get_db] = _override_get_db + app.dependency_overrides[get_api_key] = lambda: api_key + app.dependency_overrides[get_organization_id] = lambda: org_id + app.dependency_overrides[get_workspace_id] = lambda: default_workspace.id + app.dependency_overrides[get_workspace_context] = _override_workspace_context + app.dependency_overrides[require_enterprise_feature] = lambda: None + + with TestClient(app) as test_client: + yield test_client + + app.dependency_overrides.clear() + + +@pytest.fixture +def telephony_client(db_session): + """Telephony edge TestClient (Vobiz carrier webhooks + media WebSocket routes only).""" + from contextlib import asynccontextmanager + + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from app.api.v1.routes import vobiz_telephony + from app.database import get_db + + app = FastAPI() + + @asynccontextmanager + async def _noop_lifespan(_: object): + yield + + app.router.lifespan_context = _noop_lifespan + app.include_router(vobiz_telephony.webhook_router, prefix="/api/v1") + app.include_router(vobiz_telephony.ws_router, prefix="/api/v1") + + def _override_get_db(): + yield db_session + + app.dependency_overrides[get_db] = _override_get_db + + with TestClient(app) as test_client: + yield test_client + + app.dependency_overrides.clear() + + +@pytest.fixture +def authenticated_client(client, api_key, db_session, org_id): + """Client pre-populated with auth header and a real API key in the DB.""" + from app.models.database import APIKey, Organization, OrganizationMember, RoleEnum, User + + existing_org = db_session.query(Organization).filter(Organization.id == org_id).first() + if existing_org is None: + db_session.add(Organization(id=org_id, name="Test Organization")) + db_session.flush() + + existing_key = ( + db_session.query(APIKey) + .filter(APIKey.key == api_key, APIKey.organization_id == org_id) + .first() + ) + if existing_key is None: + user = User( + id=uuid4(), + email="owner@example.com", + name="Org Owner", + is_active=True, + ) + db_session.add(user) + db_session.flush() + db_session.add( + OrganizationMember( + organization_id=org_id, + user_id=user.id, + role=RoleEnum.ADMIN.value, + ) + ) + db_session.add( + APIKey( + id=uuid4(), + key=api_key, + name="Test API Key", + organization_id=org_id, + user_id=user.id, + is_active=True, + ) + ) + db_session.commit() + + client.headers.update({"X-API-Key": api_key}) + return client + + +@pytest.fixture +def payload_factory(): + """Factory helpers for common API payload shapes.""" + + def _agent_payload(**overrides): + payload = { + "name": "Test Agent", + "phone_number": "+1234567890", + "language": "en", + "description": "This is a test agent description with enough words to pass validation.", + "call_type": "outbound", + "call_medium": "phone_call", + "voice_ai_integration_id": str(uuid4()), + "voice_ai_agent_id": "agent_123", + } + payload.update(overrides) + return payload + + def _persona_payload(**overrides): + payload = { + "name": "Test Persona", + "gender": "neutral", + "is_custom": False, + } + payload.update(overrides) + return payload + + def _scenario_payload(**overrides): + payload = { + "name": "Test Scenario", + "description": "Simple test scenario for backend API tests.", + } + payload.update(overrides) + return payload + + def _evaluation_payload(**overrides): + payload = { + "audio_id": str(uuid4()), + "evaluation_type": "asr", + "metrics": ["wer", "latency"], + } + payload.update(overrides) + return payload + + return { + "agent": _agent_payload, + "persona": _persona_payload, + "scenario": _scenario_payload, + "evaluation": _evaluation_payload, + } diff --git a/tests/test_api/test_auth_routes.py b/tests/test_api/test_auth_routes.py index b5d7d325..215c2589 100644 --- a/tests/test_api/test_auth_routes.py +++ b/tests/test_api/test_auth_routes.py @@ -120,6 +120,7 @@ def enable_local_password(monkeypatch): """Enable the local_password provider + self-service signup for a test.""" monkeypatch.setattr(settings, "AUTH_PROVIDERS", ["api_key", "local_password"]) monkeypatch.setattr(settings, "AUTH_LOCAL_ALLOW_SIGNUP", True) + monkeypatch.setattr(settings, "AUTH_GATED_SIGNUP_ENABLED", False) return settings @@ -363,7 +364,7 @@ def test_login_rejects_user_without_membership_with_403( ) assert response.status_code == 403 - assert "not a member of any organization" in response.json()["detail"].lower() + assert "not a member of any active organization" in response.json()["detail"].lower() def _seed_user_with_multiple_orgs(db_session, email, password): diff --git a/tests/test_api/test_call_import_audit.py b/tests/test_api/test_call_import_audit.py index 60592ff0..db537807 100644 --- a/tests/test_api/test_call_import_audit.py +++ b/tests/test_api/test_call_import_audit.py @@ -1,78 +1,78 @@ -"""Audit fields (created_by / last_updated_by email) on call imports.""" - -from uuid import uuid4 - -from app.models.database import CallImport, Workspace -from app.models.enums import CallImportStatus - - -def _ensure_default_workspace(db_session, org_id): - ws = ( - db_session.query(Workspace) - .filter(Workspace.organization_id == org_id, Workspace.is_default.is_(True)) - .first() - ) - if ws is None: - ws = Workspace( - organization_id=org_id, name="Default", slug="default", is_default=True - ) - db_session.add(ws) - db_session.commit() - return ws - - -def test_update_call_import_metadata_stamps_actor_emails( - authenticated_client, db_session, org_id, seed_org -): - workspace = _ensure_default_workspace(db_session, org_id) - call_import = CallImport( - id=uuid4(), - organization_id=org_id, - workspace_id=workspace.id, - provider="exotel", - original_filename="batch.csv", - total_rows=0, - completed_rows=0, - failed_rows=0, - status=CallImportStatus.COMPLETED, - dataset="before", - ) - db_session.add(call_import) - db_session.commit() - - response = authenticated_client.patch( - f"/api/v1/call-imports/{call_import.id}", - json={"dataset": "after"}, - ) - assert response.status_code == 200, response.text - body = response.json() - assert body["dataset"] == "after" - assert body["created_by_email"] is None - assert body["last_updated_by_email"] == "owner@example.com" - - -def test_list_call_imports_includes_actor_emails( - authenticated_client, db_session, org_id, seed_org -): - workspace = _ensure_default_workspace(db_session, org_id) - call_import = CallImport( - id=uuid4(), - organization_id=org_id, - workspace_id=workspace.id, - provider=None, - original_filename="listed.csv", - total_rows=0, - completed_rows=0, - failed_rows=0, - status=CallImportStatus.UPLOADED, - ) - db_session.add(call_import) - db_session.commit() - - listing = authenticated_client.get("/api/v1/call-imports") - assert listing.status_code == 200, listing.text - items = listing.json()["items"] - match = [item for item in items if item["id"] == str(call_import.id)] - assert len(match) == 1 - assert match[0]["created_by_email"] is None - assert match[0]["last_updated_by_email"] is None +"""Audit fields (created_by / last_updated_by email) on call imports.""" + +from uuid import uuid4 + +from app.models.database import CallImport, Workspace +from app.models.enums import CallImportStatus + + +def _ensure_default_workspace(db_session, org_id): + ws = ( + db_session.query(Workspace) + .filter(Workspace.organization_id == org_id, Workspace.is_default.is_(True)) + .first() + ) + if ws is None: + ws = Workspace( + organization_id=org_id, name="Default", slug="default", is_default=True + ) + db_session.add(ws) + db_session.commit() + return ws + + +def test_update_call_import_metadata_stamps_actor_emails( + authenticated_client, db_session, org_id, seed_org +): + workspace = _ensure_default_workspace(db_session, org_id) + call_import = CallImport( + id=uuid4(), + organization_id=org_id, + workspace_id=workspace.id, + provider="exotel", + original_filename="batch.csv", + total_rows=0, + completed_rows=0, + failed_rows=0, + status=CallImportStatus.COMPLETED, + dataset="before", + ) + db_session.add(call_import) + db_session.commit() + + response = authenticated_client.patch( + f"/api/v1/call-imports/{call_import.id}", + json={"dataset": "after"}, + ) + assert response.status_code == 200, response.text + body = response.json() + assert body["dataset"] == "after" + assert body["created_by_email"] is None + assert body["last_updated_by_email"] == "owner@example.com" + + +def test_list_call_imports_includes_actor_emails( + authenticated_client, db_session, org_id, seed_org +): + workspace = _ensure_default_workspace(db_session, org_id) + call_import = CallImport( + id=uuid4(), + organization_id=org_id, + workspace_id=workspace.id, + provider=None, + original_filename="listed.csv", + total_rows=0, + completed_rows=0, + failed_rows=0, + status=CallImportStatus.UPLOADED, + ) + db_session.add(call_import) + db_session.commit() + + listing = authenticated_client.get("/api/v1/call-imports") + assert listing.status_code == 200, listing.text + items = listing.json()["items"] + match = [item for item in items if item["id"] == str(call_import.id)] + assert len(match) == 1 + assert match[0]["created_by_email"] is None + assert match[0]["last_updated_by_email"] is None diff --git a/tests/test_api/test_call_import_evaluation_pdf_report.py b/tests/test_api/test_call_import_evaluation_pdf_report.py index d24bd739..38706cec 100644 --- a/tests/test_api/test_call_import_evaluation_pdf_report.py +++ b/tests/test_api/test_call_import_evaluation_pdf_report.py @@ -17,6 +17,12 @@ ) +def _disable_pdf_report_storage(monkeypatch): + """Force pdf-report to stream bytes instead of the S3-backed JSON response.""" + s3_module = import_module("app.services.storage.s3_service") + monkeypatch.setattr(s3_module.s3_service, "is_enabled", lambda: False) + + def _default_workspace_id(db_session, org_id) -> UUID: workspace = ( db_session.query(Workspace) @@ -132,6 +138,7 @@ def test_pdf_report_generates_selected_external_pdf( "_render_weasyprint", lambda _html, **_kwargs: None, ) + _disable_pdf_report_storage(monkeypatch) call_import, evaluation = _seed_completed_evaluation(db_session, org_id) response = authenticated_client.post( @@ -167,6 +174,7 @@ def test_pdf_report_generates_selected_internal_pdf( "_render_weasyprint", lambda _html, **_kwargs: None, ) + _disable_pdf_report_storage(monkeypatch) call_import, evaluation = _seed_completed_evaluation(db_session, org_id) response = authenticated_client.post( @@ -887,6 +895,7 @@ def test_external_pdf_renders_generated_user_insights( "_render_weasyprint", lambda _html, **_kwargs: None, ) + _disable_pdf_report_storage(monkeypatch) call_import, evaluation = _seed_completed_evaluation(db_session, org_id) evaluation.user_insights = { "status": "completed", diff --git a/tests/test_api/test_call_import_evaluations.py b/tests/test_api/test_call_import_evaluations.py index 43d9230b..886c47cc 100644 --- a/tests/test_api/test_call_import_evaluations.py +++ b/tests/test_api/test_call_import_evaluations.py @@ -254,6 +254,54 @@ def _make_call_import( return call_import, row_models +def _make_manual_audio_call_import( + db_session, + org_id, + *, + rows=2, +): + """Manual audio upload batch: recordings already in S3, no column mapping.""" + workspace = _ensure_default_workspace(db_session, org_id) + call_import = CallImport( + id=uuid4(), + organization_id=org_id, + workspace_id=workspace.id, + provider=None, + telephony_integration_id=None, + original_filename="Manual recordings", + source_format="audio", + column_mapping=None, + total_rows=rows, + completed_rows=rows, + failed_rows=0, + status=CallImportStatus.COMPLETED, + ) + db_session.add(call_import) + db_session.flush() + + row_models = [] + for idx in range(rows): + row = CallImportRow( + id=uuid4(), + call_import_id=call_import.id, + organization_id=org_id, + workspace_id=workspace.id, + row_index=idx, + conversation_id=f"manual-{idx}", + transcript=None, + recording_url=None, + raw_columns={"conversation_id": f"manual-{idx}"}, + status=CallImportRowStatus.COMPLETED, + recording_s3_key=f"org/{org_id}/call-imports/{call_import.id}/{uuid4()}.wav", + recording_content_type="audio/wav", + recording_size_bytes=1024, + ) + db_session.add(row) + row_models.append(row) + db_session.commit() + return call_import, row_models + + # Every Run Evaluation request now requires STT provider+model (the # diarised transcript is the only supported source and auto-diarise is # mandatory). Centralizing the minimum-valid payload here keeps the test @@ -354,6 +402,23 @@ def test_create_evaluation_accepts_production_transcript_source( assert body.get("diarisation_llm_provider") is None +def test_create_evaluation_accepts_manual_audio_without_recording_url_column( + authenticated_client, db_session, org_id, seed_org +): + """Manual audio batches diarise from stored S3 recordings, not CSV URLs.""" + metric = _make_metric(db_session, org_id) + call_import, _rows = _make_manual_audio_call_import(db_session, org_id, rows=2) + + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations", + json=_eval_body([metric.id]), + ) + assert response.status_code == 202, response.text + body = response.json() + assert body["transcript_source"] == "diarised" + assert body["total_rows"] == 2 + + def test_create_evaluation_defaults_to_diarised_source( authenticated_client, db_session, org_id, seed_org ): @@ -378,7 +443,7 @@ def test_create_evaluation_requires_stt_provider_and_model( ): """Every evaluation auto-diarises rows that don't already have a diarised transcript, so the STT provider+model are mandatory on - every request — even when auto_transcribe is not explicitly + every request ΓÇö even when auto_transcribe is not explicitly passed.""" metric = _make_metric(db_session, org_id) call_import, _ = _make_call_import(db_session, org_id, rows=1) @@ -573,7 +638,7 @@ def test_evaluations_unknown_import_returns_404(authenticated_client, seed_org): # 1. Each cancellable row flips to ``failed`` with the # ``"Evaluation cancelled by user"`` sentinel + cleared ``celery_task_id``. # 2. The parent rollup picks the new state up (``failed``/``partial``). -# 3. The Celery revoke was called with ``terminate=True, signal="SIGTERM"`` — +# 3. The Celery revoke was called with ``terminate=True, signal="SIGTERM"`` ΓÇö # that's the contract that lets the worker actually interrupt an in-flight # LLM/audio call rather than waiting up to 10 minutes for the time limit. @@ -826,7 +891,7 @@ def test_cancel_evaluation_row_flips_only_target_row( refreshed_sibling = db_session.get(CallImportEvaluationRow, sibling.id) assert refreshed_target.status == "failed" assert refreshed_target.celery_task_id is None - # Sibling untouched — only the targeted row was cancelled. + # Sibling untouched ΓÇö only the targeted row was cancelled. assert refreshed_sibling.status == "running" assert refreshed_sibling.celery_task_id is not None @@ -845,7 +910,7 @@ def test_cancel_evaluation_row_flips_only_target_row( def test_cancel_evaluation_row_idempotent_when_terminal( authenticated_client, db_session, org_id, seed_org, monkeypatch ): - """A row already in a terminal state is returned unchanged with a 200 — + """A row already in a terminal state is returned unchanged with a 200 ΓÇö no DB flip, no revoke.""" metric = _make_metric(db_session, org_id) call_import, _ = _make_call_import(db_session, org_id, rows=1) @@ -870,7 +935,7 @@ def test_cancel_evaluation_row_idempotent_when_terminal( ) assert response.status_code == 200 body = response.json() - # Row is unchanged — still completed, scores still attached. + # Row is unchanged ΓÇö still completed, scores still attached. assert body["status"] == "completed" assert body["metric_scores"][str(metric.id)]["value"] == 4 revoke.assert_not_called() @@ -972,6 +1037,145 @@ def test_cancel_row_returns_409_when_bulk_operation_active( assert "bulk abort operation" in response.json()["detail"] +def _force_diarisation_running( + db_session, + evaluation_id, + *, + task_id_prefix="dia-task", +): + """Set linked source rows to in-flight diarisation for cancel cascade tests.""" + eval_uuid = UUID(evaluation_id) + eval_rows = ( + db_session.query(CallImportEvaluationRow) + .filter(CallImportEvaluationRow.evaluation_id == eval_uuid) + .all() + ) + source_ids = [row.call_import_row_id for row in eval_rows] + source_rows = ( + db_session.query(CallImportRow) + .filter(CallImportRow.id.in_(source_ids)) + .all() + ) + for idx, row in enumerate(source_rows): + row.diarised_transcript_status = "running" + row.celery_task_id = f"{task_id_prefix}-{idx}" + db_session.commit() + return source_rows + + +def test_cancel_evaluation_cascades_diarisation_on_source_rows( + authenticated_client, db_session, org_id, seed_org, monkeypatch +): + """Run-level abort should fail in-flight diarisation on linked source rows.""" + metric = _make_metric(db_session, org_id) + call_import, _ = _make_call_import(db_session, org_id, rows=2) + created = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations", + json=_eval_body([metric.id]), + ).json() + + _stub_celery_revoke(monkeypatch) + _force_running(db_session, created["id"]) + source_rows = _force_diarisation_running(db_session, created["id"]) + + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations/{created['id']}/cancel" + ) + assert response.status_code == 202, response.text + + db_session.expire_all() + refreshed = ( + db_session.query(CallImportRow) + .filter(CallImportRow.id.in_([row.id for row in source_rows])) + .all() + ) + assert {row.diarised_transcript_status for row in refreshed} == {"failed"} + assert all( + (row.diarised_transcript_error or "") == "Diarisation cancelled by user" + for row in refreshed + ) + assert all(row.celery_task_id is None for row in refreshed) + + +def test_cancel_evaluation_row_cascades_diarisation_on_source_row( + authenticated_client, db_session, org_id, seed_org, monkeypatch +): + """Row-level abort should fail in-flight diarisation on the linked source row.""" + metric = _make_metric(db_session, org_id) + call_import, _ = _make_call_import(db_session, org_id, rows=2) + created = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations", + json=_eval_body([metric.id]), + ).json() + + _stub_celery_revoke(monkeypatch) + eval_rows = _force_running(db_session, created["id"]) + source_rows = _force_diarisation_running(db_session, created["id"]) + target = eval_rows[0] + target_source = next( + row for row in source_rows if row.id == target.call_import_row_id + ) + sibling_source = next( + row for row in source_rows if row.id != target.call_import_row_id + ) + + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations/" + f"{created['id']}/rows/{target.id}/cancel" + ) + assert response.status_code == 200, response.text + + db_session.expire_all() + db_session.refresh(target_source) + db_session.refresh(sibling_source) + assert target_source.diarised_transcript_status == "failed" + assert ( + target_source.diarised_transcript_error or "" + ) == "Diarisation cancelled by user" + assert target_source.celery_task_id is None + assert sibling_source.diarised_transcript_status == "running" + + +def test_cancel_evaluation_sweeps_diarisation_when_eval_row_already_failed( + authenticated_client, db_session, org_id, seed_org, monkeypatch, +): + """Final sweep fails in-flight diarisation even if the eval row is already terminal.""" + metric = _make_metric(db_session, org_id) + call_import, _ = _make_call_import(db_session, org_id, rows=1) + created = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations", + json=_eval_body([metric.id]), + ).json() + + _stub_celery_revoke(monkeypatch) + eval_row = ( + db_session.query(CallImportEvaluationRow) + .filter(CallImportEvaluationRow.evaluation_id == UUID(created["id"])) + .one() + ) + source_row = db_session.get(CallImportRow, eval_row.call_import_row_id) + eval_row.status = "failed" + eval_row.error_message = "Evaluation cancelled by user" + eval_row.celery_task_id = None + source_row.diarised_transcript_status = "pending" + source_row.celery_task_id = "dia-task-stale" + db_session.commit() + + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations/{created['id']}/cancel" + ) + assert response.status_code == 202, response.text + assert response.json()["target_count"] == 0 + + db_session.expire_all() + db_session.refresh(source_row) + assert source_row.diarised_transcript_status == "failed" + assert ( + source_row.diarised_transcript_error or "" + ) == "Diarisation cancelled by user" + assert source_row.celery_task_id is None + + def test_retry_failed_rows_flips_partial_run_back_to_running( authenticated_client, db_session, org_id, seed_org ): @@ -1093,7 +1297,7 @@ def test_retry_marks_existing_diarised_transcript_completed( def test_evaluation_retry_can_override_telephony_credentials( - authenticated_client, db_session, org_id, seed_org + authenticated_client, db_session, org_id, seed_org, monkeypatch ): """Retry should pin a new telephony integration on the batch when asked.""" metric = _make_metric(db_session, org_id) @@ -1140,6 +1344,15 @@ def test_evaluation_retry_can_override_telephony_credentials( eval_row.error_message = "import failed" db_session.commit() + class _GoodClient: + def test_connection(self): + return None + + monkeypatch.setattr( + "app.services.telephony.telephony_service.telephony_service.get_provider_client", + lambda *_args, **_kwargs: _GoodClient(), + ) + response = authenticated_client.post( f"/api/v1/call-imports/{call_import.id}/evaluations/{eval_uuid}/retry", json={ @@ -1152,43 +1365,97 @@ def test_evaluation_retry_can_override_telephony_credentials( db_session.refresh(call_import) assert call_import.telephony_integration_id == right_integration.id assert call_import.provider == "exotel" - - -def test_create_evaluation_sets_actor_emails( - authenticated_client, db_session, org_id, seed_org -): - metric = _make_metric(db_session, org_id) - call_import, _rows = _make_call_import(db_session, org_id, rows=2) - - response = authenticated_client.post( - f"/api/v1/call-imports/{call_import.id}/evaluations", - json=_eval_body([metric.id]), - ) - assert response.status_code == 202, response.text - body = response.json() - assert body["created_by_email"] == "owner@example.com" - assert body["last_updated_by_email"] == "owner@example.com" - - -def test_update_evaluation_name_stamps_last_updated_by_email( - authenticated_client, db_session, org_id, seed_org -): - metric = _make_metric(db_session, org_id) - call_import, _rows = _make_call_import(db_session, org_id, rows=1) - - created = authenticated_client.post( - f"/api/v1/call-imports/{call_import.id}/evaluations", - json=_eval_body([metric.id]), - ) - assert created.status_code == 202, created.text - eval_id = created.json()["id"] - - patched = authenticated_client.patch( - f"/api/v1/call-imports/{call_import.id}/evaluations/{eval_id}", - json={"name": "Renamed run"}, - ) - assert patched.status_code == 200, patched.text - body = patched.json() - assert body["name"] == "Renamed run" - assert body["created_by_email"] == "owner@example.com" - assert body["last_updated_by_email"] == "owner@example.com" + + +def test_evaluation_retry_rejects_invalid_telephony_credentials( + authenticated_client, db_session, org_id, seed_org, monkeypatch +): + metric = _make_metric(db_session, org_id) + integration = TelephonyIntegration( + id=uuid4(), + organization_id=org_id, + provider="exotel", + name="bad", + auth_id="enc-bad", + auth_token="enc-bad", + is_active=True, + is_default=True, + ) + db_session.add(integration) + db_session.commit() + + call_import, _rows = _make_call_import( + db_session, + org_id, + rows=1, + integration=integration, + ) + created = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations", + json=_eval_body([metric.id]), + ).json() + eval_uuid = UUID(created["id"]) + eval_row = ( + db_session.query(CallImportEvaluationRow) + .filter(CallImportEvaluationRow.evaluation_id == eval_uuid) + .first() + ) + eval_row.status = "failed" + db_session.commit() + + class _BadClient: + def test_connection(self): + raise ValueError("Exotel auth failed (HTTP 401): bad token") + + monkeypatch.setattr( + "app.services.telephony.telephony_service.telephony_service.get_provider_client", + lambda *_args, **_kwargs: _BadClient(), + ) + + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations/{eval_uuid}/retry", + json={ + "provider": "exotel", + "telephony_integration_id": str(integration.id), + }, + ) + assert response.status_code == 400 + assert "credentials could not be verified" in response.json()["detail"].lower() +def test_create_evaluation_sets_actor_emails( + authenticated_client, db_session, org_id, seed_org +): + metric = _make_metric(db_session, org_id) + call_import, _rows = _make_call_import(db_session, org_id, rows=2) + + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations", + json=_eval_body([metric.id]), + ) + assert response.status_code == 202, response.text + body = response.json() + assert body["created_by_email"] == "owner@example.com" + assert body["last_updated_by_email"] == "owner@example.com" + + +def test_update_evaluation_name_stamps_last_updated_by_email( + authenticated_client, db_session, org_id, seed_org +): + metric = _make_metric(db_session, org_id) + call_import, _rows = _make_call_import(db_session, org_id, rows=1) + + created = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations", + json=_eval_body([metric.id]), + ) + assert created.status_code == 202, created.text + eval_id = created.json()["id"] + + patched = authenticated_client.patch( + f"/api/v1/call-imports/{call_import.id}/evaluations/{eval_id}", + json={"name": "Renamed run"}, + ) + assert patched.status_code == 200, patched.text + body = patched.json() + assert body["name"] == "Renamed run" + assert body["created_by_email"] == "owner@example.com" + assert body["last_updated_by_email"] == "owner@example.com" diff --git a/tests/test_api/test_call_import_evaluations_mapped_async.py b/tests/test_api/test_call_import_evaluations_mapped_async.py index f072dd68..10ca28c5 100644 --- a/tests/test_api/test_call_import_evaluations_mapped_async.py +++ b/tests/test_api/test_call_import_evaluations_mapped_async.py @@ -61,6 +61,59 @@ def _make_mapped_call_import(db_session, org_id, workspace_id): return call_import +def _make_transcript_only_mapped_call_import(db_session, org_id, workspace_id): + """Mapped batch whose schema omits recording_url (conversation_id only).""" + schema = CallImportSchema( + id=uuid4(), + organization_id=org_id, + workspace_id=workspace_id, + name="Transcript only", + ) + db_session.add(schema) + db_session.flush() + db_session.add( + CallImportSchemaParameter( + id=uuid4(), + schema_id=schema.id, + name="conversation_id", + type=CallImportParameterType.CONVERSATION_ID.value, + is_required=True, + ordering=0, + ) + ) + db_session.add( + CallImportSchemaParameter( + id=uuid4(), + schema_id=schema.id, + name="transcript", + type=CallImportParameterType.TRANSCRIPT.value, + is_required=False, + ordering=1, + ) + ) + call_import = CallImport( + id=uuid4(), + organization_id=org_id, + workspace_id=workspace_id, + schema_id=schema.id, + source_s3_key="org/test/source.csv", + source_format="csv", + original_filename="source.csv", + column_mapping={}, + parameter_mapping={ + "conversation_id": "CallID", + "transcript": "Transcript", + }, + total_rows=0, + completed_rows=0, + failed_rows=0, + status=CallImportStatus.MAPPED, + ) + db_session.add(call_import) + db_session.commit() + return call_import + + def test_create_evaluation_from_mapped_enqueues_async_materialization( authenticated_client, db_session, @@ -119,6 +172,31 @@ def test_create_evaluation_from_mapped_enqueues_async_materialization( assert refreshed_import.status == CallImportStatus.PROCESSING +def test_create_diarised_evaluation_rejects_schema_without_recording_url( + authenticated_client, + db_session, + org_id, + seed_org, +): + from tests.test_api.test_call_import_evaluations import ( + _eval_body, + _make_metric, + ) + + metric = _make_metric(db_session, org_id) + workspace = metric.workspace_id + call_import = _make_transcript_only_mapped_call_import( + db_session, org_id, workspace + ) + + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations", + json=_eval_body([metric.id]), + ) + assert response.status_code == 409, response.text + assert "recording_url" in response.json()["detail"].lower() + + def test_create_evaluation_from_mapped_stamps_parent_last_updated_by( authenticated_client, db_session, diff --git a/tests/test_api/test_call_import_schemas.py b/tests/test_api/test_call_import_schemas.py index 09e142a8..8ff3471e 100644 --- a/tests/test_api/test_call_import_schemas.py +++ b/tests/test_api/test_call_import_schemas.py @@ -105,8 +105,8 @@ def test_create_schema_happy_path(authenticated_client, db_session, org_id, seed def test_create_schema_forces_system_required_parameters( authenticated_client, db_session, org_id, seed_org ): - """Even if the client sends is_required=False for conversation_id or - recording_url, the server stamps them back to True.""" + """Even if the client sends is_required=False for conversation_id, the + server stamps it back to True. recording_url respects the client flag.""" payload = _minimal_payload() payload["parameters"][0]["is_required"] = False payload["parameters"][1]["is_required"] = False @@ -125,7 +125,7 @@ def test_create_schema_forces_system_required_parameters( p for p in body["parameters"] if p["type"] == "recording_date" ) assert conv_param["is_required"] is True - assert rec_param["is_required"] is True + assert rec_param["is_required"] is False assert date_param["is_required"] is False @@ -145,7 +145,7 @@ def test_create_schema_rejects_missing_conversation_id( assert "conversation_id" in response.text.lower() -def test_create_schema_rejects_missing_recording_url( +def test_create_schema_accepts_missing_recording_url( authenticated_client, db_session, org_id, seed_org ): payload = _minimal_payload() @@ -153,8 +153,9 @@ def test_create_schema_rejects_missing_recording_url( p for p in payload["parameters"] if p["type"] != "recording_url" ] response = authenticated_client.post("/api/v1/call-import-schemas", json=payload) - assert response.status_code == 422 - assert "recording_url" in response.text.lower() + assert response.status_code == 201, response.text + names = [p["name"] for p in response.json()["parameters"]] + assert "recording_url" not in names def test_create_schema_accepts_missing_recording_date( @@ -441,7 +442,7 @@ def test_update_schema_accepts_dropping_recording_date( assert names == ["conversation_id", "recording_url", "agent_name"] -def test_update_schema_rejects_dropping_recording_url( +def test_update_schema_accepts_dropping_recording_url( authenticated_client, db_session, org_id, seed_org ): created = authenticated_client.post( @@ -458,8 +459,9 @@ def test_update_schema_rejects_dropping_recording_url( ] }, ) - assert response.status_code == 422 - assert "recording_url" in response.text.lower() + assert response.status_code == 200, response.text + names = [p["name"] for p in response.json()["parameters"]] + assert "recording_url" not in names # --------------------------------------------------------------------------- diff --git a/tests/test_api/test_call_imports_routes.py b/tests/test_api/test_call_imports_routes.py index 8ad478ad..0516b555 100644 --- a/tests/test_api/test_call_imports_routes.py +++ b/tests/test_api/test_call_imports_routes.py @@ -9,6 +9,8 @@ import io import sys +import threading +import time import types from contextlib import contextmanager from datetime import datetime @@ -21,9 +23,12 @@ from app.api.v1.routes.call_imports import ( _delete_s3_objects, + _is_stuck_pending_import_row, _parse_csv, _parse_xlsx, _revoke_pending_tasks, + _upload_manual_audio_blobs_parallel, + _validate_telephony_credentials_live, ) from app.config import settings from app.models.database import ( @@ -35,7 +40,11 @@ TelephonyIntegration, Workspace, ) -from app.models.enums import CallImportParameterType, CallImportRowStatus +from app.models.enums import ( + CallImportParameterType, + CallImportRowStatus, + CallImportStatus, +) # --------------------------------------------------------------------------- @@ -261,6 +270,26 @@ def test_parse_csv_skips_row_missing_recording_url(): assert result.skipped[0].source_row == 2 +def test_parse_csv_allows_empty_recording_url_when_optional(): + csv_text = ( + "CallID,Recording Date,Recording URL,Transcript\n" + "abc-1,18/05/2026,,Transcript only\n" + ) + params = _standard_params() + rec_param = next(p for p in params if p.type == CallImportParameterType.RECORDING_URL.value) + rec_param.is_required = False + result = _parse_csv( + _csv_bytes(csv_text), + params, + _standard_mapping(), + _standard_skipped(), + ) + assert len(result.rows) == 1 + assert result.rows[0]["conversation_id"] == "abc-1" + assert result.rows[0]["recording_url"] is None + assert len(result.skipped) == 0 + + def test_parse_csv_skips_row_invalid_recording_url(): csv_text = ( "CallID,Recording Date,Recording URL,Transcript\n" @@ -874,6 +903,35 @@ def _patched_s3(fake_s3): yield +def test_upload_manual_audio_blobs_parallel_runs_concurrently(): + lock = threading.Lock() + active = 0 + max_active = 0 + uploaded_keys: list[str] = [] + + def _slow_upload(_contents, key, content_type="audio/mpeg"): + nonlocal active, max_active + with lock: + active += 1 + max_active = max(max_active, active) + time.sleep(0.05) + with lock: + active -= 1 + uploaded_keys.append(key) + + fake_s3 = SimpleNamespace(upload_file_by_key=_slow_upload) + specs = [(f"audio/key-{index}.mp3", b"body", "audio/mpeg") for index in range(8)] + + with _patched_s3(fake_s3): + started = time.monotonic() + _upload_manual_audio_blobs_parallel(specs) + elapsed = time.monotonic() - started + + assert len(uploaded_keys) == 8 + assert max_active > 1 + assert elapsed < 0.35 + + def test_audio_upload_single_file_creates_completed_import( authenticated_client, db_session, org_id, seed_org ): @@ -926,6 +984,77 @@ def test_audio_upload_single_file_creates_completed_import( fake_s3.upload_file_by_key.assert_called_once() +def test_audio_upload_accepts_custom_batch_name( + authenticated_client, db_session, org_id, seed_org +): + fake_s3 = _fake_enabled_s3() + with _patched_s3(fake_s3): + response = authenticated_client.post( + "/api/v1/call-imports/audio-upload", + files=[ + ("files", ("a.wav", b"a", "audio/wav")), + ("files", ("b.wav", b"b", "audio/wav")), + ], + data={ + "dataset": "Manual recordings", + "batch_name": "October support calls", + }, + ) + + assert response.status_code == 201, response.text + call_import = db_session.query(CallImport).filter( + CallImport.id == UUID(response.json()["id"]) + ).one() + assert call_import.original_filename == "October support calls" + + +def test_audio_upload_multi_file_default_name_is_generic( + authenticated_client, db_session, org_id, seed_org +): + fake_s3 = _fake_enabled_s3() + with _patched_s3(fake_s3): + response = authenticated_client.post( + "/api/v1/call-imports/audio-upload", + files=[ + ("files", ("a.wav", b"a", "audio/wav")), + ("files", ("b.wav", b"b", "audio/wav")), + ], + data={"dataset": "Manual recordings"}, + ) + + assert response.status_code == 201, response.text + call_import = db_session.query(CallImport).filter( + CallImport.id == UUID(response.json()["id"]) + ).one() + assert call_import.original_filename == "Manual recordings" + + +def test_update_call_import_original_filename( + authenticated_client, db_session, org_id, seed_org +): + fake_s3 = _fake_enabled_s3() + with _patched_s3(fake_s3): + create_response = authenticated_client.post( + "/api/v1/call-imports/audio-upload", + files={"files": ("call.wav", b"wav", "audio/wav")}, + data={"dataset": "Manual recordings"}, + ) + assert create_response.status_code == 201, create_response.text + call_import_id = create_response.json()["id"] + + patch_response = authenticated_client.patch( + f"/api/v1/call-imports/{call_import_id}", + json={"original_filename": "Renamed batch"}, + ) + assert patch_response.status_code == 200, patch_response.text + assert patch_response.json()["original_filename"] == "Renamed batch" + + call_import = db_session.query(CallImport).filter( + CallImport.id == UUID(call_import_id) + ).one() + assert call_import.original_filename == "Renamed batch" + + def test_audio_upload_uses_shard_insert_when_sharding_enabled( authenticated_client, db_session, org_id, seed_org, monkeypatch ): @@ -1003,6 +1132,188 @@ def test_audio_upload_multiple_files_dedupes_filename_call_ids( assert fake_s3.upload_file_by_key.call_count == 3 +def test_audio_append_adds_rows_to_existing_batch( + authenticated_client, db_session, org_id, seed_org +): + fake_s3 = _fake_enabled_s3() + with _patched_s3(fake_s3): + with patch( + "app.api.v1.routes.call_imports.is_sharding_enabled", + return_value=False, + ): + create_response = authenticated_client.post( + "/api/v1/call-imports/audio-upload", + files={"files": ("call.wav", b"chunk-a", "audio/wav")}, + data={"dataset": "Manual recordings"}, + ) + assert create_response.status_code == 201, create_response.text + call_import_id = create_response.json()["id"] + + append_response = authenticated_client.post( + f"/api/v1/call-imports/{call_import_id}/audio-append", + files=[ + ("files", ("call.wav", b"chunk-b", "audio/wav")), + ("files", ("support.m4a", b"chunk-c", "audio/mp4")), + ], + ) + + assert append_response.status_code == 200, append_response.text + body = append_response.json() + assert body["total_rows"] == 3 + rows = ( + db_session.query(CallImportRow) + .filter(CallImportRow.call_import_id == UUID(call_import_id)) + .order_by(CallImportRow.row_index) + .all() + ) + assert [row.row_index for row in rows] == [0, 1, 2] + assert [row.conversation_id for row in rows] == ["call", "call-2", "support"] + assert fake_s3.upload_file_by_key.call_count == 3 + + +def test_audio_append_continues_row_index_when_sharding_enabled( + authenticated_client, db_session, org_id, seed_org, monkeypatch +): + fake_s3 = _fake_enabled_s3() + inserted: list = [] + registered: list = [] + + monkeypatch.setattr( + "app.api.v1.routes.call_imports.is_sharding_enabled", + lambda: True, + ) + + def _fake_bulk_insert(_db, call_import_id, mappings): + inserted.extend(mappings) + return len(mappings) + + def _fake_register_slices(_db, call_import_id, total_rows): + registered.append((call_import_id, total_rows)) + + monkeypatch.setattr( + "app.db_sharding.row_ops.bulk_insert_mappings_on_shards", + _fake_bulk_insert, + ) + monkeypatch.setattr( + "app.db_sharding.row_ops.register_shard_slices", + _fake_register_slices, + ) + + with _patched_s3(fake_s3): + create_response = authenticated_client.post( + "/api/v1/call-imports/audio-upload", + files=[ + ("files", ("a.wav", b"a", "audio/wav")), + ("files", ("b.wav", b"b", "audio/wav")), + ], + data={"dataset": "Manual recordings"}, + ) + assert create_response.status_code == 201, create_response.text + call_import_id = UUID(create_response.json()["id"]) + assert [mapping["row_index"] for mapping in inserted] == [0, 1] + inserted.clear() + + append_response = authenticated_client.post( + f"/api/v1/call-imports/{call_import_id}/audio-append", + files={"files": ("c.wav", b"c", "audio/wav")}, + ) + + assert append_response.status_code == 200, append_response.text + assert [mapping["row_index"] for mapping in inserted] == [2] + assert registered == [(call_import_id, 2), (call_import_id, 3)] + + +def test_validate_telephony_credentials_live_rejects_bad_client(db_session, org_id, seed_org): + integration = _seed_integration(db_session, org_id, provider="exotel") + + class _BadClient: + def test_connection(self): + raise ValueError("Exotel auth failed (HTTP 401): bad token") + + with patch( + "app.services.telephony.telephony_service.telephony_service.get_provider_client", + return_value=_BadClient(), + ): + with pytest.raises(HTTPException) as exc: + _validate_telephony_credentials_live(db_session, org_id, integration) + + assert exc.value.status_code == 400 + assert "credentials could not be verified" in exc.value.detail.lower() + + +def test_is_stuck_pending_import_row_detects_credential_errors(): + row = SimpleNamespace( + status=CallImportRowStatus.PENDING, + attempts=0, + error_message="Transient: Recording URL rejected credentials (HTTP 401)", + ) + assert _is_stuck_pending_import_row(row) is True + + fresh = SimpleNamespace( + status=CallImportRowStatus.PENDING, + attempts=0, + error_message=None, + ) + assert _is_stuck_pending_import_row(fresh) is False + + +def test_retry_failed_requeues_stuck_pending_rows( + authenticated_client, db_session, org_id, seed_org +): + integration = _seed_integration(db_session, org_id) + workspace_id = _default_workspace_id(db_session, org_id) + call_import = CallImport( + id=uuid4(), + organization_id=org_id, + workspace_id=workspace_id, + provider=integration.provider, + telephony_integration_id=integration.id, + original_filename="batch.csv", + source_format="csv", + total_rows=1, + completed_rows=0, + failed_rows=0, + status=CallImportStatus.PROCESSING, + ) + db_session.add(call_import) + db_session.flush() + row = CallImportRow( + id=uuid4(), + call_import_id=call_import.id, + organization_id=org_id, + workspace_id=workspace_id, + row_index=0, + conversation_id="call-1", + recording_url="https://example.com/rec.mp3", + status=CallImportRowStatus.PENDING, + attempts=2, + error_message="Transient: Recording URL rejected credentials (HTTP 401)", + ) + db_session.add(row) + db_session.commit() + + scheduled = {"called": False} + + def _schedule(*_args, **_kwargs): + scheduled["called"] = True + + with patch( + "app.workers.concurrency.fair_import_dispatch.schedule_fair_import_dispatch", + _schedule, + ): + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/retry-failed", + ) + + assert response.status_code == 202, response.text + assert response.json()["requeued"] == 1 + assert scheduled["called"] is True + db_session.refresh(row) + assert row.status == CallImportRowStatus.PENDING + assert row.error_message is None + assert row.attempts == 0 + + def test_audio_upload_rejects_invalid_inputs( authenticated_client, monkeypatch, seed_org ): diff --git a/tests/test_api/test_gateway_managed_credentials.py b/tests/test_api/test_gateway_managed_credentials.py index c2ebacbd..3b59c58f 100644 --- a/tests/test_api/test_gateway_managed_credentials.py +++ b/tests/test_api/test_gateway_managed_credentials.py @@ -27,6 +27,37 @@ def _reset_gateway_settings(): ) = original +def test_create_aiprovider_gateway_with_credential_base_url_when_org_direct( + authenticated_client, +): + _set_platform_gateway_passthrough(False) + settings.LLM_GATEWAY_ENABLED = False + settings.LLM_GATEWAY_BASE_URL = None + + disable_response = authenticated_client.put( + "/api/v1/organizations/llm-gateway", + json={"mode": "disabled"}, + ) + assert disable_response.status_code == 200 + assert disable_response.json()["effective_routing"] == "direct" + + response = authenticated_client.post( + "/api/v1/aiproviders", + json={ + "provider": "custom", + "name": "Test", + "routing_mode": "gateway", + "gateway_model": "gpt-oss-120b", + "gateway_interface": "native_openai", + "gateway_base_url": "http://localhost:8080", + }, + ) + assert response.status_code == 201 + data = response.json() + assert data["effective_routing"] == "bifrost" + assert data["gateway_model"] == "gpt-oss-120b" + + def test_create_aiprovider_without_key_when_gateway_routing( authenticated_client, db_session, org_id ): diff --git a/tests/test_api/test_metric_studio.py b/tests/test_api/test_metric_studio.py new file mode 100644 index 00000000..1e5f27d1 --- /dev/null +++ b/tests/test_api/test_metric_studio.py @@ -0,0 +1,84 @@ +"""API tests for Metrics Studio.""" + +from uuid import uuid4 + + +def test_create_and_list_metric_draft(authenticated_client): + payload = { + "name": "Studio Draft Metric", + "description": "Test draft rubric", + "metric_type": "rating", + "trigger": "always", + "metric_origin": "custom", + "studio_notes": "experiment v1", + } + create_response = authenticated_client.post("/api/v1/metrics/drafts", json=payload) + assert create_response.status_code == 201 + body = create_response.json() + assert body["lifecycle"] == "draft" + assert body["enabled"] is False + + list_active = authenticated_client.get("/api/v1/metrics") + assert list_active.status_code == 200 + assert all(m.get("lifecycle", "active") != "draft" for m in list_active.json()) + + list_drafts = authenticated_client.get( + "/api/v1/metrics", params={"drafts_only": True} + ) + assert list_drafts.status_code == 200 + assert any(m["id"] == body["id"] for m in list_drafts.json()) + + +def test_promote_metric_draft(authenticated_client): + create_response = authenticated_client.post( + "/api/v1/metrics/drafts", + json={ + "name": f"Promote Me {uuid4().hex[:6]}", + "description": "Draft to promote", + "metric_type": "boolean", + "trigger": "always", + }, + ) + assert create_response.status_code == 201 + metric_id = create_response.json()["id"] + + promote_response = authenticated_client.post( + f"/api/v1/metrics/{metric_id}/promote" + ) + assert promote_response.status_code == 200 + promoted = promote_response.json() + assert promoted["metric"]["lifecycle"] == "active" + assert promoted["metric"]["enabled"] is True + assert promoted["promoted_at"] + + +def test_list_metrics_enabled_only_excludes_disabled(authenticated_client, make_metric): + enabled = make_metric(name="Studio Enabled Metric", enabled=True) + disabled = make_metric(name="Studio Disabled Metric", enabled=False) + + all_response = authenticated_client.get("/api/v1/metrics") + assert all_response.status_code == 200 + all_ids = {m["id"] for m in all_response.json()} + assert str(enabled.id) in all_ids + assert str(disabled.id) in all_ids + + enabled_only_response = authenticated_client.get( + "/api/v1/metrics", + params={"enabled_only": True}, + ) + assert enabled_only_response.status_code == 200 + enabled_ids = {m["id"] for m in enabled_only_response.json()} + assert str(enabled.id) in enabled_ids + assert str(disabled.id) not in enabled_ids + + +def test_create_metric_studio_run_requires_sources(authenticated_client, make_metric): + metric = make_metric(name="Studio Run Metric", metric_type="rating") + response = authenticated_client.post( + "/api/v1/metric-studio/runs", + json={ + "metric_ids": [str(metric.id)], + "sources": [], + }, + ) + assert response.status_code == 422 diff --git a/tests/test_api/test_platform_admin.py b/tests/test_api/test_platform_admin.py new file mode 100644 index 00000000..4c650c0f --- /dev/null +++ b/tests/test_api/test_platform_admin.py @@ -0,0 +1,283 @@ +"""Tests for platform admin authentication and org management.""" + +from __future__ import annotations + +import pytest + +from app.config import settings +from app.core.auth.platform_admin import create_platform_access_token +from app.core.auth.tokens import create_access_token +from app.core.password import hash_password +from app.models.database import ( + Organization, + OrganizationMember, + PlatformAdmin, + RoleEnum, + User, +) + +TEST_PASSWORD = "TestPass1!" +PLATFORM_PASSWORD = "Platform1!" + + +@pytest.fixture +def enable_local_password(monkeypatch): + monkeypatch.setattr(settings, "AUTH_PROVIDERS", ["api_key", "local_password"]) + monkeypatch.setattr(settings, "AUTH_LOCAL_ALLOW_SIGNUP", True) + return settings + + +@pytest.fixture +def platform_admin_user(db_session): + admin = PlatformAdmin( + email="platform@example.com", + password_hash=hash_password(PLATFORM_PASSWORD), + is_active=True, + ) + db_session.add(admin) + db_session.commit() + db_session.refresh(admin) + return admin + + +@pytest.fixture +def platform_admin_client(client, platform_admin_user): + token, _ = create_platform_access_token( + platform_admin_id=platform_admin_user.id, + email=platform_admin_user.email, + ) + client.headers.update({"Authorization": f"Bearer {token}"}) + return client + + +def test_platform_login_returns_token(client, platform_admin_user): + response = client.post( + "/api/v1/platform/auth/login", + json={"email": "platform@example.com", "password": PLATFORM_PASSWORD}, + ) + assert response.status_code == 200 + body = response.json() + assert body["access_token"] + assert body["admin"]["email"] == "platform@example.com" + + +def test_platform_login_returns_404_when_no_admins(client, db_session): + db_session.query(PlatformAdmin).delete() + db_session.commit() + response = client.post( + "/api/v1/platform/auth/login", + json={"email": "platform@example.com", "password": PLATFORM_PASSWORD}, + ) + assert response.status_code == 404 + + +def test_platform_routes_reject_org_scoped_token( + client, db_session, platform_admin_user, enable_local_password +): + user = User( + email="user@example.com", + password_hash=hash_password(TEST_PASSWORD), + is_active=True, + auth_provider="local", + ) + org = Organization(name="Org A") + db_session.add_all([user, org]) + db_session.flush() + db_session.add( + OrganizationMember( + organization_id=org.id, + user_id=user.id, + role=RoleEnum.ADMIN.value, + ) + ) + db_session.commit() + + org_token, _, _ = create_access_token( + user_id=user.id, + organization_id=org.id, + email=user.email, + ) + client.headers.update({"Authorization": f"Bearer {org_token}"}) + response = client.get("/api/v1/platform/organizations/stats") + assert response.status_code == 401 + + +def test_list_organizations_and_stats(platform_admin_client, db_session): + initial_total = db_session.query(Organization).count() + initial_active = ( + db_session.query(Organization) + .filter(Organization.is_active == True) # noqa: E712 + .count() + ) + + org1 = Organization(name="Alpha Org") + org2 = Organization(name="Beta Org", is_active=False) + db_session.add_all([org1, org2]) + db_session.flush() + user = User(email="member@example.com", is_active=True) + db_session.add(user) + db_session.flush() + db_session.add( + OrganizationMember( + organization_id=org1.id, + user_id=user.id, + role=RoleEnum.ADMIN.value, + ) + ) + db_session.commit() + + stats = platform_admin_client.get("/api/v1/platform/organizations/stats") + assert stats.status_code == 200 + assert stats.json()["total"] == initial_total + 2 + assert stats.json()["active"] == initial_active + 1 + assert stats.json()["disabled"] == (initial_total - initial_active) + 1 + + listing = platform_admin_client.get("/api/v1/platform/organizations") + assert listing.status_code == 200 + items = listing.json()["items"] + assert len(items) == initial_total + 2 + alpha = next(item for item in items if item["name"] == "Alpha Org") + assert alpha["member_count"] == 1 + + +def test_disable_organization(platform_admin_client, db_session): + org = Organization(name="To Disable") + db_session.add(org) + db_session.commit() + + response = platform_admin_client.patch( + f"/api/v1/platform/organizations/{org.id}", + json={"is_active": False}, + ) + assert response.status_code == 200 + assert response.json()["is_active"] is False + + db_session.refresh(org) + assert org.is_active is False + assert org.disabled_at is not None + + +def test_platform_reset_password(platform_admin_client, client, db_session, enable_local_password): + org = Organization(name="Reset Org") + user = User( + email="admin@reset.org", + password_hash=hash_password(TEST_PASSWORD), + is_active=True, + auth_provider="local", + ) + db_session.add_all([org, user]) + db_session.flush() + db_session.add( + OrganizationMember( + organization_id=org.id, + user_id=user.id, + role=RoleEnum.ADMIN.value, + ) + ) + db_session.commit() + + new_password = "NewPass2!" + response = platform_admin_client.post( + f"/api/v1/platform/organizations/{org.id}/users/{user.id}/reset-password", + json={"new_password": new_password}, + ) + assert response.status_code == 200 + assert response.json()["email"] == "admin@reset.org" + + login = client.post( + "/api/v1/auth/login", + json={"email": "admin@reset.org", "password": new_password}, + ) + assert login.status_code == 200 + + +def test_create_and_use_signup_reference_code( + platform_admin_client, client, db_session, enable_local_password, monkeypatch +): + monkeypatch.setattr(settings, "AUTH_GATED_SIGNUP_ENABLED", True) + + create = platform_admin_client.post( + "/api/v1/platform/signup-codes", + json={"code": "BETA2026", "max_uses": 1, "label": "Beta invite"}, + ) + assert create.status_code == 201 + assert create.json()["code"] == "BETA2026" + + signup = client.post( + "/api/v1/auth/signup", + json={ + "email": "gated@example.com", + "password": TEST_PASSWORD, + "reference_code": "BETA2026", + }, + ) + assert signup.status_code == 200 + + duplicate = client.post( + "/api/v1/auth/signup", + json={ + "email": "gated2@example.com", + "password": TEST_PASSWORD, + "reference_code": "BETA2026", + }, + ) + assert duplicate.status_code == 403 + + +def test_gated_signup_rejects_missing_code(client, enable_local_password, monkeypatch): + monkeypatch.setattr(settings, "AUTH_GATED_SIGNUP_ENABLED", True) + response = client.post( + "/api/v1/auth/signup", + json={"email": "nogate@example.com", "password": TEST_PASSWORD}, + ) + assert response.status_code == 403 + + +def test_disabled_org_blocks_api_key_auth(authenticated_client, db_session, org_id): + from app.dependencies import get_api_key, get_organization_id + + org = db_session.query(Organization).filter(Organization.id == org_id).first() + org.is_active = False + db_session.commit() + + # The shared test client bypasses auth deps by default; restore real + # resolution so get_principal (and org-disable checks) actually run. + authenticated_client.app.dependency_overrides.pop(get_organization_id, None) + authenticated_client.app.dependency_overrides.pop(get_api_key, None) + + response = authenticated_client.get("/api/v1/iam/organization") + assert response.status_code == 403 + assert "disabled" in response.json()["detail"].lower() + + +def test_disabled_org_blocks_login(client, db_session, enable_local_password): + org = Organization(name="Disabled Org", is_active=False) + user = User( + email="locked@example.com", + password_hash=hash_password(TEST_PASSWORD), + is_active=True, + auth_provider="local", + ) + db_session.add_all([org, user]) + db_session.flush() + db_session.add( + OrganizationMember( + organization_id=org.id, + user_id=user.id, + role=RoleEnum.ADMIN.value, + ) + ) + db_session.commit() + + response = client.post( + "/api/v1/auth/login", + json={"email": "locked@example.com", "password": TEST_PASSWORD}, + ) + assert response.status_code == 403 + + +def test_auth_config_reports_gated_signup(client, enable_local_password, monkeypatch): + monkeypatch.setattr(settings, "AUTH_GATED_SIGNUP_ENABLED", True) + response = client.get("/api/v1/auth/config") + assert response.status_code == 200 + assert response.json()["gated_signup"] is True diff --git a/tests/test_api/test_workspace_iam.py b/tests/test_api/test_workspace_iam.py index 3378fa5f..b3fe341e 100644 --- a/tests/test_api/test_workspace_iam.py +++ b/tests/test_api/test_workspace_iam.py @@ -15,6 +15,7 @@ SYSTEM_ROLE_ADMIN, SYSTEM_ROLE_VIEWER, WORKSPACE_MEMBERS_MANAGE, + WORKSPACE_SETTINGS, ) from app.core.auth.principal import AuthMethod, Principal from app.core.auth.dependency import get_principal @@ -492,6 +493,152 @@ def _override_db(): assert response.status_code == 404 +def _workspaces_test_app(db_session, rbac_org, user_id): + app = FastAPI() + app.include_router(workspaces.router, prefix="/api/v1") + + def _override_db(): + yield db_session + + app.dependency_overrides[get_db] = _override_db + app.dependency_overrides[get_principal] = lambda: Principal( + organization_id=rbac_org.id, + auth_method=AuthMethod.LOCAL_PASSWORD, + user_id=user_id, + ) + app.dependency_overrides[get_organization_id] = lambda: rbac_org.id + return app + + +def test_workspace_admin_can_rename_workspace(db_session, rbac_org, rbac_users, rbac_workspace): + roles = seed_system_workspace_roles(db_session, organization_id=rbac_org.id) + add_workspace_member( + db_session, + workspace_id=rbac_workspace.id, + user_id=rbac_users["writer"].id, + role_id=roles[SYSTEM_ROLE_ADMIN].id, + ) + db_session.commit() + + app = _workspaces_test_app(db_session, rbac_org, rbac_users["writer"].id) + with TestClient(app) as client: + response = client.patch( + f"/api/v1/workspaces/{rbac_workspace.id}", + json={"name": "Project Alpha"}, + ) + + assert response.status_code == 200 + body = response.json() + assert body["name"] == "Project Alpha" + assert body["slug"] == "project_a" + assert WORKSPACE_SETTINGS in body["capabilities"] + + +def test_workspace_viewer_cannot_rename_workspace( + db_session, rbac_org, rbac_users, rbac_workspace +): + app = _workspaces_test_app(db_session, rbac_org, rbac_users["viewer"].id) + with TestClient(app) as client: + response = client.patch( + f"/api/v1/workspaces/{rbac_workspace.id}", + json={"name": "Renamed"}, + ) + + assert response.status_code == 403 + assert "Workspace Admin role" in response.json()["detail"] + db_session.refresh(rbac_workspace) + assert rbac_workspace.name == "Project A" + + +def test_inactive_workspace_blocks_non_admin_access( + db_session, rbac_org, rbac_users, rbac_workspace +): + rbac_workspace.is_active = False + db_session.commit() + + app = FastAPI() + app.include_router(metrics.router, prefix="/api/v1") + + def _override_db(): + yield db_session + + app.dependency_overrides[get_db] = _override_db + app.dependency_overrides[get_principal] = lambda: Principal( + organization_id=rbac_org.id, + auth_method=AuthMethod.LOCAL_PASSWORD, + user_id=rbac_users["viewer"].id, + ) + app.dependency_overrides[get_organization_id] = lambda: rbac_org.id + + with TestClient(app) as client: + response = client.get( + "/api/v1/metrics", + headers={"X-Workspace-Id": str(rbac_workspace.id)}, + ) + + assert response.status_code == 403 + assert "inactive" in response.json()["detail"].lower() + + +def test_inactive_workspace_hidden_from_member_list( + db_session, rbac_org, rbac_users, rbac_workspace +): + inactive_ws = Workspace( + id=uuid4(), + organization_id=rbac_org.id, + name="Hidden WS", + slug="hidden_ws", + is_default=False, + is_active=False, + ) + db_session.add(inactive_ws) + db_session.flush() + roles = seed_system_workspace_roles(db_session, organization_id=rbac_org.id) + add_workspace_member( + db_session, + workspace_id=inactive_ws.id, + user_id=rbac_users["writer"].id, + role_id=roles[SYSTEM_ROLE_VIEWER].id, + ) + db_session.commit() + + app = _workspaces_test_app(db_session, rbac_org, rbac_users["writer"].id) + with TestClient(app) as client: + listing = client.get("/api/v1/workspaces").json() + + ids = {w["id"] for w in listing} + assert str(inactive_ws.id) not in ids + + +def test_org_admin_can_access_inactive_workspace( + db_session, rbac_org, rbac_users, rbac_workspace +): + rbac_workspace.is_active = False + db_session.commit() + + app = FastAPI() + app.include_router(metrics.router, prefix="/api/v1") + + def _override_db(): + yield db_session + + app.dependency_overrides[get_db] = _override_db + app.dependency_overrides[get_principal] = lambda: Principal( + organization_id=rbac_org.id, + auth_method=AuthMethod.LOCAL_PASSWORD, + user_id=rbac_users["admin"].id, + ) + app.dependency_overrides[get_organization_id] = lambda: rbac_org.id + + with TestClient(app) as client: + response = client.get( + "/api/v1/metrics", + headers={"X-Workspace-Id": str(rbac_workspace.id)}, + ) + + assert response.status_code == 200 + + def test_require_capability_missing_capability_returns_403(): """Regression: require_capability must not NameError on status import.""" from uuid import uuid4 diff --git a/tests/test_api/test_workspaces.py b/tests/test_api/test_workspaces.py index efef7581..f07349d3 100644 --- a/tests/test_api/test_workspaces.py +++ b/tests/test_api/test_workspaces.py @@ -241,3 +241,62 @@ def test_promote_honors_explicit_capture_rationale_false( assert response.status_code == 201 detail = authenticated_client.get(f"/api/v1/metrics/{parent.id}").json() assert detail["children"][0]["capture_rationale"] is False + + +def test_org_admin_can_deactivate_and_reactivate_workspace( + authenticated_client, db_session, org_id +): + seed_system_workspace_roles(db_session, organization_id=org_id) + create = authenticated_client.post( + "/api/v1/workspaces", json={"name": "Archive Me"} + ) + assert create.status_code == 201, create.text + workspace_id = create.json()["id"] + + deactivate = authenticated_client.patch( + f"/api/v1/workspaces/{workspace_id}", + json={"is_active": False}, + ) + assert deactivate.status_code == 200 + assert deactivate.json()["is_active"] is False + + reactivate = authenticated_client.patch( + f"/api/v1/workspaces/{workspace_id}", + json={"is_active": True}, + ) + assert reactivate.status_code == 200 + assert reactivate.json()["is_active"] is True + + +def test_cannot_deactivate_default_workspace(authenticated_client, db_session, org_id): + default = ( + db_session.query(Workspace) + .filter(Workspace.organization_id == org_id, Workspace.is_default.is_(True)) + .first() + ) + response = authenticated_client.patch( + f"/api/v1/workspaces/{default.id}", + json={"is_active": False}, + ) + assert response.status_code == 400 + assert "default" in response.json()["detail"].lower() + + +def test_org_admin_list_includes_inactive_workspace( + authenticated_client, db_session, org_id +): + inactive_ws = Workspace( + id=uuid4(), + organization_id=org_id, + name="Admin Visible Inactive", + slug="admin_visible_inactive", + is_default=False, + is_active=False, + ) + db_session.add(inactive_ws) + db_session.commit() + + listing = authenticated_client.get("/api/v1/workspaces").json() + match = next((w for w in listing if w["id"] == str(inactive_ws.id)), None) + assert match is not None + assert match["is_active"] is False diff --git a/tests/test_db_sharding/test_scatter_gather_api.py b/tests/test_db_sharding/test_scatter_gather_api.py index 3bf4a806..619295f1 100644 --- a/tests/test_db_sharding/test_scatter_gather_api.py +++ b/tests/test_db_sharding/test_scatter_gather_api.py @@ -3,7 +3,10 @@ from unittest.mock import MagicMock, patch from uuid import uuid4 -from app.db_sharding.scatter_gather import fetch_call_import_rows_page +from app.db_sharding.scatter_gather import ( + fetch_call_import_rows_page, + max_call_import_row_index, +) def test_fetch_rows_page_when_sharding_off(): @@ -16,3 +19,27 @@ def test_fetch_rows_page_when_sharding_off(): with patch("app.db_sharding.scatter_gather.is_sharding_enabled", return_value=False): out = fetch_call_import_rows_page(db, call_import_id, offset=0, limit=10) assert out == mock_rows + + +def test_max_call_import_row_index_when_sharding_off(): + call_import_id = uuid4() + db = MagicMock() + db.query.return_value.filter.return_value.scalar.return_value = 41 + with patch("app.db_sharding.scatter_gather.is_sharding_enabled", return_value=False): + assert max_call_import_row_index(db, call_import_id) == 41 + + +def test_max_call_import_row_index_scatter_gather(): + call_import_id = uuid4() + db = MagicMock() + + with patch("app.db_sharding.scatter_gather.is_sharding_enabled", return_value=True): + with patch( + "app.db_sharding.scatter_gather.shard_ids_for_import", + return_value=["s1", "s2"], + ): + with patch( + "app.db_sharding.scatter_gather.scatter_gather_on_shards", + return_value=[10, 25], + ): + assert max_call_import_row_index(db, call_import_id) == 25 diff --git a/tests/test_services/test_ai/test_llm_gateway.py b/tests/test_services/test_ai/test_llm_gateway.py index 15f8033c..3e24de64 100644 --- a/tests/test_services/test_ai/test_llm_gateway.py +++ b/tests/test_services/test_ai/test_llm_gateway.py @@ -10,6 +10,7 @@ from app.services.ai.llm_gateway import ( apply_llm_gateway, CredentialRoutingContext, + get_credential_effective_routing_label, LITELLM_GATEWAY_PLACEHOLDER_API_KEY, normalize_bifrost_native_url, resolve_effective_routing, @@ -391,6 +392,23 @@ def test_credential_gateway_raises_when_no_base_url(): resolve_effective_routing(org_id, db, ctx) +def test_effective_routing_label_uses_credential_gateway_base_url(): + _set_platform_gateway(enabled=False) + org_id, db = _org_db({"enabled": False}) + provider = SimpleNamespace( + provider="custom", + routing_mode="gateway", + gateway_interface="native_openai", + gateway_base_url="http://localhost:8080/v1", + gateway_model="gpt-oss-120b", + gateway_auth_header=None, + gateway_auth_secret_env=None, + gateway_auth_secret=None, + gateway_extra_headers=None, + ) + assert get_credential_effective_routing_label(org_id, db, provider) == "bifrost" + + def test_resolve_litellm_model_uses_gateway_model_when_active(): ctx = CredentialRoutingContext(routing_mode="gateway", gateway_model="production-gpt4") assert resolve_litellm_model( diff --git a/tests/test_services/test_metric_studio_source_resolver.py b/tests/test_services/test_metric_studio_source_resolver.py new file mode 100644 index 00000000..5a0d6a58 --- /dev/null +++ b/tests/test_services/test_metric_studio_source_resolver.py @@ -0,0 +1,123 @@ +"""Tests for Metrics Studio source resolver.""" + +from uuid import uuid4 + +import pytest +from fastapi import HTTPException + +from app.models.database import ( + CallImport, + CallImportRow, + CallImportRowStatus, + CallImportStatus, +) +from app.services.metric_studio.source_resolver import resolve_source + + +def test_resolve_unknown_source_kind_raises(db_session, org_id, default_workspace): + with pytest.raises(HTTPException) as exc: + resolve_source( + db_session, + organization_id=org_id, + workspace_id=default_workspace.id, + source_kind="unknown", + source_ref="not-a-real-ref", + ) + assert exc.value.status_code == 400 + + +def test_resolve_call_import_row(db_session, org_id, default_workspace): + call_import = CallImport( + id=uuid4(), + organization_id=org_id, + workspace_id=default_workspace.id, + provider="exotel", + original_filename="studio.csv", + column_mapping={"external_call_id": "CallID", "transcript": "Transcript"}, + extra_columns=[], + total_rows=1, + completed_rows=1, + failed_rows=0, + status=CallImportStatus.COMPLETED, + ) + db_session.add(call_import) + db_session.flush() + + row = CallImportRow( + id=uuid4(), + call_import_id=call_import.id, + organization_id=org_id, + workspace_id=default_workspace.id, + row_index=0, + conversation_id="conv-studio-1", + transcript="Production CSV transcript", + status=CallImportRowStatus.COMPLETED, + ) + db_session.add(row) + db_session.commit() + + sample = resolve_source( + db_session, + organization_id=org_id, + workspace_id=default_workspace.id, + source_kind="call_import_row", + source_ref=str(row.id), + ) + assert sample.transcript == "Production CSV transcript" + assert sample.label == "conv-studio-1" + assert sample.metadata["call_import_id"] == str(call_import.id) + + +def test_resolve_call_import_row_uses_locate_when_sharded( + db_session, org_id, default_workspace, monkeypatch +): + """When rows live on shards, catalog-only queries must not be used.""" + row_id = uuid4() + call_import_id = uuid4() + fake_row = CallImportRow( + id=row_id, + call_import_id=call_import_id, + organization_id=org_id, + workspace_id=default_workspace.id, + row_index=0, + conversation_id="sharded-conv", + transcript="sharded transcript", + status=CallImportRowStatus.COMPLETED, + ) + call_import = CallImport( + id=call_import_id, + organization_id=org_id, + workspace_id=default_workspace.id, + provider="exotel", + original_filename="sharded.csv", + column_mapping={"external_call_id": "CallID"}, + extra_columns=[], + total_rows=1, + completed_rows=1, + failed_rows=0, + status=CallImportStatus.COMPLETED, + ) + db_session.add(call_import) + db_session.commit() + + def fake_locate(rid): + assert rid == row_id + return db_session, db_session, fake_row, "data-shard-01" + + monkeypatch.setattr( + "app.db_sharding.row_ops.locate_call_import_row", + fake_locate, + ) + monkeypatch.setattr( + "app.db_sharding.sessions.is_sharding_enabled", + lambda: True, + ) + + sample = resolve_source( + db_session, + organization_id=org_id, + workspace_id=default_workspace.id, + source_kind="call_import_row", + source_ref=str(row_id), + ) + assert sample.transcript == "sharded transcript" diff --git a/tests/test_workers/test_call_import_unified_dispatch.py b/tests/test_workers/test_call_import_unified_dispatch.py index 32c5ab22..03c18757 100644 --- a/tests/test_workers/test_call_import_unified_dispatch.py +++ b/tests/test_workers/test_call_import_unified_dispatch.py @@ -1,5 +1,6 @@ """Unit tests for the unified call-import eval dispatch pipeline.""" +import importlib from types import SimpleNamespace from unittest.mock import MagicMock, patch from uuid import uuid4 @@ -14,6 +15,13 @@ ) +def _import_task_module(): + """Load process_call_import_row with the test Celery wrapper.""" + from tests.test_workers.test_process_call_import_row import _load_task_module + + return _load_task_module() + + def _evaluation(**kwargs): defaults = dict( id=uuid4(), @@ -74,12 +82,11 @@ def test_needs_import_false_when_failed(): assert _needs_import_for_eval(row) is False -def test_needs_import_for_production_source_still_fetches_recording(): - """Production transcript runs skip diarisation but still import recordings - when a recording_url is present (same as diarised runs).""" +def test_needs_import_skipped_for_production_source(): + """Production transcript runs skip recording import even when a URL exists.""" row = _source_row() evaluation = _evaluation(transcript_source="production") - assert _needs_import_for_eval(row) is True + assert _needs_import_for_eval(row, evaluation) is False def test_needs_transcribe_after_recording_ready(): @@ -184,3 +191,66 @@ def fake_reserve(**kwargs): eval_row=eval_row, reserved_task_id="reserved-id", ) + + +def test_try_dispatch_skips_import_for_production_transcript(monkeypatch): + monkeypatch.setattr( + "app.db_sharding.sessions.is_sharding_enabled", + lambda: False, + ) + evaluation = _evaluation(transcript_source="production") + eval_row = SimpleNamespace(id=uuid4(), celery_task_id=None, status="pending") + call_import = SimpleNamespace( + id=evaluation.call_import_id, + organization_id=evaluation.organization_id, + workspace_id=evaluation.workspace_id, + provider=None, + telephony_integration_id=None, + ) + source_row = _source_row( + call_import_id=evaluation.call_import_id, + transcript="Agent: hello", + ) + import_called = {"value": False} + eval_called = {"value": False} + + class _AsyncResult: + id = "eval-task-123" + + import_mod = _import_task_module() + fake_import_task = SimpleNamespace( + apply_async=lambda *a, **kw: import_called.update({"value": True}) + or _AsyncResult(), + ) + monkeypatch.setattr(import_mod, "process_call_import_row_task", fake_import_task) + eval_mod = importlib.import_module("app.workers.tasks.evaluate_call_import_row") + fake_eval_task = SimpleNamespace( + apply_async=lambda *a, **kw: eval_called.update({"value": True}) + or _AsyncResult(), + ) + monkeypatch.setattr(eval_mod, "evaluate_call_import_row_task", fake_eval_task) + monkeypatch.setattr( + "app.workers.tasks.evaluate_call_import_row_core.row_needs_audio_phase", + lambda *_a, **_kw: False, + ) + + def fake_reserve(**kwargs): + kwargs["enqueue_fn"]("reserved-id") + return True + + monkeypatch.setattr( + "app.workers.concurrency.eval_dispatch._reserve_slot_and_enqueue", + fake_reserve, + ) + + result = _try_dispatch_single_row( + db=SimpleNamespace(commit=lambda: None, flush=lambda: None), + evaluation=evaluation, + eval_row=eval_row, + source_row=source_row, + call_import=call_import, + ) + + assert result == EvalDispatchOutcome("dispatched") + assert import_called["value"] is False + assert eval_called["value"] is True diff --git a/tests/test_workers/test_eval_dispatch_import.py b/tests/test_workers/test_eval_dispatch_import.py index 084b6af1..3e1fe1a2 100644 --- a/tests/test_workers/test_eval_dispatch_import.py +++ b/tests/test_workers/test_eval_dispatch_import.py @@ -2,6 +2,7 @@ from __future__ import annotations +from datetime import datetime, timezone from uuid import uuid4 from app.models.database import ( @@ -165,3 +166,17 @@ def test_recover_eval_row_for_eval_chain_clears_stale_import_failure(db_session) assert eval_row.status == "pending" assert eval_row.error_message is None assert eval_row.finished_at is None + + +def test_recover_eval_row_for_eval_chain_preserves_user_abort(db_session): + _, _, eval_row, _ = _seed_eval_row(db_session) + eval_row.status = "failed" + eval_row.error_message = "Evaluation cancelled by user" + eval_row.finished_at = datetime.now(timezone.utc) + db_session.commit() + + recover_eval_row_for_eval_chain(eval_row) + + assert eval_row.status == "failed" + assert eval_row.error_message == "Evaluation cancelled by user" + assert eval_row.finished_at is not None diff --git a/tests/test_workers/test_eval_transcribe_queue_routing.py b/tests/test_workers/test_eval_transcribe_queue_routing.py index b1a6a991..229cac9d 100644 --- a/tests/test_workers/test_eval_transcribe_queue_routing.py +++ b/tests/test_workers/test_eval_transcribe_queue_routing.py @@ -40,6 +40,16 @@ def stub_worker_task_modules(): fake_core = types.ModuleType("app.workers.tasks.evaluate_call_import_row_core") fake_core.row_needs_audio_phase = lambda *_a, **_kw: False + fake_core.EVAL_CANCELLED_BY_USER_ERROR = "Evaluation cancelled by user" + + def _is_eval_row_user_cancelled(eval_row): + return ( + (getattr(eval_row, "status", "") or "").lower() == "failed" + and (getattr(eval_row, "error_message", "") or "") + == fake_core.EVAL_CANCELLED_BY_USER_ERROR + ) + + fake_core.is_eval_row_user_cancelled = _is_eval_row_user_cancelled fake_process_import = types.ModuleType("app.workers.tasks.process_call_import_row") fake_process_import.process_call_import_row_task = MagicMock() diff --git a/tests/test_workers/test_fair_dispatch_retry.py b/tests/test_workers/test_fair_dispatch_retry.py index 3d4fd011..815ff190 100644 --- a/tests/test_workers/test_fair_dispatch_retry.py +++ b/tests/test_workers/test_fair_dispatch_retry.py @@ -101,6 +101,18 @@ def test_workspaces_with_pending_rows_includes_partial_status(db_session): assert workspace_id in workspaces +def test_workspaces_with_pending_rows_excludes_inactive_workspace(db_session): + workspace_id, _evaluation_id = _seed_partial_run_with_pending_retry(db_session) + + ws = db_session.query(Workspace).filter(Workspace.id == workspace_id).one() + ws.is_active = False + db_session.commit() + + workspaces = fair_dispatch._workspaces_with_pending_rows(db_session) + + assert workspace_id not in workspaces + + def test_evaluations_with_pending_rows_includes_partial_status(db_session): workspace_id, evaluation_id = _seed_partial_run_with_pending_retry(db_session) diff --git a/tests/test_workers/test_process_call_import_row.py b/tests/test_workers/test_process_call_import_row.py index 0dcdbc22..fc464a47 100644 --- a/tests/test_workers/test_process_call_import_row.py +++ b/tests/test_workers/test_process_call_import_row.py @@ -83,6 +83,53 @@ def _seed(db_session, *, row_count: int = 1): return org, call_import, rows +def _seed_transcript_only(db_session, *, row_count: int = 1): + """Generic CSV import batch with no telephony provider.""" + org = Organization(id=uuid4(), name="Transcript Only Org") + db_session.add(org) + workspace = Workspace( + id=uuid4(), + organization_id=org.id, + name="Default", + slug="default", + is_default=True, + ) + db_session.add(workspace) + db_session.commit() + + call_import = CallImport( + organization_id=org.id, + workspace_id=workspace.id, + provider="", + telephony_integration_id=None, + original_filename="transcripts.csv", + total_rows=row_count, + completed_rows=0, + failed_rows=0, + status=CallImportStatus.PROCESSING, + ) + db_session.add(call_import) + db_session.flush() + + rows = [] + for idx in range(row_count): + row = CallImportRow( + call_import_id=call_import.id, + organization_id=org.id, + workspace_id=workspace.id, + row_index=idx, + conversation_id=f"call-{idx}", + recording_url=None, + transcript=f"transcript {idx}", + status=CallImportRowStatus.PENDING, + ) + db_session.add(row) + rows.append(row) + db_session.commit() + + return org, call_import, rows + + class _FakeExotelClient: """Stand-in for ExotelClient that the worker can call.""" @@ -218,7 +265,8 @@ class _FakeBindTask: _bind_task_wrapped = True def __init__(self): - self.request = types.SimpleNamespace(id="test-task-id") + self.request = types.SimpleNamespace(id="test-task-id", retries=0) + self.max_retries = 3 def run(self, row_id, *args, **kwargs): return fn(self, row_id, *args, **kwargs) @@ -246,6 +294,11 @@ def retry(self, *args, **kwargs): return _CeleryDelegate(task) +def _reset_task_retry_state(task) -> None: + task.request = types.SimpleNamespace(id="test-task-id", retries=0) + task.max_retries = 3 + + def _load_task_module(): """Load the real task module even when conftest/API tests stub workers.tasks. @@ -449,6 +502,7 @@ def download_recording(self, _url): fake_s3 = _FakeS3(enabled=True) task_module = _patch_dependencies(monkeypatch, db_session, _ThrottledClient(), fake_s3) + _reset_task_retry_state(task_module.process_call_import_row_task) from app.workers.concurrency.telephony_credential_rate_limit import CreditStatus monkeypatch.setattr( @@ -618,6 +672,88 @@ def download_recording(self, _url): assert fake_s3.uploads == [] +def test_process_call_import_row_fails_on_repeated_credential_rejection( + db_session, monkeypatch +): + _, call_import, rows = _seed(db_session, row_count=1) + row = rows[0] + row.attempts = 1 + db_session.commit() + + from app.services.telephony.exotel_client import CredentialedRecordingThrottledError + + class _AuthRejectClient: + _credential_fingerprint = "fp-auth-reject" + + def download_recording(self, _url): + raise CredentialedRecordingThrottledError( + "Recording URL rejected credentials (HTTP 401)", + retry_after_seconds=15, + ) + + fake_s3 = _FakeS3(enabled=True) + task_module = _patch_dependencies( + monkeypatch, db_session, _AuthRejectClient(), fake_s3 + ) + from app.workers.concurrency.telephony_credential_rate_limit import CreditStatus + + monkeypatch.setattr( + "app.workers.concurrency.telephony_credential_rate_limit.consume_telephony_import_credit", + lambda _fp: CreditStatus(allowed=True, remaining=4), + ) + monkeypatch.setattr( + task_module.process_call_import_row_task, + "retry", + lambda exc, countdown: (_ for _ in ()).throw(RetryCalled((exc, countdown))), + ) + + result = task_module.process_call_import_row_task.run(str(row.id)) + + assert result["status"] == "failed" + db_session.refresh(row) + db_session.refresh(call_import) + assert row.status == CallImportRowStatus.FAILED + assert "credentials rejected" in (row.error_message or "").lower() + assert call_import.failed_rows == 1 + + +def test_process_call_import_row_fails_when_max_retries_exhausted( + db_session, monkeypatch +): + _, call_import, rows = _seed(db_session, row_count=1) + row = rows[0] + + from app.services.telephony.exotel_client import ExotelTransientError + + class _TransientClient: + def download_recording(self, _url): + raise ExotelTransientError("flaky network") + + fake_s3 = _FakeS3(enabled=True) + task_module = _patch_dependencies( + monkeypatch, db_session, _TransientClient(), fake_s3 + ) + task = task_module.process_call_import_row_task + _reset_task_retry_state(task) + task.request = types.SimpleNamespace(id="test-task-id", retries=3) + task.max_retries = 3 + try: + monkeypatch.setattr( + task, + "retry", + lambda exc, countdown: (_ for _ in ()).throw(RetryCalled((exc, countdown))), + ) + + result = task.run(str(row.id)) + + assert result["status"] == "failed" + db_session.refresh(row) + assert row.status == CallImportRowStatus.FAILED + assert "after 4 attempts" in (row.error_message or "").lower() + finally: + _reset_task_retry_state(task) + + def test_process_call_import_row_retries_on_transient_error(db_session, monkeypatch): _, _call_import, rows = _seed(db_session, row_count=1) row = rows[0] @@ -702,6 +838,36 @@ def test_process_call_import_row_fails_when_csv_omits_recording_url( assert call_import.status == CallImportStatus.FAILED +def test_process_call_import_row_completes_without_recording_url_for_generic_import( + db_session, monkeypatch +): + """Transcript-only rows on generic imports complete without downloading audio.""" + + _, call_import, rows = _seed_transcript_only(db_session, row_count=1) + row = rows[0] + + fake_client = _FakeExotelClient() + fake_s3 = _FakeS3(enabled=True) + task_module = _patch_dependencies(monkeypatch, db_session, fake_client, fake_s3) + public_calls = _patch_public_download( + monkeypatch, + return_value=(b"should-not-be-used", "audio/mpeg"), + ) + + result = task_module.process_call_import_row_task.run(str(row.id)) + + assert result["status"] == "completed" + assert result["s3_key"] is None + db_session.refresh(row) + db_session.refresh(call_import) + assert fake_client.calls == [] + assert public_calls == [] + assert fake_s3.uploads == [] + assert row.status == CallImportRowStatus.COMPLETED + assert row.recording_s3_key is None + assert call_import.completed_rows == 1 + + def test_process_call_import_row_exotel_csv_url_uses_credentialed_download( db_session, monkeypatch ): diff --git a/tests/test_workers/test_process_call_import_row_sharding.py b/tests/test_workers/test_process_call_import_row_sharding.py index fbcc6ab5..61b71f4f 100644 --- a/tests/test_workers/test_process_call_import_row_sharding.py +++ b/tests/test_workers/test_process_call_import_row_sharding.py @@ -131,7 +131,7 @@ def fake_enqueue( def test_production_eval_chain_skips_transcribe_and_redispatches( db_session, monkeypatch ): - """Production-transcript evals should import recordings then evaluate, not diarise.""" + """Production-transcript eval chains skip diarisation after import completes.""" org, call_import, rows = _seed(db_session, row_count=1) row = rows[0] row.transcript = "Agent: hello\nUser: hi" diff --git a/tests/test_workers/test_process_evaluator_result_helpers.py b/tests/test_workers/test_process_evaluator_result_helpers.py index 794248a3..0624134b 100644 --- a/tests/test_workers/test_process_evaluator_result_helpers.py +++ b/tests/test_workers/test_process_evaluator_result_helpers.py @@ -1,28 +1,28 @@ -"""Helper tests for process_evaluator_result task module.""" - -import importlib.util -from pathlib import Path - -_TASK_PATH = Path(__file__).resolve().parents[2] / "app" / "workers" / "tasks" / "process_evaluator_result.py" -_TASK_SPEC = importlib.util.spec_from_file_location("process_evaluator_result_under_test", _TASK_PATH) -process_evaluator_result = importlib.util.module_from_spec(_TASK_SPEC) -assert _TASK_SPEC is not None and _TASK_SPEC.loader is not None -_TASK_SPEC.loader.exec_module(process_evaluator_result) - - -def test_extract_audio_url_supports_smallest_recordings(): - call_data = {"recording_url": "https://audio.smallest.ai/call.wav"} - audio_url = process_evaluator_result._extract_audio_url(call_data, "smallest") - - assert audio_url == "https://audio.smallest.ai/call.wav" -"""Unit tests for process_evaluator_result helper utilities.""" - - -def test_extract_audio_url_supports_smallest_recordings(): - import importlib - - task_module = importlib.import_module("app.workers.tasks.process_evaluator_result") - - call_data = {"recording_url": "https://audio.smallest.ai/call.wav"} - - assert task_module._extract_audio_url(call_data, "smallest") == "https://audio.smallest.ai/call.wav" +"""Helper tests for process_evaluator_result task module.""" + +import importlib.util +from pathlib import Path + +_TASK_PATH = Path(__file__).resolve().parents[2] / "app" / "workers" / "tasks" / "process_evaluator_result.py" +_TASK_SPEC = importlib.util.spec_from_file_location("process_evaluator_result_under_test", _TASK_PATH) +process_evaluator_result = importlib.util.module_from_spec(_TASK_SPEC) +assert _TASK_SPEC is not None and _TASK_SPEC.loader is not None +_TASK_SPEC.loader.exec_module(process_evaluator_result) + + +def test_extract_audio_url_supports_smallest_recordings(): + call_data = {"recording_url": "https://audio.smallest.ai/call.wav"} + audio_url = process_evaluator_result._extract_audio_url(call_data, "smallest") + + assert audio_url == "https://audio.smallest.ai/call.wav" +"""Unit tests for process_evaluator_result helper utilities.""" + + +def test_extract_audio_url_supports_smallest_recordings(): + import importlib + + task_module = importlib.import_module("app.workers.tasks.process_evaluator_result") + + call_data = {"recording_url": "https://audio.smallest.ai/call.wav"} + + assert task_module._extract_audio_url(call_data, "smallest") == "https://audio.smallest.ai/call.wav"