diff --git a/Makefile b/Makefile index 24c474cf..395172ae 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,8 @@ PYTEST_FLAGS ?= -q TEST_DB_HOST ?= localhost TEST_DB_PORT ?= 5432 TEST_DB_NAME ?= efficientai +# Dedicated DB for test-docker-db / local Postgres runs (matches CI; not the dev DB). +TEST_DOCKER_DB_NAME ?= efficientai_test TEST_DB_USER ?= efficientai TEST_DB_PASSWORD ?= password TEST_DATABASE_URL ?= postgresql://$(TEST_DB_USER):$(TEST_DB_PASSWORD)@$(TEST_DB_HOST):$(TEST_DB_PORT)/$(TEST_DB_NAME) @@ -49,8 +51,11 @@ test-parallel: check-pytest-xdist ## Run the full test suite in parallel (pytest $(PYTEST) tests $(PYTEST_FLAGS) -n auto --dist loadscope $(PYTEST_ARGS) test-docker-db: check-pytest ## Run tests against running Docker Compose Postgres - TEST_DATABASE_URL="$(TEST_DATABASE_URL)" DATABASE_URL="$(TEST_DATABASE_URL)" \ - POSTGRES_HOST="$(TEST_DB_HOST)" POSTGRES_PORT="$(TEST_DB_PORT)" POSTGRES_DB="$(TEST_DB_NAME)" \ + @PGPASSWORD="$(TEST_DB_PASSWORD)" psql -h "$(TEST_DB_HOST)" -p "$(TEST_DB_PORT)" -U "$(TEST_DB_USER)" -d postgres -tc "SELECT 1 FROM pg_database WHERE datname='$(TEST_DOCKER_DB_NAME)'" | grep -q 1 \ + || PGPASSWORD="$(TEST_DB_PASSWORD)" psql -h "$(TEST_DB_HOST)" -p "$(TEST_DB_PORT)" -U "$(TEST_DB_USER)" -d postgres -c "CREATE DATABASE \"$(TEST_DOCKER_DB_NAME)\";" + TEST_DATABASE_URL="postgresql://$(TEST_DB_USER):$(TEST_DB_PASSWORD)@$(TEST_DB_HOST):$(TEST_DB_PORT)/$(TEST_DOCKER_DB_NAME)" \ + DATABASE_URL="postgresql://$(TEST_DB_USER):$(TEST_DB_PASSWORD)@$(TEST_DB_HOST):$(TEST_DB_PORT)/$(TEST_DOCKER_DB_NAME)" \ + POSTGRES_HOST="$(TEST_DB_HOST)" POSTGRES_PORT="$(TEST_DB_PORT)" POSTGRES_DB="$(TEST_DOCKER_DB_NAME)" \ POSTGRES_USER="$(TEST_DB_USER)" POSTGRES_PASSWORD="$(TEST_DB_PASSWORD)" \ $(PYTEST) tests $(PYTEST_FLAGS) $(PYTEST_ARGS) diff --git a/app/api/v1/routes/auth.py b/app/api/v1/routes/auth.py index 2a4437c5..9077f120 100644 --- a/app/api/v1/routes/auth.py +++ b/app/api/v1/routes/auth.py @@ -58,6 +58,13 @@ consume_reference_code, validate_reference_code_for_signup, ) +from app.services.invitation_service import ( + InvitationError, + accept_invitation as accept_invitation_record, + get_invitation_preview, + get_valid_pending_invitation_by_token, +) +from app.api.v1.routes.profile import get_current_user router = APIRouter(prefix="/auth", tags=["Authentication"]) @@ -95,12 +102,28 @@ class SignupRequest(BaseModel): first_name: Optional[str] = Field(default=None, max_length=255) last_name: Optional[str] = Field(default=None, max_length=255) reference_code: Optional[str] = Field(default=None, max_length=64) + invite_token: Optional[str] = Field(default=None, max_length=255) + + +class InvitationPreviewResponse(BaseModel): + organization_name: Optional[str] = None + email: str + role: str + expires_at: datetime + status: str + user_exists: bool = False + has_password: bool = False + + +class AcceptInviteByTokenRequest(BaseModel): + token: str = Field(min_length=1, max_length=255) class LoginRequest(BaseModel): email: EmailStr password: str organization_id: Optional[str] = None + invite_token: Optional[str] = Field(default=None, max_length=255) class LoginOrgOption(BaseModel): @@ -290,6 +313,38 @@ def _issue_session_tokens( ) +@router.get("/invitations/preview/{token}", response_model=InvitationPreviewResponse) +def preview_invitation(token: str, db: Session = Depends(get_db)) -> InvitationPreviewResponse: + """Public preview of an organization invite (no auth required).""" + try: + preview = get_invitation_preview(db, token) + except InvitationError as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc + return InvitationPreviewResponse(**preview) + + +@router.post("/invitations/accept-by-token", response_model=TokenResponse) +def accept_invitation_by_token( + payload: AcceptInviteByTokenRequest, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +) -> TokenResponse: + """Accept an invitation and return a session scoped to the invited organization.""" + try: + invitation = get_valid_pending_invitation_by_token(db, payload.token) + member = accept_invitation_record(db, invitation, current_user) + except InvitationError as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc + + role_value = member.role.value if hasattr(member.role, "value") else member.role + return _issue_session_tokens( + db, + user=current_user, + organization_id=invitation.organization_id, + role_value=role_value, + ) + + @router.post("/signup", response_model=TokenResponse) def signup(payload: SignupRequest, db: Session = Depends(get_db)) -> TokenResponse: """ @@ -307,31 +362,81 @@ def signup(payload: SignupRequest, db: Session = Depends(get_db)) -> TokenRespon ) reference_row = None - if settings.AUTH_GATED_SIGNUP_ENABLED: + invite_token = (payload.invite_token or "").strip() or None + pending_invitation = None + + if invite_token: + try: + pending_invitation = get_valid_pending_invitation_by_token(db, invite_token) + except InvitationError as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc + if pending_invitation.email.lower() != payload.email.lower(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Email does not match the invitation.", + ) + elif settings.AUTH_GATED_SIGNUP_ENABLED: reference_row = validate_reference_code_for_signup(db, payload.reference_code) existing = db.query(User).filter(User.email == payload.email).first() - if existing: + if existing and existing.password_hash: raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail="An account with this email already exists. Try signing in instead.", ) - org_name = (payload.organization_name or payload.email.split("@")[0] + "'s Org").strip() _validate_password_or_400(payload.password) - user = User( - email=payload.email, - password_hash=hash_password(payload.password), - first_name=payload.first_name, - last_name=payload.last_name, - name=((payload.first_name or "") + " " + (payload.last_name or "")).strip() or None, - is_active=True, - auth_provider="local", - ) + if existing and not existing.password_hash: + user = existing + user.password_hash = hash_password(payload.password) + if payload.first_name: + user.first_name = payload.first_name + if payload.last_name: + user.last_name = payload.last_name + if payload.first_name or payload.last_name: + user.name = ((payload.first_name or "") + " " + (payload.last_name or "")).strip() or user.name + user.is_active = True + if not user.auth_provider: + user.auth_provider = "local" + db.flush() + else: + user = User( + email=payload.email, + password_hash=hash_password(payload.password), + first_name=payload.first_name, + last_name=payload.last_name, + name=((payload.first_name or "") + " " + (payload.last_name or "")).strip() or None, + is_active=True, + auth_provider="local", + ) + db.add(user) + db.flush() + + if pending_invitation is not None: + member = accept_invitation_record( + db, + pending_invitation, + user, + require_email_match=True, + ) + if reference_row is not None: + consume_reference_code(db, reference_row) + user.last_login_at = datetime.now(timezone.utc) + db.commit() + db.refresh(user) + + role_value = member.role.value if hasattr(member.role, "value") else member.role + return _issue_session_tokens( + db, + user=user, + organization_id=pending_invitation.organization_id, + role_value=role_value, + ) + + org_name = (payload.organization_name or payload.email.split("@")[0] + "'s Org").strip() organization = Organization(name=org_name) db.add(organization) - db.add(user) db.flush() membership = OrganizationMember( @@ -395,6 +500,27 @@ def login(payload: LoginRequest, db: Session = Depends(get_db)) -> LoginResponse .order_by(OrganizationMember.joined_at.asc()) .all() ) + + invite_token = (payload.invite_token or "").strip() or None + if not memberships and invite_token: + try: + invitation = get_valid_pending_invitation_by_token(db, invite_token) + if invitation.email.lower() != user.email.lower(): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="This invitation was sent to a different email address.", + ) + member = accept_invitation_record(db, invitation, user) + organization = ( + db.query(Organization) + .filter(Organization.id == invitation.organization_id) + .first() + ) + if organization is not None and organization.is_active: + memberships = [(member, organization)] + except InvitationError as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc + if not memberships: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, diff --git a/app/api/v1/routes/call_import_evaluations.py b/app/api/v1/routes/call_import_evaluations.py index 39675b13..18749f6d 100644 --- a/app/api/v1/routes/call_import_evaluations.py +++ b/app/api/v1/routes/call_import_evaluations.py @@ -563,6 +563,29 @@ def _serialize_eval( user_emails = emails_for_user_ids(db, user_ids_from_evaluations([row])) created_email, updated_email = actor_emails_for_evaluation(row, user_emails) + from app.workers.tasks.evaluate_call_import_row_core import ( + count_distinct_llm_configs_for_metrics, + ) + + selected_set = set(selected_ids) + scoring_metrics = [m for m in metrics if m.id in selected_set] + expected_llm_calls_per_row = count_distinct_llm_configs_for_metrics( + scoring_metrics, + overrides=( + row.metric_llm_overrides + if isinstance(row.metric_llm_overrides, dict) + else {} + ), + run_provider=(row.llm_provider or "").strip() or None, + run_model=(row.llm_model or "").strip() or None, + run_llm_config=( + row.llm_config if isinstance(getattr(row, "llm_config", None), dict) else None + ), + run_credential_id=( + str(row.llm_credential_id) if row.llm_credential_id else None + ), + ) + return CallImportEvaluationResponse( id=row.id, call_import_id=row.call_import_id, @@ -618,6 +641,7 @@ def _serialize_eval( ), transcript_source=(row.transcript_source or "diarised"), sibling_evaluation_ids=list(sibling_evaluation_ids or []), + expected_llm_calls_per_row=expected_llm_calls_per_row, started_at=row.started_at, finished_at=row.finished_at, created_at=row.created_at, @@ -1014,7 +1038,6 @@ async def create_call_import_evaluation( from app.models.enums import CallImportParameterType, CallImportStatus from app.services.call_imports.bulk_ops import ( count_all_source_rows, - count_completed_source_rows, count_source_rows_with_production_transcript, ) @@ -1124,8 +1147,12 @@ async def create_call_import_evaluation( db.refresh(call_import) starting_from_mapped = True + all_source_row_count = count_all_source_rows(db, call_import.id) + if use_diarised: - total_row_count = count_completed_source_rows(db, call_import.id) + # Diarized runs materialize every import row; recording fetch may + # still be pending when switching from a production eval-primary run. + total_row_count = all_source_row_count else: # Production runs score CSV text — rows need not wait for # recording fetch to finish before they are evaluable. @@ -1133,6 +1160,17 @@ async def create_call_import_evaluation( db, call_import.id ) + if ( + use_diarised + and not starting_from_mapped + and all_source_row_count > 0 + and call_import.status != CallImportStatus.DELETING + ): + call_import.status = CallImportStatus.PROCESSING + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + requested_sources: List[str] = list(payload.transcript_sources) if ( @@ -1209,7 +1247,7 @@ def _name_for_source(source: str) -> Optional[str]: primary_evaluation = created_evaluations[0] sibling_ids = [e.id for e in created_evaluations[1:]] - if not total_row_count and not starting_from_mapped: + if not all_source_row_count and not starting_from_mapped: for evaluation in created_evaluations: evaluation.status = "completed" db.commit() diff --git a/app/api/v1/routes/call_imports.py b/app/api/v1/routes/call_imports.py index 094845ae..70551508 100644 --- a/app/api/v1/routes/call_imports.py +++ b/app/api/v1/routes/call_imports.py @@ -148,7 +148,10 @@ def _serialize_call_import( user_emails: Optional[Dict[UUID, str]] = None, ) -> CallImportResponse: """Catalog parent fields; counters come from SQL rollup (not Redis merge).""" - from app.services.call_imports.bulk_ops import rollup_call_import_batch_status + from app.services.call_imports.bulk_ops import ( + _latest_evaluation_status, + rollup_call_import_batch_status, + ) from app.services.call_imports.progress_counters import ( clear_import_progress_redis, read_import_progress, @@ -185,6 +188,9 @@ def _serialize_call_import( update={ "completed_rows": completed, "failed_rows": failed, + "latest_evaluation_status": _latest_evaluation_status( + db, call_import.id + ), "created_by_email": created_email, "last_updated_by_email": updated_email, } diff --git a/app/api/v1/routes/evaluator_results.py b/app/api/v1/routes/evaluator_results.py index f44baebd..962207c6 100644 --- a/app/api/v1/routes/evaluator_results.py +++ b/app/api/v1/routes/evaluator_results.py @@ -8,7 +8,7 @@ from app.database import get_db from app.dependencies import get_organization_id, get_workspace_id, get_api_key -from app.models.database import EvaluatorResult, Evaluator, Metric, EvaluatorResultStatus, Scenario, CallRecording +from app.models.database import EvaluatorResult, Evaluator, Metric, EvaluatorResultStatus, Scenario, CallRecording, Agent, Integration import random from datetime import datetime from app.models.schemas import ( @@ -34,6 +34,32 @@ router = APIRouter(prefix="/evaluator-results", tags=["evaluator-results"]) +def _lookup_evaluator_result( + db: Session, + id: str, + organization_id: UUID, + workspace_id: UUID, +) -> EvaluatorResult | None: + """Resolve an evaluator result by UUID or 6-digit result_id.""" + try: + result_uuid = UUID(id) + return db.query(EvaluatorResult).filter( + and_( + EvaluatorResult.id == result_uuid, + EvaluatorResult.organization_id == organization_id, + EvaluatorResult.workspace_id == workspace_id, + ) + ).first() + except ValueError: + return db.query(EvaluatorResult).filter( + and_( + EvaluatorResult.result_id == id, + EvaluatorResult.organization_id == organization_id, + EvaluatorResult.workspace_id == workspace_id, + ) + ).first() + + def _detach_call_recordings_from_evaluator_results( db: Session, evaluator_result_ids: List[UUID], @@ -617,6 +643,107 @@ def get_evaluator_result_metrics( } +@router.get("/{id}/audio") +async def stream_evaluator_result_audio( + id: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """Stream evaluator result audio from S3 or proxy auth-gated provider URLs.""" + from io import BytesIO + + import requests as http_requests + from fastapi.responses import RedirectResponse, StreamingResponse + + from app.core.encryption import decrypt_api_key + from app.services.storage.s3_service import s3_service + from app.services.voice_providers.vapi_recording import is_presigned_storage_url + from app.workers.tasks.process_evaluator_result import _extract_audio_url + + del api_key + + result = _lookup_evaluator_result(db, id, organization_id, workspace_id) + if not result: + raise HTTPException(status_code=404, detail="Evaluator result not found") + + s3_key = result.audio_s3_key + if not s3_key and isinstance(result.call_data, dict): + s3_key = result.call_data.get("recording_s3_key") + + if s3_key: + if not s3_service.is_enabled(): + raise HTTPException(status_code=400, detail="S3 storage is not configured") + try: + audio_bytes = s3_service.download_file_by_key(s3_key) + except Exception as exc: + raise HTTPException(status_code=404, detail="Audio file not found in storage") from exc + + extension = s3_key.rsplit(".", 1)[-1].lower() if "." in s3_key else "wav" + content_type_map = {"webm": "audio/webm", "mp3": "audio/mpeg", "wav": "audio/wav", "ogg": "audio/ogg"} + content_type = content_type_map.get(extension, "audio/wav") + return StreamingResponse( + BytesIO(audio_bytes), + media_type=content_type, + headers={ + "Content-Disposition": f'inline; filename="result_{result.result_id}.{extension}"', + }, + ) + + call_data = result.call_data if isinstance(result.call_data, dict) else {} + platform = (result.provider_platform or "").lower() + audio_url = _extract_audio_url(call_data, platform) + if not audio_url: + raise HTTPException(status_code=404, detail="No recording available") + + if platform in {"retell", "smallest"}: + return RedirectResponse(audio_url) + + if platform == "vapi" and is_presigned_storage_url(audio_url): + return RedirectResponse(audio_url) + + decrypted_key = None + agent = db.query(Agent).filter(Agent.id == result.agent_id).first() if result.agent_id else None + if agent and agent.voice_ai_integration_id: + integration = db.query(Integration).filter( + Integration.id == agent.voice_ai_integration_id, + Integration.organization_id == organization_id, + ).first() + if integration: + try: + decrypted_key = decrypt_api_key(integration.api_key) + except Exception: + decrypted_key = None + + headers = None + if platform == "elevenlabs" and decrypted_key: + headers = {"xi-api-key": decrypted_key} + elif platform == "vapi" and decrypted_key: + headers = {"Authorization": f"Bearer {decrypted_key}"} + elif platform == "vapi": + return RedirectResponse(audio_url) + + if platform == "elevenlabs" and not headers: + raise HTTPException(status_code=400, detail="Agent integration not found for ElevenLabs audio") + + upstream = http_requests.get(audio_url, headers=headers, stream=True, timeout=60) + if upstream.status_code != 200: + raise HTTPException( + status_code=upstream.status_code, + detail=f"Provider audio fetch failed ({upstream.status_code})", + ) + + content_type = upstream.headers.get("content-type", "audio/mpeg") + return StreamingResponse( + upstream.iter_content(chunk_size=8192), + media_type=content_type, + headers={ + "Content-Disposition": f'inline; filename="result_{result.result_id}.mp3"', + }, + ) + + @router.post("", response_model=EvaluatorResultResponse, status_code=status.HTTP_201_CREATED) def create_evaluator_result_manual( result_data: EvaluatorResultCreateManual, @@ -791,20 +918,19 @@ def re_evaluate_result( if resp.status_code == 200: audio_bytes = resp.content elif platform == "vapi": - audio_url = ( - call_data.get("recordingUrl") - or call_data.get("stereoRecordingUrl") - or artifact.get("recordingUrl") - or artifact.get("stereoRecordingUrl") - or mono_recording.get("combinedUrl") - or recording_urls.get("combined_url") - or recording_urls.get("stereo_url") - or call_data.get("recordingUrl") - or provider_payload.get("recordingUrl") - or provider_payload.get("stereoRecordingUrl") + from app.services.voice_providers.vapi_recording import ( + extract_vapi_recording_url, + is_presigned_storage_url, ) + + audio_url = extract_vapi_recording_url(call_data) if audio_url: - resp = _http.get(audio_url, timeout=120) + headers = ( + None + if is_presigned_storage_url(audio_url) + else ({"Authorization": f"Bearer {decrypted_key}"} if decrypted_key else None) + ) + resp = _http.get(audio_url, headers=headers, timeout=120) if resp.status_code == 200: audio_bytes = resp.content elif platform == "smallest": diff --git a/app/api/v1/routes/iam.py b/app/api/v1/routes/iam.py index 09321275..169f82e2 100644 --- a/app/api/v1/routes/iam.py +++ b/app/api/v1/routes/iam.py @@ -23,17 +23,11 @@ ) from app.core.password import hash_password, validate_password_strength from app.core.auth.refresh_tokens import revoke_all_user_refresh_tokens +from app.services.invitation_service import invitation_to_response_dict, to_aware_utc router = APIRouter(prefix="/iam", tags=["IAM"]) -def _to_aware_utc(dt: datetime) -> datetime: - """Normalize datetimes to timezone-aware UTC for safe comparisons.""" - if dt.tzinfo is None: - return dt.replace(tzinfo=timezone.utc) - return dt.astimezone(timezone.utc) - - def get_user_from_api_key(api_key: str, db: Session) -> User: """ Get user associated with API key. @@ -309,19 +303,13 @@ async def invite_user( db.commit() db.refresh(invitation) - # Get organization name org = db.query(Organization).filter(Organization.id == organization_id).first() - - return { - "id": invitation.id, - "organization_id": invitation.organization_id, - "email": invitation.email, - "role": invitation.role, - "status": invitation.status, - "expires_at": invitation.expires_at, - "created_at": invitation.created_at, - "organization_name": org.name if org else None - } + + return invitation_to_response_dict( + db, + invitation, + organization_name=org.name if org else None, + ) @router.get("/invitations", response_model=List[InvitationResponse], operation_id="listInvitations") @@ -345,21 +333,12 @@ async def list_invitations( result = [] for invitation in invitations: # Mark expired pending invites in the DB but do not surface them here. - if _to_aware_utc(invitation.expires_at) < datetime.now(timezone.utc): + if to_aware_utc(invitation.expires_at) < datetime.now(timezone.utc): invitation.status = InvitationStatus.EXPIRED db.commit() continue - - result.append({ - "id": invitation.id, - "organization_id": invitation.organization_id, - "email": invitation.email, - "role": invitation.role, - "status": invitation.status, - "expires_at": invitation.expires_at, - "created_at": invitation.created_at, - "organization_name": org_name - }) + + result.append(invitation_to_response_dict(db, invitation, organization_name=org_name)) return result diff --git a/app/api/v1/routes/profile.py b/app/api/v1/routes/profile.py index f137e596..09cdc31f 100644 --- a/app/api/v1/routes/profile.py +++ b/app/api/v1/routes/profile.py @@ -20,19 +20,11 @@ InvitationUpdate ) from app.core.password import hash_password - - -def _to_aware_utc(dt: datetime) -> datetime: - """Normalize a datetime to timezone-aware UTC. - - SQLite loses tz info on round-trip, so `invitation.expires_at` can come - back naive even though we wrote it aware. Normalizing here keeps the - comparison with ``datetime.now(timezone.utc)`` safe across SQLite (dev, - tests) and Postgres (prod). - """ - if dt.tzinfo is None: - return dt.replace(tzinfo=timezone.utc) - return dt.astimezone(timezone.utc) +from app.services.invitation_service import ( + InvitationError, + accept_invitation as accept_invitation_record, + to_aware_utc, +) router = APIRouter(prefix="/profile", tags=["Profile"]) @@ -264,7 +256,7 @@ async def get_my_invitations( # Check if expired. Normalize expires_at to tz-aware UTC so the # comparison works on SQLite (which drops tz info) as well as # Postgres. - if invitation.status == InvitationStatus.PENDING and _to_aware_utc(invitation.expires_at) < now_utc: + if invitation.status == InvitationStatus.PENDING and to_aware_utc(invitation.expires_at) < now_utc: invitation.status = InvitationStatus.EXPIRED db.commit() @@ -302,48 +294,12 @@ async def accept_invitation( if not invitation: raise HTTPException(status_code=404, detail="Invitation not found") - - if invitation.status != InvitationStatus.PENDING: - # `status` is a plain String column on the model, so it comes back - # as a str here - don't assume it has an `.value` attribute. - current_status = getattr(invitation.status, "value", invitation.status) - raise HTTPException( - status_code=400, - detail=f"Cannot accept invitation with status: {current_status}" - ) - - if _to_aware_utc(invitation.expires_at) < datetime.now(timezone.utc): - invitation.status = InvitationStatus.EXPIRED - db.commit() - raise HTTPException(status_code=400, detail="Invitation has expired") - - # Check if user is already a member - existing_member = db.query(OrganizationMember).filter( - OrganizationMember.organization_id == invitation.organization_id, - OrganizationMember.user_id == current_user.id - ).first() - - if existing_member: - raise HTTPException( - status_code=400, - detail="User is already a member of this organization" - ) - - # Create organization membership - member = OrganizationMember( - organization_id=invitation.organization_id, - user_id=current_user.id, - role=invitation.role - ) - db.add(member) - - # Update invitation - invitation.status = InvitationStatus.ACCEPTED - invitation.accepted_at = datetime.now(timezone.utc) - invitation.invited_user_id = current_user.id - - db.commit() - + + try: + accept_invitation_record(db, invitation, current_user) + except InvitationError as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc + return {"message": "Invitation accepted successfully"} diff --git a/app/config.py b/app/config.py index fc194f81..8242409c 100644 --- a/app/config.py +++ b/app/config.py @@ -123,6 +123,7 @@ class Settings(BaseSettings): # Frontend FRONTEND_DIR: str = "./frontend/dist" + FRONTEND_BASE_URL: str = "" # Content Security Policy (Report-Only by default; set CSP_REPORT_ONLY=false to enforce) CSP_ENABLED: bool = True @@ -450,8 +451,8 @@ def load_config_from_file(config_path: str) -> None: settings.DEBUG = app_config["debug"] if "secret_key" in app_config: settings.SECRET_KEY = app_config["secret_key"] - - if "server" in config_data: + if "frontend_base_url" in app_config: + settings.FRONTEND_BASE_URL = app_config["frontend_base_url"] server_config = config_data["server"] if "host" in server_config: settings.HOST = server_config["host"] diff --git a/app/models/schemas.py b/app/models/schemas.py index 25241e77..40e988e9 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -410,6 +410,13 @@ class AgentResponse(BaseModel): voice_ai_integration_id: Optional[UUID] voice_ai_agent_id: Optional[str] provider_prompt: Optional[str] = None + prompt_variables: Optional[Dict[str, str]] = None + silence_hangup_secs: int = Field( + default=15, + ge=0, + le=600, + description="End live calls after this many seconds of silence (0 disables)", + ) provider_prompt_synced_at: Optional[datetime] = None created_at: datetime updated_at: datetime @@ -688,6 +695,8 @@ class InvitationResponse(BaseModel): expires_at: datetime created_at: datetime organization_name: Optional[str] = None # Include organization name + invite_path: Optional[str] = None + invite_url: Optional[str] = None @field_validator('role', mode='before') @classmethod @@ -3328,6 +3337,13 @@ class CallImportResponse(BaseModel): failed_rows: int status: CallImportStatus error_message: Optional[str] = None + latest_evaluation_status: Optional[str] = Field( + None, + description=( + "Status of the most recent evaluation run for this batch, " + "when any evaluation exists." + ), + ) created_at: datetime updated_at: datetime created_by_email: Optional[str] = None @@ -4147,6 +4163,13 @@ class CallImportEvaluationResponse(BaseModel): # field so the frontend can deep-link to either run). Empty for all # other reads. sibling_evaluation_ids: List[UUID] = Field(default_factory=list) + expected_llm_calls_per_row: Optional[int] = Field( + None, + description=( + "Number of distinct LLM API calls made per evaluation row " + "(one per unique provider/model/config among selected metrics)." + ), + ) started_at: Optional[datetime] = None finished_at: Optional[datetime] = None created_at: datetime diff --git a/app/services/call_imports/bulk_ops.py b/app/services/call_imports/bulk_ops.py index 09c1acee..f58826ba 100644 --- a/app/services/call_imports/bulk_ops.py +++ b/app/services/call_imports/bulk_ops.py @@ -927,6 +927,30 @@ def _aggregate_import_row_status_counts( ) +def _call_import_status_from_evaluation_status(eval_status: str) -> CallImportStatus: + """Map a terminal evaluation run status onto batch status.""" + normalized = (eval_status or "").strip().lower() + if normalized == "completed": + return CallImportStatus.COMPLETED + if normalized == "partial": + return CallImportStatus.PARTIAL + if normalized in ("failed", "cancelled"): + return CallImportStatus.FAILED + if normalized in ("pending", "running"): + return CallImportStatus.PROCESSING + return CallImportStatus.PROCESSING + + +def _latest_evaluation_status(db: Session, call_import_id: UUID) -> Optional[str]: + row = ( + db.query(CallImportEvaluation.status) + .filter(CallImportEvaluation.call_import_id == call_import_id) + .order_by(CallImportEvaluation.created_at.desc()) + .first() + ) + return row[0] if row else None + + def rollup_call_import_batch_status(db: Session, call_import: CallImport) -> None: """Recompute batch counters and terminal status on the parent import. @@ -973,7 +997,13 @@ def rollup_call_import_batch_status(db: Session, call_import: CallImport) -> Non ) if has_evaluations: if failed == 0 and completed == 0: - call_import.status = CallImportStatus.FAILED + latest_eval_status = _latest_evaluation_status(db, call_import.id) + if latest_eval_status is not None: + call_import.status = _call_import_status_from_evaluation_status( + latest_eval_status + ) + else: + call_import.status = CallImportStatus.FAILED else: call_import.status = CallImportStatus.PARTIAL else: diff --git a/app/services/invitation_service.py b/app/services/invitation_service.py new file mode 100644 index 00000000..810b8de0 --- /dev/null +++ b/app/services/invitation_service.py @@ -0,0 +1,209 @@ +"""Shared invitation validation, acceptance, and URL helpers.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Optional + +from sqlalchemy.orm import Session + +from app.config import settings +from app.models.database import ( + Invitation, + InvitationStatus, + Organization, + OrganizationMember, + User, +) +from app.services.workspace_rbac import backfill_org_workspace_memberships + + +class InvitationError(Exception): + """Raised when an invitation cannot be used.""" + + def __init__(self, detail: str, status_code: int = 400): + super().__init__(detail) + self.detail = detail + self.status_code = status_code + + +def to_aware_utc(dt: datetime) -> datetime: + """Normalize a datetime to timezone-aware UTC.""" + if dt.tzinfo is None: + return dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) + + +def build_invite_path(token: str) -> str: + """Build the frontend-relative invite path for the given token.""" + return f"/invite/{token}" + + +def build_invite_url(token: str) -> str: + """Build an absolute invite link (for server-side/email use when base URL is configured).""" + base = (settings.FRONTEND_BASE_URL or "").strip().rstrip("/") + if not base and settings.CORS_ORIGINS: + base = settings.CORS_ORIGINS[0].rstrip("/") + if not base: + base = "http://localhost:8000" + return f"{base}{build_invite_path(token)}" + + +def get_invitation_by_token(db: Session, token: str) -> Optional[Invitation]: + return db.query(Invitation).filter(Invitation.token == token).first() + + +def expire_invitation_if_needed(db: Session, invitation: Invitation) -> None: + if ( + invitation.status == InvitationStatus.PENDING + and to_aware_utc(invitation.expires_at) < datetime.now(timezone.utc) + ): + invitation.status = InvitationStatus.EXPIRED + db.commit() + + +def get_valid_pending_invitation_by_token(db: Session, token: str) -> Invitation: + invitation = get_invitation_by_token(db, token) + if invitation is None: + raise InvitationError("Invitation not found", status_code=404) + + expire_invitation_if_needed(db, invitation) + + if invitation.status == InvitationStatus.EXPIRED: + raise InvitationError("Invitation has expired", status_code=410) + + if invitation.status != InvitationStatus.PENDING: + current_status = getattr(invitation.status, "value", invitation.status) + raise InvitationError( + f"Invitation is no longer valid (status: {current_status})", + status_code=410, + ) + + return invitation + + +def accept_invitation( + db: Session, + invitation: Invitation, + user: User, + *, + require_email_match: bool = True, +) -> OrganizationMember: + """ + Accept an invitation and add the user to the organization. + + Idempotent when the user is already a member of the invited organization. + """ + if require_email_match and invitation.email.lower() != user.email.lower(): + raise InvitationError( + "This invitation was sent to a different email address", + status_code=403, + ) + + expire_invitation_if_needed(db, invitation) + if invitation.status == InvitationStatus.EXPIRED: + raise InvitationError("Invitation has expired", status_code=410) + + if invitation.status != InvitationStatus.PENDING: + current_status = getattr(invitation.status, "value", invitation.status) + raise InvitationError( + f"Cannot accept invitation with status: {current_status}", + status_code=400, + ) + + existing_member = ( + db.query(OrganizationMember) + .filter( + OrganizationMember.organization_id == invitation.organization_id, + OrganizationMember.user_id == user.id, + ) + .first() + ) + if existing_member: + invitation.status = InvitationStatus.ACCEPTED + invitation.accepted_at = datetime.now(timezone.utc) + invitation.invited_user_id = user.id + db.commit() + return existing_member + + member = OrganizationMember( + organization_id=invitation.organization_id, + user_id=user.id, + role=invitation.role, + ) + db.add(member) + + invitation.status = InvitationStatus.ACCEPTED + invitation.accepted_at = datetime.now(timezone.utc) + invitation.invited_user_id = user.id + + db.flush() + backfill_org_workspace_memberships(db, organization_id=invitation.organization_id) + db.commit() + db.refresh(member) + return member + + +def get_invitation_preview(db: Session, token: str) -> dict: + """Return public preview data for an invite token.""" + invitation = get_invitation_by_token(db, token) + if invitation is None: + raise InvitationError("Invitation not found", status_code=404) + + expire_invitation_if_needed(db, invitation) + + org = ( + db.query(Organization) + .filter(Organization.id == invitation.organization_id) + .first() + ) + existing_user = db.query(User).filter(User.email == invitation.email).first() + user_exists = existing_user is not None + has_password = bool(existing_user and existing_user.password_hash) + + status_value = getattr(invitation.status, "value", invitation.status) + return { + "organization_name": org.name if org else None, + "email": invitation.email, + "role": invitation.role, + "expires_at": invitation.expires_at, + "status": status_value, + "user_exists": user_exists, + "has_password": has_password, + } + + +def invitation_to_response_dict( + db: Session, + invitation: Invitation, + *, + organization_name: Optional[str] = None, +) -> dict: + """Build an InvitationResponse-compatible dict including invite_url when pending.""" + if organization_name is None: + org = ( + db.query(Organization) + .filter(Organization.id == invitation.organization_id) + .first() + ) + organization_name = org.name if org else None + + status_value = getattr(invitation.status, "value", invitation.status) + invite_path = None + invite_url = None + if status_value == InvitationStatus.PENDING.value: + invite_path = build_invite_path(invitation.token) + invite_url = build_invite_url(invitation.token) + + return { + "id": invitation.id, + "organization_id": invitation.organization_id, + "email": invitation.email, + "role": invitation.role, + "status": invitation.status, + "expires_at": invitation.expires_at, + "created_at": invitation.created_at, + "organization_name": organization_name, + "invite_path": invite_path, + "invite_url": invite_url, + } diff --git a/app/services/metric_failure_policy.py b/app/services/metric_failure_policy.py index 3599f175..c40c5fc6 100644 --- a/app/services/metric_failure_policy.py +++ b/app/services/metric_failure_policy.py @@ -109,6 +109,24 @@ def suggest_failure_policy( failure_child_names=negative_children, ) + if is_parent and selection_mode == "single_choice": + children = [str(n).strip() for n in (child_names or []) if str(n).strip()] + # Never auto-flag bare "No"/"False" for pick-one categories — for + # Yes/No style metrics (AI reveal, bot gibberish) the good outcome + # is often the "No" child. Descriptive negative labels still apply. + negative_children = [ + n + for n in children + if _label_looks_negative(n) + and normalize_label(n) not in ("no", "false") + ] + if negative_children: + return MetricFailurePolicy( + metric_id=metric_id, + failure_values=[normalize_label(l) for l in negative_children], + ) + return MetricFailurePolicy(metric_id=metric_id, failure_values=[]) + labels = [str(l).strip() for l in observed_labels if str(l).strip()] negative_labels = [l for l in labels if _label_looks_negative(l)] if negative_labels: diff --git a/app/services/testing/test_agent_bridge_service.py b/app/services/testing/test_agent_bridge_service.py index 4a05cdb0..b13b78b1 100644 --- a/app/services/testing/test_agent_bridge_service.py +++ b/app/services/testing/test_agent_bridge_service.py @@ -1000,23 +1000,19 @@ async def _poll_call_results( if resp.status_code == 200: audio_bytes = resp.content elif plat == "vapi": - artifact = call_metrics.get("artifact", {}) if isinstance(call_metrics, dict) else {} - recording = artifact.get("recording", {}) if isinstance(artifact, dict) else {} - mono_recording = recording.get("mono", {}) if isinstance(recording, dict) else {} - audio_url = ( - call_metrics.get("recordingUrl") - or call_metrics.get("stereoRecordingUrl") - or artifact.get("recordingUrl") - or artifact.get("stereoRecordingUrl") - or mono_recording.get("combinedUrl") - or recording_urls.get("combined_url") - or recording_urls.get("stereo_url") - or call_metrics.get("recordingUrl") - or provider_payload.get("recordingUrl") - or provider_payload.get("stereoRecordingUrl") + from app.services.voice_providers.vapi_recording import ( + extract_vapi_recording_url, + is_presigned_storage_url, ) + + audio_url = extract_vapi_recording_url(call_metrics) if audio_url: - resp = _http.get(audio_url, timeout=120) + headers = ( + None + if is_presigned_storage_url(audio_url) + else {"Authorization": f"Bearer {provider.api_key}"} + ) + resp = _http.get(audio_url, headers=headers, timeout=120) if resp.status_code == 200: audio_bytes = resp.content diff --git a/app/services/voice_providers/vapi_recording.py b/app/services/voice_providers/vapi_recording.py new file mode 100644 index 00000000..6565ff0d --- /dev/null +++ b/app/services/voice_providers/vapi_recording.py @@ -0,0 +1,47 @@ +"""Helpers for resolving Vapi call recording URLs.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + + +def is_presigned_storage_url(url: Optional[str]) -> bool: + """Return True when the URL includes S3/R2 presigned query parameters.""" + if not url: + return False + return "X-Amz-Signature=" in url or "X-Amz-Algorithm=" in url + + +def extract_vapi_recording_url(call_data: Any) -> Optional[str]: + """ + Resolve the best Vapi recording URL from provider call_data. + + HIPAA deployments expose time-limited presigned URLs on the artifact; + those must be preferred over raw R2 object URLs that reject browser playback. + """ + if not isinstance(call_data, dict): + return None + + artifact = call_data.get("artifact") if isinstance(call_data.get("artifact"), dict) else {} + recording = artifact.get("recording") if isinstance(artifact.get("recording"), dict) else {} + mono = recording.get("mono") if isinstance(recording.get("mono"), dict) else {} + recording_urls = call_data.get("recording_urls") if isinstance(call_data.get("recording_urls"), dict) else {} + provider_payload = ( + call_data.get("provider_payload") if isinstance(call_data.get("provider_payload"), dict) else {} + ) + + return ( + artifact.get("presignedMonoUrl") + or artifact.get("presignedStereoUrl") + or call_data.get("presignedMonoUrl") + or call_data.get("presignedStereoUrl") + or call_data.get("recordingUrl") + or call_data.get("stereoRecordingUrl") + or artifact.get("recordingUrl") + or artifact.get("stereoRecordingUrl") + or mono.get("combinedUrl") + or recording_urls.get("combined_url") + or recording_urls.get("stereo_url") + or provider_payload.get("recordingUrl") + or provider_payload.get("stereoRecordingUrl") + ) diff --git a/app/workers/tasks/evaluate_call_import_row.py b/app/workers/tasks/evaluate_call_import_row.py index c4090f6c..3c31bf79 100644 --- a/app/workers/tasks/evaluate_call_import_row.py +++ b/app/workers/tasks/evaluate_call_import_row.py @@ -31,6 +31,8 @@ from app.workers.tasks.evaluate_call_import_row_core import ( as_json_dict, build_all_columns_block, + build_llm_config_buckets, + bucket_needs_comparison_pair, build_parent_groups, categorize_metrics, load_enabled_metrics, @@ -43,6 +45,7 @@ ) from app.workers.tasks.helpers.llm_evaluation import ( evaluate_with_llm, + flatten_metric_groups, handle_llm_evaluation_error, ) @@ -52,7 +55,6 @@ _was_cancelled_externally = was_cancelled_externally _build_all_columns_block = build_all_columns_block _categorize_metrics = categorize_metrics -_build_parent_groups = build_parent_groups _rollup_parent = rollup_parent _metric_text_references_production = metric_text_references_production @@ -91,21 +93,16 @@ def _run_llm_scoring( transcript: str, production_transcript: str, diarised_transcript: str, - transcript_metrics: list[Metric], - comparison_metrics: list[Metric], + llm_config_buckets: dict, + comparison_metric_ids: set[str], all_columns_block: str | None, ai_providers: list, llm_provider: str | None, llm_model: str | None, llm_credential_id: str | None, llm_config: dict | None, - metric_llm_overrides: dict, discover_new_metrics: bool, running_discovered_metrics: list, - running_discovered_by_parent: dict[UUID, list], - parents_by_id: dict[UUID, Metric], - children_by_parent: dict[UUID, list[Metric]], - standalone_metrics: list[Metric], transcript_unavailable: bool, missing_label: str, ) -> dict[str, Any]: @@ -120,182 +117,86 @@ def _run_llm_scoring( else None ) - run_provider = (llm_provider or "").strip() or None - run_model = (llm_model or "").strip() or None - run_llm_config = llm_config if isinstance(llm_config, dict) else None - run_credential_id = (llm_credential_id or "").strip() or None - overrides = metric_llm_overrides if isinstance(metric_llm_overrides, dict) else {} - - if comparison_metrics: - for cmp_metric in comparison_metrics: - override = overrides.get(str(cmp_metric.id)) or {} - provider = override.get("provider") or run_provider or None - model = override.get("model") or run_model or None - llm_cfg = override.get("llm_config") or run_llm_config - credential_id = override.get("credential_id") or run_credential_id - evaluator_obj = None - if provider and model: - evaluator_obj = SimpleNamespace( - llm_provider=provider, - llm_model=model, - llm_config=llm_cfg, - llm_credential_id=credential_id, - custom_prompt=None, - ) - try: - llm_db = SessionLocal() - try: - cmp_scores, _eval_time = evaluate_with_llm( - transcription="", - llm_metrics=[cmp_metric], - ai_providers=ai_providers, - organization_id=organization_id, - result_id=result_id, - db=llm_db, - evaluator=evaluator_obj, - agent=None, - persona=None, - scenario=None, - parent_metric=None, - running_discovered=None, - all_columns_block=all_columns_block, - comparison_pair=( - production_transcript, - diarised_transcript, - ), - ) - finally: - llm_db.close() - metric_scores.update(cmp_scores) - except Exception as exc: # noqa: BLE001 - logger.exception( - "[CallImportEval {}] Transcript-compare LLM " - "evaluation failed for metric={} provider={} model={}", - eval_row_id, - cmp_metric.id, - provider, - model, - ) - metric_scores.update( - handle_llm_evaluation_error([cmp_metric], exc) - ) - evaluation_failed = True - primary_error_message = primary_error_message or str(exc) - - if transcript_metrics and transcript: - def _llm_config_key(cfg: dict | None) -> str | None: - if not cfg: - return None - return json.dumps(cfg, sort_keys=True, default=str) - - def _resolve_pm( - metric: Metric, - ) -> tuple[str | None, str | None, dict | None, str | None]: - override = overrides.get(str(metric.id)) or {} - provider = override.get("provider") or run_provider or None - model = override.get("model") or run_model or None - llm_cfg = override.get("llm_config") or run_llm_config - credential_id = override.get("credential_id") or run_credential_id - return provider, model, llm_cfg, credential_id - - BucketKey = tuple[tuple[str | None, str | None, str | None, str | None], UUID | None] - groups: dict[BucketKey, list[Metric]] = {} - for metric in standalone_metrics: - provider, model, llm_cfg, credential_id = _resolve_pm(metric) - groups.setdefault( - ((provider, model, _llm_config_key(llm_cfg), credential_id), None), - [], - ).append(metric) - for parent_id, children in children_by_parent.items(): - provider, model, llm_cfg, credential_id = _resolve_pm(children[0]) - groups.setdefault( - ((provider, model, _llm_config_key(llm_cfg), credential_id), parent_id), - [], - ).extend(children) - - metric_discovery_emitted = False - for (config, parent_id), bucket in groups.items(): - provider, model, llm_config_key, credential_id = config - llm_cfg = json.loads(llm_config_key) if llm_config_key else None - evaluator_obj = None - if provider and model: - evaluator_obj = SimpleNamespace( - llm_provider=provider, - llm_model=model, - llm_config=llm_cfg, - llm_credential_id=credential_id, - custom_prompt=None, - ) - parent_metric = parents_by_id.get(parent_id) if parent_id else None - running_discovered = ( - running_discovered_by_parent.get(parent_id, []) - if parent_id is not None - else [] + if not llm_config_buckets: + return { + "metric_scores": metric_scores, + "evaluation_failed": evaluation_failed, + "primary_error_message": primary_error_message, + } + + metric_discovery_emitted = False + + for config, groups in llm_config_buckets.items(): + provider, model, llm_config_key, credential_id = config + llm_cfg = json.loads(llm_config_key) if llm_config_key else None + evaluator_obj = None + if provider and model: + evaluator_obj = SimpleNamespace( + llm_provider=provider, + llm_model=model, + llm_config=llm_cfg, + llm_credential_id=credential_id, + custom_prompt=None, ) - emit_metric_discovery = ( - discover_new_metrics and not metric_discovery_emitted + bucket_metrics = flatten_metric_groups(groups) + emit_metric_discovery = discover_new_metrics and not metric_discovery_emitted + bucket_comparison_pair: tuple[str, str] | None = None + if bucket_needs_comparison_pair( + groups, + production_transcript=production_transcript, + diarised_transcript=diarised_transcript, + comparison_metric_ids=comparison_metric_ids, + ): + bucket_comparison_pair = ( + production_transcript, + diarised_transcript, ) - bucket_comparison_pair: tuple[str, str] | None = None - if ( - production_transcript - and diarised_transcript - and ( - ( - parent_metric is not None - and _metric_text_references_production(parent_metric) - ) - or any( - _metric_text_references_production(m, parent=parent_metric) - for m in bucket - ) - ) - ): - bucket_comparison_pair = ( - production_transcript, - diarised_transcript, - ) + bucket_has_transcript_metrics = any( + str(metric.id) not in comparison_metric_ids for metric in bucket_metrics + ) + bucket_transcription = ( + transcript if bucket_has_transcript_metrics else "" + ) + try: + llm_db = SessionLocal() try: - llm_db = SessionLocal() - try: - llm_scores, _eval_time = evaluate_with_llm( - transcription=transcript, - llm_metrics=bucket, - ai_providers=ai_providers, - organization_id=organization_id, - result_id=result_id, - db=llm_db, - evaluator=evaluator_obj, - agent=None, - persona=None, - scenario=None, - parent_metric=parent_metric, - running_discovered=running_discovered, - all_columns_block=all_columns_block, - comparison_pair=bucket_comparison_pair, - discover_new_metrics=emit_metric_discovery, - running_discovered_metrics=( - running_discovered_metrics - if emit_metric_discovery - else None - ), - ) - finally: - llm_db.close() - if emit_metric_discovery: - metric_discovery_emitted = True - metric_scores.update(llm_scores) - except Exception as exc: # noqa: BLE001 - logger.exception( - "[CallImportEval {}] LLM evaluation failed for " - "provider={} model={} parent={}", - eval_row_id, - provider, - model, - parent_id, - ) - metric_scores.update(handle_llm_evaluation_error(bucket, exc)) - evaluation_failed = True - primary_error_message = str(exc) + llm_scores, _eval_time = evaluate_with_llm( + transcription=bucket_transcription, + llm_metrics=bucket_metrics, + ai_providers=ai_providers, + organization_id=organization_id, + result_id=result_id, + db=llm_db, + evaluator=evaluator_obj, + agent=None, + persona=None, + scenario=None, + all_columns_block=all_columns_block, + comparison_pair=bucket_comparison_pair, + discover_new_metrics=emit_metric_discovery, + running_discovered_metrics=( + running_discovered_metrics + if emit_metric_discovery + else None + ), + metric_groups=groups, + ) + finally: + llm_db.close() + if emit_metric_discovery: + metric_discovery_emitted = True + metric_scores.update(llm_scores) + except Exception as exc: # noqa: BLE001 + logger.exception( + "[CallImportEval {}] LLM evaluation failed for " + "provider={} model={}", + eval_row_id, + provider, + model, + ) + metric_scores.update(handle_llm_evaluation_error(bucket_metrics, exc)) + evaluation_failed = True + primary_error_message = str(exc) return { "metric_scores": metric_scores, @@ -583,15 +484,15 @@ def evaluate_call_import_row_task( .all() ) - parents_by_id: dict[UUID, Metric] = {} - children_by_parent: dict[UUID, list[Metric]] = {} - standalone_metrics: list[Metric] = [] running_discovered_metrics: list = [] running_discovered_by_parent: dict[UUID, list] = {} - if transcript_metrics and transcript: - parents_by_id, children_by_parent, standalone_metrics = ( - _build_parent_groups(catalog_db, transcript_metrics) - ) + llm_metrics_for_scoring: list[Metric] = list(comparison_metrics) + if transcript: + llm_metrics_for_scoring = transcript_metrics + comparison_metrics + + llm_config_buckets: dict = {} + comparison_metric_ids = {str(m.id) for m in comparison_metrics} + if llm_metrics_for_scoring: if bool(getattr(evaluation, "discover_new_metrics", False)): from app.api.v1.routes.call_import_evaluations import ( _get_running_discovered_metrics, @@ -613,7 +514,10 @@ def evaluate_call_import_row_task( _get_running_discovered_labels, ) - for parent_id, children in children_by_parent.items(): + parents_by_id, children_by_parent, _ = build_parent_groups( + catalog_db, llm_metrics_for_scoring + ) + for parent_id in children_by_parent: parent_metric = parents_by_id.get(parent_id) if parent_metric is None: continue @@ -636,14 +540,38 @@ def evaluate_call_import_row_task( ) ) + overrides = ( + evaluation.metric_llm_overrides + if isinstance(evaluation.metric_llm_overrides, dict) + else {} + ) + llm_config_buckets = build_llm_config_buckets( + catalog_db, + llm_metrics_for_scoring, + overrides=overrides, + run_provider=(evaluation.llm_provider or "").strip() or None, + run_model=(evaluation.llm_model or "").strip() or None, + run_llm_config=( + evaluation.llm_config + if isinstance(evaluation.llm_config, dict) + else None + ), + run_credential_id=( + str(evaluation.llm_credential_id) + if evaluation.llm_credential_id + else None + ), + running_discovered_by_parent=running_discovered_by_parent, + ) + scoring_inputs = { "eval_row_id": eval_row.id, "organization_id": evaluation.organization_id, "transcript": transcript, "production_transcript": production_transcript, "diarised_transcript": diarised_transcript, - "transcript_metrics": transcript_metrics, - "comparison_metrics": comparison_metrics, + "llm_config_buckets": llm_config_buckets, + "comparison_metric_ids": comparison_metric_ids, "all_columns_block": all_columns_block, "ai_providers": ai_providers, "llm_provider": evaluation.llm_provider, @@ -654,15 +582,10 @@ def evaluate_call_import_row_task( else None ), "llm_config": evaluation.llm_config, - "metric_llm_overrides": evaluation.metric_llm_overrides, "discover_new_metrics": bool( getattr(evaluation, "discover_new_metrics", False) ), "running_discovered_metrics": running_discovered_metrics, - "running_discovered_by_parent": running_discovered_by_parent, - "parents_by_id": parents_by_id, - "children_by_parent": children_by_parent, - "standalone_metrics": standalone_metrics, "transcript_unavailable": transcript_unavailable, "missing_label": missing_label, "pre_llm_metric_scores": dict(metric_scores), diff --git a/app/workers/tasks/evaluate_call_import_row_core.py b/app/workers/tasks/evaluate_call_import_row_core.py index 98d9d8f2..30148887 100644 --- a/app/workers/tasks/evaluate_call_import_row_core.py +++ b/app/workers/tasks/evaluate_call_import_row_core.py @@ -16,6 +16,7 @@ Metric, ) from app.workers.tasks.helpers.constants import AUDIO_ONLY_METRIC_NAMES +from app.workers.tasks.helpers.llm_evaluation import MetricPromptGroup from app.workers.tasks.helpers.score_utils import get_metric_type_value EVAL_CANCELLED_BY_USER_ERROR: str = "Evaluation cancelled by user" @@ -219,6 +220,145 @@ def build_parent_groups( return parents_by_id, children_by_parent, standalone +LlmConfigKey = tuple[str | None, str | None, str | None, str | None] + + +def _llm_config_key(cfg: dict | None) -> str | None: + import json + + if not cfg: + return None + return json.dumps(cfg, sort_keys=True, default=str) + + +def resolve_metric_llm_config( + metric: Metric, + *, + overrides: dict, + run_provider: str | None, + run_model: str | None, + run_llm_config: dict | None, + run_credential_id: str | None, +) -> LlmConfigKey: + override = overrides.get(str(metric.id)) or {} + provider = override.get("provider") or run_provider or None + model = override.get("model") or run_model or None + llm_cfg = override.get("llm_config") or run_llm_config + credential_id = override.get("credential_id") or run_credential_id + return (provider, model, _llm_config_key(llm_cfg), credential_id) + + +def build_metric_prompt_groups( + db, + metrics: list[Metric], + *, + running_discovered_by_parent: dict[UUID, list] | None = None, +) -> list[MetricPromptGroup]: + parents_by_id, children_by_parent, standalone = build_parent_groups(db, metrics) + discovered_map = running_discovered_by_parent or {} + groups: list[MetricPromptGroup] = [] + if standalone: + groups.append(MetricPromptGroup(None, standalone, None)) + for parent_id, children in children_by_parent.items(): + parent_metric = parents_by_id.get(parent_id) + if parent_metric is None: + groups.append(MetricPromptGroup(None, children, None)) + else: + groups.append( + MetricPromptGroup( + parent_metric, + children, + discovered_map.get(parent_id), + ) + ) + return groups + + +def build_llm_config_buckets( + db, + metrics: list[Metric], + *, + overrides: dict, + run_provider: str | None, + run_model: str | None, + run_llm_config: dict | None, + run_credential_id: str | None, + running_discovered_by_parent: dict[UUID, list] | None = None, +) -> dict[LlmConfigKey, list[MetricPromptGroup]]: + """Group metrics by LLM config; each bucket holds prompt groups for one call.""" + buckets: dict[LlmConfigKey, list[Metric]] = {} + for metric in metrics: + key = resolve_metric_llm_config( + metric, + overrides=overrides, + run_provider=run_provider, + run_model=run_model, + run_llm_config=run_llm_config, + run_credential_id=run_credential_id, + ) + buckets.setdefault(key, []).append(metric) + + return { + config: build_metric_prompt_groups( + db, + bucket_metrics, + running_discovered_by_parent=running_discovered_by_parent, + ) + for config, bucket_metrics in buckets.items() + } + + +def bucket_needs_comparison_pair( + groups: list[MetricPromptGroup], + *, + production_transcript: str, + diarised_transcript: str, + comparison_metric_ids: set[str], +) -> bool: + if not production_transcript or not diarised_transcript: + return False + for group in groups: + if group.parent_metric is not None and metric_text_references_production( + group.parent_metric + ): + return True + for metric in group.metrics: + if str(metric.id) in comparison_metric_ids: + return True + if metric_text_references_production( + metric, parent=group.parent_metric + ): + return True + return False + + +def count_distinct_llm_configs_for_metrics( + metrics: list[Metric], + *, + overrides: dict, + run_provider: str | None, + run_model: str | None, + run_llm_config: dict | None, + run_credential_id: str | None, +) -> int: + """Expected LLM calls per row (one per distinct LLM config among metrics).""" + configs: set[LlmConfigKey] = set() + for metric in metrics: + if (metric.name or "").strip().lower() in AUDIO_ONLY_METRIC_NAMES: + continue + configs.add( + resolve_metric_llm_config( + metric, + overrides=overrides, + run_provider=run_provider, + run_model=run_model, + run_llm_config=run_llm_config, + run_credential_id=run_credential_id, + ) + ) + return len(configs) + + _TERMINAL_ROW_STATUSES = frozenset({"completed", "failed"}) diff --git a/app/workers/tasks/evaluate_studio_run_item.py b/app/workers/tasks/evaluate_studio_run_item.py index 96a56d53..83a4b6e8 100644 --- a/app/workers/tasks/evaluate_studio_run_item.py +++ b/app/workers/tasks/evaluate_studio_run_item.py @@ -20,7 +20,8 @@ from app.services.metric_studio.source_resolver import resolve_source from app.workers.config import celery_app from app.workers.tasks.evaluate_call_import_row_core import ( - build_parent_groups, + bucket_needs_comparison_pair, + build_llm_config_buckets, categorize_metrics, ) from app.workers.tasks.helpers.audio_evaluation import ( @@ -29,6 +30,7 @@ ) from app.workers.tasks.helpers.llm_evaluation import ( evaluate_with_llm, + flatten_metric_groups, handle_llm_evaluation_error, ) @@ -166,74 +168,47 @@ def evaluate_studio_run_item_task(self, result_row_id: str) -> dict[str, Any]: ) if llm_metrics and transcript: - parents_by_id, children_by_parent, standalone = build_parent_groups( - db, llm_metrics - ) result_id = f"studio:{result_row.id}" - - for parent_id, children in children_by_parent.items(): - parent = parents_by_id.get(parent_id) - if not parent or not children: - continue - try: + production_text = (sample.transcript or "").strip() + diarised_text = (sample.diarised_transcript or "").strip() + comparison_ids = { + str(m.id) + for m in llm_metrics + if getattr(m, "compare_transcripts", False) + } + buckets = build_llm_config_buckets( + db, + llm_metrics, + overrides={}, + run_provider=None, + run_model=None, + run_llm_config=None, + run_credential_id=None, + ) + try: + for _config, groups in buckets.items(): comparison_pair = None - if any(getattr(m, "compare_transcripts", False) for m in children): - comparison_pair = ( - sample.transcript or "", - sample.diarised_transcript or sample.transcript or "", - ) + if bucket_needs_comparison_pair( + groups, + production_transcript=production_text, + diarised_transcript=diarised_text, + comparison_metric_ids=comparison_ids, + ): + comparison_pair = (production_text, diarised_text) + bucket_metrics = flatten_metric_groups(groups) scores, _ = evaluate_with_llm( transcription=transcript, - llm_metrics=children, + llm_metrics=bucket_metrics, ai_providers=ai_providers, organization_id=run.organization_id, result_id=result_id, db=db, - parent_metric=parent, comparison_pair=comparison_pair, + metric_groups=groups, ) metric_scores.update(scores) - except Exception as llm_err: - metric_scores.update( - handle_llm_evaluation_error(children, llm_err) - ) - - if standalone: - try: - comparison_standalone = [ - m - for m in standalone - if getattr(m, "compare_transcripts", False) - ] - transcript_standalone = [ - m for m in standalone if m not in comparison_standalone - ] - if transcript_standalone: - scores, _ = evaluate_with_llm( - transcription=transcript, - llm_metrics=transcript_standalone, - ai_providers=ai_providers, - organization_id=run.organization_id, - result_id=result_id, - db=db, - ) - metric_scores.update(scores) - for metric in comparison_standalone: - scores, _ = evaluate_with_llm( - transcription=transcript, - llm_metrics=[metric], - ai_providers=ai_providers, - organization_id=run.organization_id, - result_id=result_id, - db=db, - comparison_pair=( - sample.transcript or "", - sample.diarised_transcript or sample.transcript or "", - ), - ) - metric_scores.update(scores) - except Exception as llm_err: - metric_scores.update(handle_llm_evaluation_error(standalone, llm_err)) + except Exception as llm_err: + metric_scores.update(handle_llm_evaluation_error(llm_metrics, llm_err)) result_row.metric_scores = metric_scores flag_modified(result_row, "metric_scores") diff --git a/app/workers/tasks/helpers/llm_evaluation.py b/app/workers/tasks/helpers/llm_evaluation.py index 66f78c14..33d24247 100644 --- a/app/workers/tasks/helpers/llm_evaluation.py +++ b/app/workers/tasks/helpers/llm_evaluation.py @@ -1,11 +1,31 @@ """LLM-based evaluation: prompt building and response parsing.""" +from __future__ import annotations + import json import re import time +from dataclasses import dataclass from typing import Any, Optional from uuid import UUID + +@dataclass +class MetricPromptGroup: + """One prompt section: flat metrics or a categorization parent + children.""" + + parent_metric: Any | None + metrics: list + running_discovered: list | None = None + + +def flatten_metric_groups(metric_groups: list[MetricPromptGroup]) -> list: + """All leaf metrics across groups (for token budgeting and mapping).""" + out: list = [] + for group in metric_groups: + out.extend(group.metrics) + return out + from loguru import logger from app.models.database import ModelProvider @@ -124,6 +144,56 @@ def _parent_key(parent_metric) -> str: return (parent_metric.name or "parent").lower().replace(" ", "_") +def _child_slug(child_metric) -> str: + """Stable slug for a categorization child label name.""" + return child_metric.name.lower().replace(" ", "_") + + +def _namespaced_child_key(parent_metric, child_metric) -> str: + """LLM JSON key for a child boolean scoped under its parent. + + Prevents collisions when multiple categorization parents (e.g. two + Yes/No categories) share one batched JSON response. + """ + return f"{_parent_key(parent_metric)}__{_child_slug(child_metric)}" + + +def _use_namespaced_child_keys( + metric_groups: list[MetricPromptGroup] | None, +) -> bool: + """True when batched ``metric_groups`` includes categorization parents.""" + if metric_groups is None: + return False + return any(g.parent_metric is not None for g in metric_groups) + + +def _read_child_boolean_raw( + evaluation_data: dict, + response_keys: list[str], + parent_metric, + child, + *, + use_namespaced: bool, +) -> Any: + """Read a child's boolean from LLM JSON (namespaced key, then bare slug).""" + child_key = _child_slug(child) + if use_namespaced: + ns_key = _namespaced_child_key(parent_metric, child) + raw = evaluation_data.get(ns_key) + if raw is None: + matched = find_matching_key(ns_key, response_keys) + if matched: + raw = evaluation_data.get(matched) + if raw is not None: + return raw + raw = evaluation_data.get(child_key) + if raw is None: + matched = find_matching_key(child.name, response_keys) + if matched: + raw = evaluation_data.get(matched) + return raw + + def _sequence_key(parent_metric) -> str: """LLM JSON key holding the temporal flow of children for a parent. @@ -312,6 +382,8 @@ def _render_parent_block( parent_metric, children: list, running_discovered: list | None = None, + *, + use_namespaced_child_keys: bool = False, ) -> str: """Build the per-parent prompt section for a hierarchical group. @@ -334,16 +406,27 @@ def _render_parent_block( f"Context: {parent_desc}\n" ) if selection_mode == "single_choice": + child_key_hint = ( + "namespaced child keys (`parent__child`) listed below" + if use_namespaced_child_keys + else "child keys listed below" + ) block += ( "Mode: SINGLE-CHOICE. Pick EXACTLY ONE child label below that " "best describes what happened in this call. Output a JSON " - f'field `{parent_key}` set to the chosen child key, AND set ' - "every child key to true/false such that EXACTLY ONE is true. " + f'field `{parent_key}` set to the chosen child slug (without ' + f"the parent prefix), AND set every {child_key_hint} to " + "true/false such that EXACTLY ONE is true. " "Any other configuration is invalid.\n" ) else: + child_key_hint = ( + "namespaced child key (`parent__child`)" + if use_namespaced_child_keys + else "child key" + ) block += ( - "Mode: MULTI-LABEL. Set each child key to true/false " + f"Mode: MULTI-LABEL. Set each {child_key_hint} to true/false " "INDEPENDENTLY. Some siblings are logically contradictory " "(e.g., 'customer_completed_survey' and 'angry_hangup' cannot " "both be true). MAINTAIN LOGICAL CONSISTENCY: do not mark " @@ -352,7 +435,11 @@ def _render_parent_block( block += "Children (set each true/false):\n" for child in children: - child_key = child.name.lower().replace(" ", "_") + child_key = ( + _namespaced_child_key(parent_metric, child) + if use_namespaced_child_keys + else _child_slug(child) + ) child_desc = child.description or f"Detect {child.name}" block += f'- "{child_key}" (true/false): {child_desc}\n' # When the user attached an illustrative example to this label @@ -454,6 +541,62 @@ def _render_parent_block( return block +def _render_flat_metric_lines(metrics: list) -> str: + """Render standalone (non-hierarchical) metric definition lines.""" + prompt = "" + for metric in metrics: + metric_key = metric.name.lower().replace(" ", "_") + metric_desc = metric.description or f"Evaluate {metric.name}" + m_type = get_metric_type_value(metric) + custom_type = _get_custom_data_type(metric) + + if m_type == "text": + prompt += ( + f'\n- "{metric_key}" (free-form text, 1-3 concise sentences, ' + f'plain string): {metric_desc}' + ) + continue + + line_added = False + if custom_type == "enum": + options = _get_enum_options(metric) + if options: + opts_str = ", ".join(f'"{o}"' for o in options) + prompt += f'\n- "{metric_key}" (one of: {opts_str}): {metric_desc}' + line_added = True + + if not line_added and custom_type == "number_range": + rng = _get_number_range(metric) + if rng: + bounds = [] + if rng.get("min") is not None: + bounds.append(f"min={rng['min']}") + if rng.get("max") is not None: + bounds.append(f"max={rng['max']}") + if rng.get("step") is not None: + bounds.append(f"step={rng['step']}") + bound_str = ", ".join(bounds) if bounds else "numeric value" + prompt += f'\n- "{metric_key}" (numeric, {bound_str}): {metric_desc}' + line_added = True + + if not line_added: + if m_type == "rating": + prompt += f'\n- "{metric_key}" (rating 0.0-1.0): {metric_desc}' + elif m_type == "boolean": + prompt += f'\n- "{metric_key}" (true/false): {metric_desc}' + elif m_type == "number": + prompt += f'\n- "{metric_key}" (numeric value): {metric_desc}' + + if _wants_rationale(metric): + prompt += ( + f'\n- "{_rationale_key(metric_key)}" (free-form text, ' + f'1-2 concise sentences explaining why "{metric_key}" was chosen): ' + f"Justification for the value above. Reference specific lines or " + f"behaviors from the transcript when possible." + ) + return prompt + + def build_evaluation_prompt( transcription: str, llm_metrics: list, @@ -468,6 +611,7 @@ def build_evaluation_prompt( comparison_pair: tuple[str, str] | None = None, discover_new_metrics: bool = False, running_discovered_metrics: list | None = None, + metric_groups: list[MetricPromptGroup] | None = None, ) -> str: """ Build the evaluation prompt for LLM-based metric evaluation. @@ -508,6 +652,9 @@ def build_evaluation_prompt( ``evaluate_call_import_row``). Can be combined with ``parent_metric`` so a categorisation parent can score against the pair. + metric_groups: Optional list of :class:`MetricPromptGroup` for + multi-parent / mixed flat+hierarchical prompts in a single + LLM call. When set, ``parent_metric`` is ignored. Returns: Complete evaluation prompt string @@ -633,6 +780,29 @@ def build_evaluation_prompt( ## Metrics to Evaluate (use EXACT keys below) """ + if metric_groups is not None: + use_namespaced = _use_namespaced_child_keys(metric_groups) + for group in metric_groups: + if group.parent_metric is not None: + prompt += _render_parent_block( + group.parent_metric, + group.metrics, + running_discovered=group.running_discovered, + use_namespaced_child_keys=use_namespaced, + ) + elif group.metrics: + prompt += _render_flat_metric_lines(group.metrics) + if discover_new_metrics: + prompt += _render_discovered_metrics_block( + running_discovered_metrics + ) + prompt += _build_response_format_instructions( + llm_metrics=flatten_metric_groups(metric_groups), + metric_groups=metric_groups, + discover_new_metrics=discover_new_metrics, + ) + return prompt + if parent_metric is not None: # Hierarchical mode: render ONE category block with the children # plus a sequence array. Falls through to the format @@ -658,85 +828,117 @@ def build_evaluation_prompt( ) return prompt + prompt += _render_flat_metric_lines(llm_metrics) + + if discover_new_metrics: + prompt += _render_discovered_metrics_block( + running_discovered_metrics + ) + + prompt += _build_response_format_instructions( + llm_metrics, + discover_new_metrics=discover_new_metrics, + ) + return prompt + + +def _append_parent_response_example( + instructions: str, + parent_metric, + llm_metrics: list, + *, + use_namespaced_child_keys: bool = False, +) -> str: + """Append JSON example lines for one categorization parent group.""" + parent_key = _parent_key(parent_metric) + sequence_key = _sequence_key(parent_metric) + selection_mode = (parent_metric.selection_mode or "multi_label").lower() + child_keys = [_child_slug(c) for c in llm_metrics] + if not child_keys: + child_keys = ["example_child"] + chosen_child = child_keys[0] + if selection_mode == "single_choice": + instructions += f' "{parent_key}": "{chosen_child}",\n' + for i, ck in enumerate(child_keys): + if use_namespaced_child_keys and i < len(llm_metrics): + json_key = _namespaced_child_key(parent_metric, llm_metrics[i]) + elif use_namespaced_child_keys: + json_key = f"{parent_key}__{ck}" + else: + json_key = ck + instructions += f' "{json_key}": {"true" if i == 0 else "false"},\n' + else: + for i, ck in enumerate(child_keys): + if use_namespaced_child_keys and i < len(llm_metrics): + json_key = _namespaced_child_key(parent_metric, llm_metrics[i]) + elif use_namespaced_child_keys: + json_key = f"{parent_key}__{ck}" + else: + json_key = ck + instructions += f' "{json_key}": {"true" if i % 2 == 0 else "false"},\n' + seq_sample = child_keys[: min(2, len(child_keys))] + seq_str = ", ".join(f'"{k}"' for k in seq_sample) + instructions += f' "{sequence_key}": [{seq_str}],\n' + if _discovery_enabled(parent_metric): + discovered_key = _discovered_key(parent_metric) + instructions += ( + f' "{discovered_key}": [\n' + ' {"key": "new_outcome_key", "name": "New Outcome", ' + '"description": "one short sentence", ' + '"rationale": "verbatim transcript line"}\n' + ' ],\n' + ) + if _wants_rationale(parent_metric): + parent_rationale_key = _rationale_key(parent_key) + instructions += ( + f' "{parent_rationale_key}": ' + f'"Brief justification referencing the transcript.",\n' + ) + return instructions + + +def _append_flat_response_example(instructions: str, llm_metrics: list) -> str: + """Append JSON example lines for standalone flat metrics.""" for metric in llm_metrics: metric_key = metric.name.lower().replace(" ", "_") - metric_desc = metric.description or f"Evaluate {metric.name}" m_type = get_metric_type_value(metric) custom_type = _get_custom_data_type(metric) - # Text metrics are unstructured by definition; ignore any stale - # ``custom_data_type`` (e.g. left over from when the metric used to be - # an enum) and ask the LLM for a free-form string. Rationale doesn't - # apply to text metrics (they are themselves free-form prose), so we - # short-circuit the rest of the loop body. if m_type == "text": - prompt += ( - f'\n- "{metric_key}" (free-form text, 1-3 concise sentences, ' - f'plain string): {metric_desc}' + instructions += ( + f' "{metric_key}": ' + f'"A brief 1-3 sentence summary describing what was observed.",\n' ) continue - # Emit the metric line. We deliberately do NOT ``continue`` after - # the enum / number_range branches — falling through lets the - # rationale companion block at the end of the loop run for those - # metric shapes too. line_added = False if custom_type == "enum": options = _get_enum_options(metric) if options: - opts_str = ", ".join(f'"{o}"' for o in options) - prompt += f'\n- "{metric_key}" (one of: {opts_str}): {metric_desc}' - line_added = True - - if not line_added and custom_type == "number_range": - rng = _get_number_range(metric) - if rng: - bounds = [] - if rng.get("min") is not None: - bounds.append(f"min={rng['min']}") - if rng.get("max") is not None: - bounds.append(f"max={rng['max']}") - if rng.get("step") is not None: - bounds.append(f"step={rng['step']}") - bound_str = ", ".join(bounds) if bounds else "numeric value" - prompt += f'\n- "{metric_key}" (numeric, {bound_str}): {metric_desc}' + instructions += f' "{metric_key}": "{options[0]}",\n' line_added = True if not line_added: if m_type == "rating": - prompt += f'\n- "{metric_key}" (rating 0.0-1.0): {metric_desc}' + instructions += f' "{metric_key}": 0.75,\n' elif m_type == "boolean": - prompt += f'\n- "{metric_key}" (true/false): {metric_desc}' + instructions += f' "{metric_key}": true,\n' elif m_type == "number": - prompt += f'\n- "{metric_key}" (numeric value): {metric_desc}' + instructions += f' "{metric_key}": 5,\n' - # When capture_rationale is on, ask for a sibling free-form rationale - # key in the SAME flat JSON object. Stays consistent with the "no - # nested objects" rule enforced below. if _wants_rationale(metric): - prompt += ( - f'\n- "{_rationale_key(metric_key)}" (free-form text, ' - f'1-2 concise sentences explaining why "{metric_key}" was chosen): ' - f"Justification for the value above. Reference specific lines or " - f"behaviors from the transcript when possible." + instructions += ( + f' "{_rationale_key(metric_key)}": ' + f'"Brief justification referencing the transcript.",\n' ) - - if discover_new_metrics: - prompt += _render_discovered_metrics_block( - running_discovered_metrics - ) - - prompt += _build_response_format_instructions( - llm_metrics, - discover_new_metrics=discover_new_metrics, - ) - return prompt + return instructions def _build_response_format_instructions( llm_metrics: list, parent_metric=None, discover_new_metrics: bool = False, + metric_groups: list[MetricPromptGroup] | None = None, ) -> str: """Build the response format section of the prompt. @@ -753,6 +955,73 @@ def _build_response_format_instructions( { """ + if metric_groups is not None: + use_namespaced = _use_namespaced_child_keys(metric_groups) + has_hierarchical = False + for group in metric_groups: + if group.parent_metric is not None: + has_hierarchical = True + instructions = _append_parent_response_example( + instructions, + group.parent_metric, + group.metrics, + use_namespaced_child_keys=use_namespaced, + ) + elif group.metrics: + instructions = _append_flat_response_example( + instructions, group.metrics + ) + + if discover_new_metrics: + instructions += ( + f' "{DISCOVERED_METRICS_KEY}": [\n' + ' {"key": "new_metric_key", "name": "New Metric", ' + '"description": "one short sentence", ' + '"suggested_type": "boolean", ' + '"rationale": "verbatim transcript line"}\n' + ' ],\n' + ) + + hierarchical_rules = "" + if has_hierarchical: + ns_rule = ( + " Use `{parent_slug}__{child_slug}` keys for child booleans " + "(never bare child slugs at the root when multiple categories " + "are present)." + if use_namespaced + else "" + ) + hierarchical_rules = ( + "\n6. Categorization groups: child keys are BOOLEAN (true/false)." + + ns_rule + + " Single-choice parents require EXACTLY ONE true child. " + "Multi-label parents set children independently. " + "Each parent's sequence array uses child slugs only (no parent " + "prefix) in temporal order." + ) + + metrics_discovery_rule = "" + if discover_new_metrics: + metrics_discovery_rule = ( + f'\n7. Top-level "{DISCOVERED_METRICS_KEY}" array: ' + "propose only BRAND-NEW metrics not already covered by " + "the metrics block above." + ) + + instructions += ( + "}\n\n" + "CRITICAL RULES:\n" + "1. Use the EXACT keys shown above - copy them character-for-character.\n" + "2. For numeric/boolean standalone metrics, values must be numbers or true/false.\n" + "3. For enum metrics, values must match listed options verbatim.\n" + "4. For text metrics, values must be plain JSON strings.\n" + "5. Do NOT wrap in \"metrics\" or any other object." + + hierarchical_rules + + metrics_discovery_rule + + "\nN. Do NOT add comments or explanations. Return ONLY the JSON object, nothing else." + ) + return instructions + if parent_metric is not None: parent_key = _parent_key(parent_metric) sequence_key = _sequence_key(parent_metric) @@ -850,43 +1119,7 @@ def _build_response_format_instructions( return instructions - for metric in llm_metrics: - metric_key = metric.name.lower().replace(" ", "_") - m_type = get_metric_type_value(metric) - custom_type = _get_custom_data_type(metric) - - # ``text`` short-circuits any stale custom_data_type for the same - # reason it does in ``build_evaluation_prompt``. - if m_type == "text": - instructions += ( - f' "{metric_key}": ' - f'"A brief 1-3 sentence summary describing what was observed.",\n' - ) - continue - - # Like build_evaluation_prompt: do NOT ``continue`` out of the - # enum branch — fall through so the rationale example line gets - # appended for enum metrics too when ``capture_rationale`` is on. - line_added = False - if custom_type == "enum": - options = _get_enum_options(metric) - if options: - instructions += f' "{metric_key}": "{options[0]}",\n' - line_added = True - - if not line_added: - if m_type == "rating": - instructions += f' "{metric_key}": 0.75,\n' - elif m_type == "boolean": - instructions += f' "{metric_key}": true,\n' - elif m_type == "number": - instructions += f' "{metric_key}": 5,\n' - - if _wants_rationale(metric): - instructions += ( - f' "{_rationale_key(metric_key)}": ' - f'"Brief justification referencing the transcript.",\n' - ) + instructions = _append_flat_response_example(instructions, llm_metrics) if discover_new_metrics: instructions += ( @@ -929,34 +1162,60 @@ def _build_system_message( llm_metrics: list, parent_metric=None, discover_new_metrics: bool = False, + metric_groups: list[MetricPromptGroup] | None = None, ) -> str: """Build the system message for LLM evaluation.""" exact_keys: list[str] = [] - if parent_metric is not None: + if metric_groups is not None: + use_namespaced = _use_namespaced_child_keys(metric_groups) + for group in metric_groups: + if group.parent_metric is not None: + parent_root_key = _parent_key(group.parent_metric) + exact_keys.append(parent_root_key) + exact_keys.append(_sequence_key(group.parent_metric)) + if _discovery_enabled(group.parent_metric): + exact_keys.append(_discovered_key(group.parent_metric)) + if _wants_rationale(group.parent_metric): + exact_keys.append(_rationale_key(parent_root_key)) + for metric in group.metrics: + key = ( + _namespaced_child_key(group.parent_metric, metric) + if use_namespaced + else _child_slug(metric) + ) + exact_keys.append(key) + for metric in group.metrics: + if group.parent_metric is not None: + continue + key = metric.name.lower().replace(" ", "_") + exact_keys.append(key) + if _wants_rationale(metric) and get_metric_type_value(metric) != "text": + exact_keys.append(_rationale_key(key)) + elif parent_metric is not None: parent_root_key = _parent_key(parent_metric) exact_keys.append(parent_root_key) exact_keys.append(_sequence_key(parent_metric)) if _discovery_enabled(parent_metric): exact_keys.append(_discovered_key(parent_metric)) - # Parent-level rationale key (one per group, never per child). if _wants_rationale(parent_metric): exact_keys.append(_rationale_key(parent_root_key)) - for metric in llm_metrics: - key = metric.name.lower().replace(" ", "_") - exact_keys.append(key) - # In hierarchical mode children no longer emit rationale keys; - # the parent owns the single rationale string for the group. - if ( - parent_metric is None - and _wants_rationale(metric) - and get_metric_type_value(metric) != "text" - ): - exact_keys.append(_rationale_key(key)) + for metric in llm_metrics: + key = metric.name.lower().replace(" ", "_") + exact_keys.append(key) + else: + for metric in llm_metrics: + key = metric.name.lower().replace(" ", "_") + exact_keys.append(key) + if _wants_rationale(metric) and get_metric_type_value(metric) != "text": + exact_keys.append(_rationale_key(key)) if discover_new_metrics: exact_keys.append(DISCOVERED_METRICS_KEY) + metrics_for_enums = ( + flatten_metric_groups(metric_groups) if metric_groups else llm_metrics + ) enum_constraints: list[str] = [] - for metric in llm_metrics: + for metric in metrics_for_enums: if _get_custom_data_type(metric) == "enum": options = _get_enum_options(metric) if options: @@ -974,7 +1233,7 @@ def _build_system_message( text_keys = [ metric.name.lower().replace(" ", "_") - for metric in llm_metrics + for metric in metrics_for_enums if get_metric_type_value(metric) == "text" ] text_block = "" @@ -986,10 +1245,24 @@ def _build_system_message( + "\n".join(f' - "{k}"' for k in text_keys) ) - if parent_metric is not None: - # Hierarchical mode: only the parent emits a rationale key (when - # capture_rationale is on); children never do. + if metric_groups is not None: rationale_keys: list[str] = [] + for group in metric_groups: + if group.parent_metric is not None and _wants_rationale( + group.parent_metric + ): + rationale_keys.append( + _rationale_key(_parent_key(group.parent_metric)) + ) + elif group.parent_metric is None: + rationale_keys.extend( + _rationale_key(metric.name.lower().replace(" ", "_")) + for metric in group.metrics + if _wants_rationale(metric) + and get_metric_type_value(metric) != "text" + ) + elif parent_metric is not None: + rationale_keys = [] if _wants_rationale(parent_metric): rationale_keys.append(_rationale_key(_parent_key(parent_metric))) else: @@ -1095,6 +1368,7 @@ def evaluate_with_llm( comparison_pair: tuple[str, str] | None = None, discover_new_metrics: bool = False, running_discovered_metrics: list | None = None, + metric_groups: list[MetricPromptGroup] | None = None, ) -> tuple[dict[str, dict[str, Any]], float | None]: """ Evaluate metrics using LLM. @@ -1127,9 +1401,13 @@ def evaluate_with_llm( """ from app.services.ai.llm_service import llm_service + metrics_for_call = ( + flatten_metric_groups(metric_groups) if metric_groups is not None else llm_metrics + ) + evaluation_prompt = build_evaluation_prompt( transcription=transcription, - llm_metrics=llm_metrics, + llm_metrics=metrics_for_call, evaluator=evaluator, agent=agent, persona=persona, @@ -1141,6 +1419,7 @@ def evaluate_with_llm( comparison_pair=comparison_pair, discover_new_metrics=discover_new_metrics, running_discovered_metrics=running_discovered_metrics, + metric_groups=metric_groups, ) evaluator_llm_provider = getattr(evaluator, "llm_provider", None) if evaluator else None @@ -1169,9 +1448,10 @@ def evaluate_with_llm( { "role": "system", "content": _build_system_message( - llm_metrics, + metrics_for_call, parent_metric=parent_metric, discover_new_metrics=discover_new_metrics, + metric_groups=metric_groups, ), }, {"role": "user", "content": evaluation_prompt}, @@ -1184,11 +1464,28 @@ def evaluate_with_llm( # 300 tokens per metric (covers value + rationale + comma/quotes), # clamped to a reasonable ceiling. ``llm_service`` will additionally # disable thinking and enforce a floor for Gemini 2.5. - metric_count = max(1, len(llm_metrics)) - rationale_count = sum(1 for m in llm_metrics if _wants_rationale(m)) + metric_count = max(1, len(metrics_for_call)) + rationale_count = sum(1 for m in metrics_for_call if _wants_rationale(m)) + hierarchical_extra = 0 + if metric_groups is not None: + for group in metric_groups: + if group.parent_metric is None: + continue + parent = group.parent_metric + hierarchical_extra += 2 # parent_key + sequence + hierarchical_extra += len(group.metrics) # namespaced child booleans + if _wants_rationale(parent): + hierarchical_extra += 1 + if _discovery_enabled(parent): + hierarchical_extra += 1 dynamic_max_tokens = min( 8192, - max(2000, 300 * metric_count + 200 * rationale_count), + max( + 2000, + 300 * metric_count + + 200 * rationale_count + + 150 * hierarchical_extra, + ), ) evaluation_start_time = time.time() @@ -1225,7 +1522,10 @@ def evaluate_with_llm( evaluation_data = evaluation_data["metrics"] metric_scores = _map_evaluation_to_metrics( - evaluation_data, llm_metrics, parent_metric=parent_metric + evaluation_data, + metrics_for_call, + parent_metric=parent_metric, + metric_groups=metric_groups, ) # Top-level metric discovery is independent of the per-row metric @@ -1236,7 +1536,7 @@ def evaluate_with_llm( # paths share the same code. if discover_new_metrics: discovered_metrics = _parse_discovered_metrics( - evaluation_data, llm_metrics + evaluation_data, metrics_for_call ) if discovered_metrics: metric_scores[DISCOVERED_METRICS_KEY] = discovered_metrics @@ -1248,6 +1548,7 @@ def _map_evaluation_to_metrics( evaluation_data: dict, llm_metrics: list, parent_metric=None, + metric_groups: list[MetricPromptGroup] | None = None, ) -> dict[str, dict[str, Any]]: """Map LLM evaluation response to metric scores. @@ -1266,11 +1567,38 @@ def _map_evaluation_to_metrics( metric_scores: dict[str, dict[str, Any]] = {} response_keys = list(evaluation_data.keys()) + if metric_groups is not None: + use_namespaced = _use_namespaced_child_keys(metric_groups) + for group in metric_groups: + if group.parent_metric is not None: + _map_hierarchical_group( + evaluation_data, + group.metrics, + group.parent_metric, + metric_scores, + use_namespaced_child_keys=use_namespaced, + ) + else: + metric_scores.update( + _map_flat_metrics(evaluation_data, group.metrics, response_keys) + ) + return metric_scores + if parent_metric is not None: return _map_hierarchical_group( evaluation_data, llm_metrics, parent_metric, metric_scores ) + return _map_flat_metrics(evaluation_data, llm_metrics, response_keys) + + +def _map_flat_metrics( + evaluation_data: dict, + llm_metrics: list, + response_keys: list[str], +) -> dict[str, dict[str, Any]]: + metric_scores: dict[str, dict[str, Any]] = {} + for metric in llm_metrics: metric_key = metric.name.lower().replace(" ", "_") m_type = get_metric_type_value(metric) @@ -1356,6 +1684,8 @@ def _map_hierarchical_group( children: list, parent_metric, metric_scores: dict[str, dict[str, Any]], + *, + use_namespaced_child_keys: bool = False, ) -> dict[str, dict[str, Any]]: """Parse the LLM response for a parent + children group. @@ -1370,17 +1700,19 @@ def _map_hierarchical_group( child_key_to_metric: dict[str, Any] = {} for child in children: - child_key_to_metric[child.name.lower().replace(" ", "_")] = child + child_key_to_metric[_child_slug(child)] = child # ----- Per-child booleans ----- child_results: dict[str, dict[str, Any]] = {} for child in children: - child_key = child.name.lower().replace(" ", "_") - raw_value = evaluation_data.get(child_key) - if raw_value is None: - matched = find_matching_key(child.name, response_keys) - if matched: - raw_value = evaluation_data.get(matched) + child_key = _child_slug(child) + raw_value = _read_child_boolean_raw( + evaluation_data, + response_keys, + parent_metric, + child, + use_namespaced=use_namespaced_child_keys, + ) score = extract_score(raw_value) score = normalize_score(score, "boolean") if not isinstance(score, bool): diff --git a/app/workers/tasks/process_evaluator_result.py b/app/workers/tasks/process_evaluator_result.py index 7a5c5c49..56a1ac80 100644 --- a/app/workers/tasks/process_evaluator_result.py +++ b/app/workers/tasks/process_evaluator_result.py @@ -249,50 +249,35 @@ def _evaluate_llm_metrics_grouped( agent, persona, scenario, + comparison_pair: tuple[str, str] | None = None, + all_columns_block: str | None = None, + discover_new_metrics: bool = False, + running_discovered_metrics: list | None = None, ) -> tuple[dict, float | None]: - """Evaluate LLM metrics, grouping categorization children by parent.""" - from app.workers.tasks.evaluate_call_import_row_core import build_parent_groups - - parents_by_id, children_by_parent, standalone_metrics = build_parent_groups( - db, llm_metrics + """Evaluate all LLM metrics in a single call (grouped by parent in prompt).""" + from app.workers.tasks.evaluate_call_import_row_core import ( + build_metric_prompt_groups, ) - metric_scores: dict = {} - evaluation_time: float | None = None - - def _run_bucket(bucket, parent_metric=None): - nonlocal evaluation_time - scores, eval_time = evaluate_with_llm( - transcription=transcription, - llm_metrics=bucket, - ai_providers=ai_providers, - organization_id=organization_id, - result_id=result_id, - db=db, - evaluator=evaluator, - agent=agent, - persona=persona, - scenario=scenario, - parent_metric=parent_metric, - ) - metric_scores.update(scores) - if eval_time is not None: - evaluation_time = eval_time - - if standalone_metrics: - _run_bucket(standalone_metrics, parent_metric=None) - - for parent_id, children in children_by_parent.items(): - parent_metric = parents_by_id.get(parent_id) - if not parent_metric: - logger.warning( - f"[EvaluatorResult {result_id}] Parent metric {parent_id} not found; " - "evaluating children as flat metrics" - ) - _run_bucket(children, parent_metric=None) - continue - _run_bucket(children, parent_metric=parent_metric) - return metric_scores, evaluation_time + metric_groups = build_metric_prompt_groups(db, llm_metrics) + scores, eval_time = evaluate_with_llm( + transcription=transcription, + llm_metrics=llm_metrics, + ai_providers=ai_providers, + organization_id=organization_id, + result_id=result_id, + db=db, + evaluator=evaluator, + agent=agent, + persona=persona, + scenario=scenario, + comparison_pair=comparison_pair, + all_columns_block=all_columns_block, + discover_new_metrics=discover_new_metrics, + running_discovered_metrics=running_discovered_metrics, + metric_groups=metric_groups, + ) + return scores, eval_time _PERMANENT_VAPI_ENDED_REASONS = frozenset( @@ -408,18 +393,9 @@ def _extract_audio_url(call_data: dict, platform: str) -> str | None: if platform == "retell": return call_data.get("recording_url") if platform == "vapi": - return ( - call_data.get("recordingUrl") - or call_data.get("stereoRecordingUrl") - or artifact.get("recordingUrl") - or artifact.get("stereoRecordingUrl") - or mono_recording.get("combinedUrl") - or recording_urls.get("combined_url") - or recording_urls.get("stereo_url") - or call_data.get("recordingUrl") - or provider_payload.get("recordingUrl") - or provider_payload.get("stereoRecordingUrl") - ) + from app.services.voice_providers.vapi_recording import extract_vapi_recording_url + + return extract_vapi_recording_url(call_data) if platform == "smallest": return ( call_data.get("recording_url") @@ -497,7 +473,10 @@ def _recover_missing_audio_for_result(result, db, refresh_call_data: bool = True if platform == "elevenlabs" and decrypted_key: headers = {"xi-api-key": decrypted_key} elif platform == "vapi" and decrypted_key: - headers = {"Authorization": f"Bearer {decrypted_key}"} + from app.services.voice_providers.vapi_recording import is_presigned_storage_url + + if not is_presigned_storage_url(audio_url): + headers = {"Authorization": f"Bearer {decrypted_key}"} try: response = _http.get(audio_url, headers=headers, timeout=120) except Exception as download_err: diff --git a/config.docker.yml b/config.docker.yml index 490f43d3..3d169129 100644 --- a/config.docker.yml +++ b/config.docker.yml @@ -7,6 +7,7 @@ app: version: "0.1.0" debug: true secret_key: "your-secret-key-here-change-in-production" + frontend_base_url: "http://localhost:8000" # Server Settings server: diff --git a/config.yml.example b/config.yml.example index 91b43e2e..b361321b 100644 --- a/config.yml.example +++ b/config.yml.example @@ -6,6 +6,10 @@ app: version: "0.1.0" debug: true secret_key: "your-secret-key-here-change-in-production" + # Public URL where users open the app (used for organization invite links). + # Local: http://localhost:8000 when the API serves the built SPA on port 8000. + # Production: https://your-company.efficientai.com (or your custom domain). + frontend_base_url: "http://localhost:8000" # Server Settings server: diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 72664080..b62b6e72 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -5,6 +5,7 @@ import Layout from './components/Layout' // Auth import Login from './pages/auth/Login' +import InviteAccept from './pages/auth/InviteAccept' import LoginCallback from './pages/auth/LoginCallback' import SelectOrganization from './pages/auth/SelectOrganization' import PlatformLogin from './pages/platform/PlatformLogin' @@ -140,6 +141,7 @@ function App() { } /> + } /> } /> } /> } /> diff --git a/frontend/src/components/WorkspaceSwitcher.tsx b/frontend/src/components/WorkspaceSwitcher.tsx index 7ca29b41..d905af68 100644 --- a/frontend/src/components/WorkspaceSwitcher.tsx +++ b/frontend/src/components/WorkspaceSwitcher.tsx @@ -1,8 +1,9 @@ import { useCallback, useEffect, useMemo, useState } from 'react' import { useQuery, useQueryClient } from '@tanstack/react-query' import { useLocation, useNavigate } from 'react-router-dom' -import { Check, ChevronDown, FolderKanban, Plus } from 'lucide-react' +import { Check, ChevronDown, Copy, FolderKanban, Plus } from 'lucide-react' import { apiClient } from '../lib/api' +import { copyTextToClipboard } from '../lib/clipboard' import { resolveWorkspaceSwitchPath } from '../lib/workspaceNavigation' import type { Workspace } from '../types/api' import { useCanWrite } from '../hooks/useRole' @@ -19,6 +20,7 @@ export default function WorkspaceSwitcher() { const setActiveCapabilities = useWorkspaceStore((s) => s.setActiveCapabilities) const [open, setOpen] = useState(false) const [showCreateModal, setShowCreateModal] = useState(false) + const [copiedId, setCopiedId] = useState(null) const onWorkspaceChanged = useCallback( async (workspace: Workspace) => { @@ -102,6 +104,15 @@ export default function WorkspaceSwitcher() { setShowCreateModal(true) } + const handleCopyId = (event: React.MouseEvent, workspaceId: string) => { + event.preventDefault() + event.stopPropagation() + copyTextToClipboard(workspaceId, () => { + setCopiedId(workspaceId) + setTimeout(() => setCopiedId(null), 2000) + }) + } + return ( <>
@@ -126,7 +137,7 @@ export default function WorkspaceSwitcher() { className="fixed inset-0 z-10" onClick={() => setOpen(false)} /> -
+
Workspaces {canWrite && ( @@ -158,43 +169,95 @@ export default function WorkspaceSwitcher() { {workspaces.map((ws) => { const isCurrent = ws.id === activeId const isInactive = !ws.is_active + const isCopied = copiedId === ws.id return ( - +
+ {!isInactive && ( + + )} + {isCurrent && !isInactive && ( + + )}
- {isCurrent && !isInactive && ( - - )} - +
) })}
+ + {activeWorkspace?.is_active && ( +
+
+ + Active workspace ID + + +
+ + {activeWorkspace.id} + +

+ Use as X-Workspace-Id header +

+
+ )}
)} diff --git a/frontend/src/components/call-recordings/VapiCallDetails.tsx b/frontend/src/components/call-recordings/VapiCallDetails.tsx index 21f90480..09ae62a8 100644 --- a/frontend/src/components/call-recordings/VapiCallDetails.tsx +++ b/frontend/src/components/call-recordings/VapiCallDetails.tsx @@ -154,11 +154,13 @@ export default function VapiCallDetails({ callData, hideTranscript = false }: Va const recordingLinks = { combined: + (raw.artifact as any)?.presignedMonoUrl || callData.recording_urls?.combined_url || raw.recordingUrl || artifact.recordingUrl || artifactMono.combinedUrl, stereo: + (raw.artifact as any)?.presignedStereoUrl || callData.recording_urls?.stereo_url || raw.stereoRecordingUrl || artifact.stereoRecordingUrl || diff --git a/frontend/src/components/iam/WorkspaceMembersSection.tsx b/frontend/src/components/iam/WorkspaceMembersSection.tsx index f08fbb86..2a2123ea 100644 --- a/frontend/src/components/iam/WorkspaceMembersSection.tsx +++ b/frontend/src/components/iam/WorkspaceMembersSection.tsx @@ -316,122 +316,131 @@ export default function WorkspaceMembersSection() {
{selectedWorkspace && ( -
-

+
+

Workspace Settings

-
- - {editingName && canEditWorkspaceName ? ( -
- setNameDraft(e.target.value)} - maxLength={255} - className="flex-1 px-3 py-2 border border-gray-300 rounded-lg text-sm focus:outline-none focus:ring-2 focus:ring-primary-500" - /> +
+
+

+ Workspace Name +

+ {editingName && canEditWorkspaceName ? ( +
+ setNameDraft(e.target.value)} + maxLength={255} + className="flex-1 min-w-0 px-3 py-2 border border-gray-300 rounded-lg text-sm focus:outline-none focus:ring-2 focus:ring-primary-500" + /> +
+ + +
+
+ ) : ( +
+ + {selectedWorkspace.name} + + {canEditWorkspaceName && ( + + )} +
+ )} +

+ Slug: {selectedWorkspace.slug} +

+
+ +
+

+ Status +

+ + {selectedWorkspace.is_active ? 'Active' : 'Inactive'} + +
+
+ + {isAdmin && ( +
+

+ {selectedWorkspace.is_active + ? 'Deactivate to lock this workspace for non–org-admin users.' + : 'Reactivate to restore access for workspace members.'} +

+ {selectedWorkspace.is_active ? ( + ) : ( -
- ) : ( -
- - {selectedWorkspace.name} - - {canEditWorkspaceName && ( - - )} -
- )} -

- Slug stays fixed for links:{' '} - {selectedWorkspace.slug} -

-
- - {isAdmin && ( -
-
-
-

- Status -

- - {selectedWorkspace.is_active ? 'Active' : 'Inactive'} - - {!selectedWorkspace.is_active && ( -

- Inactive workspaces are fully locked for all non-org-admin - users. Reactivate to restore access. -

- )} -
- {selectedWorkspace.is_active ? ( - - ) : ( - - )} -
+ )}
)} + + {!selectedWorkspace.is_active && ( +

+ Inactive workspaces are fully locked for all non-org-admin users. +

+ )}
)} diff --git a/frontend/src/components/shared/StatusBadge.tsx b/frontend/src/components/shared/StatusBadge.tsx index 2ff59db3..59c52e06 100644 --- a/frontend/src/components/shared/StatusBadge.tsx +++ b/frontend/src/components/shared/StatusBadge.tsx @@ -18,6 +18,8 @@ const STATUS_STYLES: Record = { queued: 'bg-gray-100 text-gray-700', generating: 'bg-yellow-100 text-yellow-800', evaluating: 'bg-blue-100 text-blue-800', + running: 'bg-blue-100 text-blue-800', + cancelled: 'bg-gray-100 text-gray-600', processing: 'bg-blue-100 text-blue-800', completed: 'bg-green-100 text-green-800', partial: 'bg-amber-100 text-amber-800', @@ -41,7 +43,7 @@ export default function StatusBadge({ status, size = 'md' }: StatusBadgeProps) { const textClass = size === 'sm' ? 'text-[10px]' : 'text-xs' const renderIcon = () => { - if (['generating', 'evaluating', 'processing', 'call_initiating', 'call_connecting', 'call_in_progress', 'transcribing', 'deleting'].includes(normalizedStatus)) { + if (['generating', 'evaluating', 'processing', 'running', 'call_initiating', 'call_connecting', 'call_in_progress', 'transcribing', 'deleting'].includes(normalizedStatus)) { return } if (normalizedStatus === 'completed' || normalizedStatus === 'call_ended') { diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index aaca0b6c..c432705b 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -23,6 +23,7 @@ import type { OrganizationMember, Invitation, InvitationCreate, + InvitationPreview, Profile, UserUpdate, UserPreferences, @@ -801,11 +802,22 @@ class ApiClient { first_name?: string last_name?: string reference_code?: string + invite_token?: string }): Promise { const response = await this.client.post('/api/v1/auth/signup', data) return response.data } + async previewInvitation(token: string): Promise { + const response = await axios.get(`${API_BASE_URL}/api/v1/auth/invitations/preview/${encodeURIComponent(token)}`) + return response.data + } + + async acceptInvitationByToken(token: string): Promise { + const response = await this.client.post('/api/v1/auth/invitations/accept-by-token', { token }) + return response.data + } + private platformHeaders() { const token = localStorage.getItem('platformAccessToken') return token ? { Authorization: `Bearer ${token}` } : {} @@ -918,12 +930,14 @@ class ApiClient { async loginWithPassword( email: string, password: string, - organizationId?: string + organizationId?: string, + inviteToken?: string, ): Promise { const response = await this.client.post('/api/v1/auth/login', { email, password, ...(organizationId ? { organization_id: organizationId } : {}), + ...(inviteToken ? { invite_token: inviteToken } : {}), }) return response.data } @@ -4471,6 +4485,14 @@ class ApiClient { return response.data } + async getEvaluatorResultAudioUrl(resultId: string): Promise { + const response = await this.client.get( + `/api/v1/evaluator-results/${resultId}/audio`, + { responseType: 'blob' }, + ) + return URL.createObjectURL(response.data) + } + async createEvaluatorResultManual(data: { evaluator_id: string audio_s3_key: string diff --git a/frontend/src/lib/inviteToken.ts b/frontend/src/lib/inviteToken.ts new file mode 100644 index 00000000..2cb2d254 --- /dev/null +++ b/frontend/src/lib/inviteToken.ts @@ -0,0 +1,21 @@ +export const PENDING_INVITE_TOKEN_KEY = 'pendingInviteToken' + +export function storePendingInviteToken(token: string): void { + sessionStorage.setItem(PENDING_INVITE_TOKEN_KEY, token) +} + +export function getPendingInviteToken(): string | null { + return sessionStorage.getItem(PENDING_INVITE_TOKEN_KEY) +} + +export function consumePendingInviteToken(): string | null { + const token = sessionStorage.getItem(PENDING_INVITE_TOKEN_KEY) + if (token) { + sessionStorage.removeItem(PENDING_INVITE_TOKEN_KEY) + } + return token +} + +export function clearPendingInviteToken(): void { + sessionStorage.removeItem(PENDING_INVITE_TOKEN_KEY) +} diff --git a/frontend/src/lib/inviteUrl.ts b/frontend/src/lib/inviteUrl.ts new file mode 100644 index 00000000..c0e1692f --- /dev/null +++ b/frontend/src/lib/inviteUrl.ts @@ -0,0 +1,19 @@ +import type { Invitation } from '../types/api' + +type InviteLinkFields = Pick + +/** Build a shareable invite URL using the current browser origin (matches blind-test sharing). */ +export function buildInviteShareUrl(invitation: InviteLinkFields): string | null { + if (invitation.invite_path) { + return `${window.location.origin}${invitation.invite_path}` + } + if (!invitation.invite_url) { + return null + } + try { + const url = new URL(invitation.invite_url) + return `${window.location.origin}${url.pathname}` + } catch { + return invitation.invite_url + } +} diff --git a/frontend/src/lib/recordingUrls.ts b/frontend/src/lib/recordingUrls.ts new file mode 100644 index 00000000..030e7cbb --- /dev/null +++ b/frontend/src/lib/recordingUrls.ts @@ -0,0 +1,59 @@ +/** Resolve a browser-playable recording URL from provider call_data. */ + +export function getProviderRecordingUrl( + callData: Record | null | undefined, + platform?: string | null, +): string | null { + if (!callData) return null + + const plat = (platform || '').toLowerCase() + + if (plat === 'vapi') { + const artifact = (callData.artifact || {}) as Record + const recording = (artifact.recording || {}) as Record + const mono = (recording.mono || {}) as Record + const recordingUrls = (callData.recording_urls || {}) as Record + + return ( + pickString(artifact.presignedMonoUrl) || + pickString(artifact.presignedStereoUrl) || + pickString(callData.presignedMonoUrl) || + pickString(callData.presignedStereoUrl) || + pickString(callData.recordingUrl) || + pickString(callData.stereoRecordingUrl) || + pickString(artifact.recordingUrl) || + pickString(artifact.stereoRecordingUrl) || + pickString(mono.combinedUrl) || + pickString(recordingUrls.combined_url) || + pickString(recordingUrls.stereo_url) || + null + ) + } + + if (plat === 'elevenlabs') { + const recordingUrls = (callData.recording_urls || {}) as Record + return pickString(callData.recording_url) || pickString(recordingUrls.conversation_audio) + } + + return ( + pickString(callData.recording_url) || + pickString(callData.recordingUrl) || + null + ) +} + +export function hasEvaluatorResultRecording( + result: { + audio_s3_key?: string | null + provider_platform?: string | null + call_data?: Record | null + } | null | undefined, +): boolean { + if (!result) return false + const audioS3Key = result.audio_s3_key || pickString(result.call_data?.recording_s3_key) + return Boolean(audioS3Key || getProviderRecordingUrl(result.call_data, result.provider_platform)) +} + +function pickString(value: unknown): string | null { + return typeof value === 'string' && value.trim() ? value.trim() : null +} diff --git a/frontend/src/pages/agents/components/AgentInfoView.tsx b/frontend/src/pages/agents/components/AgentInfoView.tsx index 9e4906b2..272953f1 100644 --- a/frontend/src/pages/agents/components/AgentInfoView.tsx +++ b/frontend/src/pages/agents/components/AgentInfoView.tsx @@ -196,7 +196,7 @@ export default function AgentInfoView({ > + +

{message}

+
+ ) +} + +function InvitePageShell({ children }: { children: React.ReactNode }) { + return ( +
+
+
+
+
+
{children}
+
+ ) +} + +export default function InviteAccept() { + const { token } = useParams<{ token: string }>() + const navigate = useNavigate() + const { user, accessToken, setSession, logout } = useAuthStore() + + const [preview, setPreview] = useState(null) + const [loadingPreview, setLoadingPreview] = useState(true) + const [previewError, setPreviewError] = useState('') + const [authConfig, setAuthConfig] = useState(null) + const [mode, setMode] = useState('signup') + const [email, setEmail] = useState('') + const [password, setPassword] = useState('') + const [firstName, setFirstName] = useState('') + const [lastName, setLastName] = useState('') + const [error, setError] = useState('') + const [isLoading, setIsLoading] = useState(false) + const [showPassword, setShowPassword] = useState(false) + const [accepting, setAccepting] = useState(false) + const [autoAcceptFailed, setAutoAcceptFailed] = useState(false) + + useEffect(() => { + if (!token) { + setPreviewError('Invalid invitation link') + setLoadingPreview(false) + return + } + + storePendingInviteToken(token) + + let active = true + Promise.all([apiClient.previewInvitation(token), apiClient.getAuthConfig()]) + .then(([previewData, cfg]) => { + if (!active) return + setPreview(previewData) + setEmail(previewData.email) + setAuthConfig(cfg) + setMode(previewData.has_password ? 'password' : 'signup') + if (previewData.status !== 'pending') { + setPreviewError( + previewData.status === 'expired' + ? 'This invitation has expired. Ask your administrator to send a new one.' + : 'This invitation is no longer valid.', + ) + } + }) + .catch((err: any) => { + if (!active) return + setPreviewError(err?.response?.data?.detail || 'Invitation not found') + }) + .finally(() => active && setLoadingPreview(false)) + + return () => { + active = false + } + }, [token]) + + useEffect(() => { + if (!token || !preview || preview.status !== 'pending' || !accessToken || !user) { + return + } + + if (user.email.toLowerCase() !== preview.email.toLowerCase()) { + setPreviewError( + `You're signed in as ${user.email}, but this invitation was sent to ${preview.email}. Sign out to continue with the invited account.`, + ) + return + } + + let active = true + setAccepting(true) + apiClient + .acceptInvitationByToken(token) + .then((res) => { + if (!active) return + setSession(res.access_token, res.user, res.refresh_token) + navigate('/', { replace: true }) + }) + .catch((err: any) => { + if (!active) return + setAutoAcceptFailed(true) + logout() + setError(err?.response?.data?.detail || 'Could not accept invitation') + }) + .finally(() => active && setAccepting(false)) + + return () => { + active = false + } + }, [token, preview, accessToken, user, setSession, navigate]) + + const localPwd = authConfig?.providers.find((p) => p.name === 'local_password' && p.enabled) + const oidc = authConfig?.providers.find((p) => p.name === 'external_oidc' && p.enabled) + + const tabs = useMemo(() => { + const items: Array<{ key: Mode; label: string }> = [] + const canSignUp = localPwd?.supports_signup && !preview?.has_password + const canSignIn = !!localPwd && preview?.has_password + + if (canSignUp) { + items.push({ key: 'signup', label: 'Create account' }) + } + if (canSignIn) { + items.push({ key: 'password', label: 'Sign in' }) + } + if (oidc) { + items.push({ key: 'sso', label: 'SSO' }) + } + return items + }, [preview?.has_password, localPwd, oidc]) + + useEffect(() => { + if (tabs.length === 0) return + if (!tabs.some((tab) => tab.key === mode)) { + setMode(tabs[0].key) + } + }, [tabs, mode]) + + const handleSignup = async (e: React.FormEvent) => { + e.preventDefault() + if (!token) return + setError('') + const policy = validatePasswordPolicy(password) + if (!policy.valid) { + setError(policy.message || 'Invalid password') + return + } + setIsLoading(true) + try { + const res = await apiClient.signup({ + email, + password, + first_name: firstName || undefined, + last_name: lastName || undefined, + invite_token: token, + }) + setSession(res.access_token, res.user, res.refresh_token) + navigate('/', { replace: true }) + } catch (err: any) { + setError(err?.response?.data?.detail || 'Sign up failed') + } finally { + setIsLoading(false) + } + } + + const handlePasswordLogin = async (e: React.FormEvent) => { + e.preventDefault() + if (!token) return + setError('') + setIsLoading(true) + try { + const res = await apiClient.loginWithPassword(email, password, undefined, token) + if (isLoginOrgSelectionResponse(res)) { + setError('Multiple organizations found. Accept the invitation from your profile after signing in.') + return + } + const accepted = await apiClient.acceptInvitationByToken(token) + setSession(accepted.access_token, accepted.user, accepted.refresh_token) + navigate('/', { replace: true }) + } catch (err: any) { + setError(err?.response?.data?.detail || 'Sign in failed') + } finally { + setIsLoading(false) + } + } + + const handleSsoRedirect = async (provider: AuthProviderConfig) => { + if (!token || !provider.oidc_client_id) { + setError('SSO is not fully configured on this server.') + return + } + storePendingInviteToken(token) + try { + const redirect = `${window.location.origin}/login/callback` + const authorizeUrl = await buildAuthorizeUrl(provider, redirect) + window.location.href = authorizeUrl + } catch (err: any) { + setError(err?.message || 'Could not start SSO sign-in') + } + } + + if (loadingPreview || accepting) { + return ( + +
+ +

{accepting ? 'Accepting invitation…' : 'Loading invitation…'}

+
+ + ) + } + + const orgName = preview?.organization_name || 'your organization' + const roleLabel = preview?.role || 'member' + const showAuthForms = + !previewError && + preview?.status === 'pending' && + (!accessToken || autoAcceptFailed) + + return ( + +
+
+ +
+

You're invited

+

+ Join {orgName} as{' '} + {roleLabel} +

+
+ + + + {preview && !previewError && preview.has_password && showAuthForms && ( +
+ An account already exists for {preview.email}. + Sign in below to join {orgName} — you don't need to create a new account. +
+ )} + + {preview && !previewError && ( +
+ +

+ Invitation for {preview.email} + {preview.expires_at && ( + <> · expires {new Date(preview.expires_at).toLocaleDateString()} + )} +

+
+ )} + + {previewError && } + + {previewError && accessToken && user && ( +
+ +
+ )} + + {error && !showAuthForms && ( +
+ +
+ )} + + {showAuthForms && ( + <> + {tabs.length > 1 && ( + { + setMode(k as Mode) + setError('') + setShowPassword(false) + }} + variant="solid" + radius="full" + fullWidth + classNames={TAB_CLASS_NAMES} + > + {tabs.map((tab) => ( + + ))} + + )} + + {mode === 'signup' && localPwd?.supports_signup && !preview?.has_password && ( +
+
+ setFirstName(e.target.value)} + className={INPUT_CLASS} + /> + setLastName(e.target.value)} + className={INPUT_CLASS} + /> +
+ +
+ setPassword(e.target.value)} + required + minLength={8} + maxLength={32} + className={`${INPUT_CLASS} pr-12`} + /> + +
+ + {error && } +

+ You'll join {orgName} directly — no separate organization is created. +

+ + )} + + {mode === 'password' && localPwd && preview?.has_password && ( +
+ +
+ setPassword(e.target.value)} + required + className={`${INPUT_CLASS} pr-12`} + /> + +
+ + {error && } + + )} + + {mode === 'sso' && oidc && ( +
+ + {error && } +

+ After SSO, you'll be added to {orgName} automatically. +

+
+ )} + + )} +
+ + + ) +} diff --git a/frontend/src/pages/auth/Login.tsx b/frontend/src/pages/auth/Login.tsx index f395437f..35837549 100644 --- a/frontend/src/pages/auth/Login.tsx +++ b/frontend/src/pages/auth/Login.tsx @@ -1,11 +1,16 @@ import { useEffect, useState } from 'react' -import { useNavigate } from 'react-router-dom' +import { useNavigate, useSearchParams } from 'react-router-dom' import { useAuthStore } from '../../store/authStore' import { apiClient, isLoginOrgSelectionResponse } from '../../lib/api' import type { AuthConfigResponse, AuthProviderConfig, LoginOrgOption } from '../../lib/api' import { buildAuthorizeUrl } from '../../lib/oidc' import { PASSWORD_POLICY_HINT, validatePasswordPolicy } from '../../lib/passwordPolicy' import { consumeAuthRedirectMessage } from '../../lib/authSession' +import { + consumePendingInviteToken, + getPendingInviteToken, + storePendingInviteToken, +} from '../../lib/inviteToken' import { AlertCircle, Building2, Eye, EyeOff, Loader2 } from 'lucide-react' import Logo from '../../components/Logo' import { Card, CardBody, Button, Divider, Tabs, Tab } from '@heroui/react' @@ -39,6 +44,7 @@ function LoginFormError({ message }: { message: string }) { export default function Login() { const navigate = useNavigate() + const [searchParams] = useSearchParams() const { setSession } = useAuthStore() const [authConfig, setAuthConfig] = useState(null) @@ -58,6 +64,35 @@ export default function Login() { const [orgOptions, setOrgOptions] = useState([]) const [selectingOrgId, setSelectingOrgId] = useState(null) const [showPassword, setShowPassword] = useState(false) + const [inviteToken, setInviteToken] = useState(null) + const [invitePreview, setInvitePreview] = useState<{ + organization_name?: string | null + email: string + has_password?: boolean + } | null>(null) + + useEffect(() => { + const fromQuery = searchParams.get('invite') + const fromStorage = getPendingInviteToken() + const token = fromQuery || fromStorage + if (!token) return + setInviteToken(token) + storePendingInviteToken(token) + apiClient + .previewInvitation(token) + .then((preview) => { + setInvitePreview(preview) + setEmail(preview.email) + if (preview.has_password) { + setMode('password') + } else { + setMode('signup') + } + }) + .catch(() => { + setInviteToken(null) + }) + }, [searchParams]) useEffect(() => { const redirectMessage = consumeAuthRedirectMessage() @@ -75,8 +110,8 @@ export default function Login() { setAuthConfig(cfg) // Pick a sensible default tab based on what's enabled server-side. const byName = (n: string) => cfg.providers.find((p) => p.name === n && p.enabled) - if (byName('local_password')) setMode('password') - else if (byName('external_oidc')) setMode('sso') + if (byName('local_password') && !inviteToken) setMode('password') + else if (byName('external_oidc') && !inviteToken) setMode('sso') }) .catch(() => { if (!active) return @@ -96,19 +131,43 @@ export default function Login() { const localPwd = providerByName('local_password') const oidc = providerByName('external_oidc') + const completeInviteIfNeeded = async (accessToken: string, authUser: Parameters[1], refreshToken?: string | null) => { + const token = inviteToken || getPendingInviteToken() + if (!token) { + setSession(accessToken, authUser, refreshToken) + navigate('/') + return + } + try { + apiClient.setAccessToken(accessToken) + const accepted = await apiClient.acceptInvitationByToken(token) + consumePendingInviteToken() + setSession(accepted.access_token, accepted.user, accepted.refresh_token) + navigate('/') + } catch (err: any) { + setSession(accessToken, authUser, refreshToken) + setError(err?.response?.data?.detail || 'Signed in, but could not accept the invitation') + navigate('/') + } + } + const handlePasswordLogin = async (e: React.FormEvent) => { e.preventDefault() setError('') setIsLoading(true) try { - const res = await apiClient.loginWithPassword(email, password) + const res = await apiClient.loginWithPassword( + email, + password, + undefined, + inviteToken || getPendingInviteToken() || undefined, + ) if (isLoginOrgSelectionResponse(res)) { setOrgOptions(res.organizations) setLoginStep('org-select') return } - setSession(res.access_token, res.user, res.refresh_token) - navigate('/') + await completeInviteIfNeeded(res.access_token, res.user, res.refresh_token) } catch (err: any) { setError(err?.response?.data?.detail || 'Invalid email or password') } finally { @@ -120,13 +179,17 @@ export default function Login() { setError('') setSelectingOrgId(organizationId) try { - const res = await apiClient.loginWithPassword(email, password, organizationId) + const res = await apiClient.loginWithPassword( + email, + password, + organizationId, + inviteToken || getPendingInviteToken() || undefined, + ) if (isLoginOrgSelectionResponse(res)) { setError('Organization selection failed — try again') return } - setSession(res.access_token, res.user, res.refresh_token) - navigate('/') + await completeInviteIfNeeded(res.access_token, res.user, res.refresh_token) } catch (err: any) { setError(err?.response?.data?.detail || 'Could not sign in to the selected organization') } finally { @@ -147,11 +210,13 @@ export default function Login() { const res = await apiClient.signup({ email, password, - organization_name: orgName || undefined, + organization_name: inviteToken ? undefined : orgName || undefined, first_name: firstName || undefined, last_name: lastName || undefined, - reference_code: authConfig?.gated_signup ? referenceCode : undefined, + reference_code: inviteToken || !authConfig?.gated_signup ? undefined : referenceCode, + invite_token: inviteToken || undefined, }) + consumePendingInviteToken() setSession(res.access_token, res.user, res.refresh_token) navigate('/') } catch (err: any) { @@ -170,6 +235,9 @@ export default function Login() { } try { + if (inviteToken) { + storePendingInviteToken(inviteToken) + } const redirect = `${window.location.origin}/login/callback` const authorizeUrl = await buildAuthorizeUrl(provider, redirect) window.location.href = authorizeUrl @@ -210,7 +278,16 @@ export default function Login() {

- {authConfig?.tier === 'enterprise' ? 'Enterprise' : 'Self-Hosted'} · Sign in to continue + {invitePreview ? ( + <> + Join {invitePreview.organization_name || 'your organization'} + + ) : authConfig?.tier === 'enterprise' ? ( + 'Enterprise' + ) : ( + 'Self-Hosted' + )}{' '} + · Sign in to continue

@@ -331,7 +408,7 @@ export default function Login() { setFirstName(e.target.value)} className="px-4 py-3 bg-gray-50 border-2 border-gray-200 rounded-xl focus:outline-none focus:border-[#ca8a04] focus:bg-white" /> setLastName(e.target.value)} className="px-4 py-3 bg-gray-50 border-2 border-gray-200 rounded-xl focus:outline-none focus:border-[#ca8a04] focus:bg-white" />

- setEmail(e.target.value)} required className="w-full px-4 py-3 bg-gray-50 border-2 border-gray-200 rounded-xl focus:outline-none focus:border-[#ca8a04] focus:bg-white" /> + setEmail(e.target.value)} required readOnly={!!inviteToken} className={`w-full px-4 py-3 border-2 border-gray-200 rounded-xl focus:outline-none focus:border-[#ca8a04] ${inviteToken ? 'bg-gray-100 text-gray-700' : 'bg-gray-50 focus:bg-white'}`} />
: }
- setOrgName(e.target.value)} className="w-full px-4 py-3 bg-gray-50 border-2 border-gray-200 rounded-xl focus:outline-none focus:border-[#ca8a04] focus:bg-white" /> - {authConfig?.gated_signup && ( + {!inviteToken && ( + setOrgName(e.target.value)} className="w-full px-4 py-3 bg-gray-50 border-2 border-gray-200 rounded-xl focus:outline-none focus:border-[#ca8a04] focus:bg-white" /> + )} + {authConfig?.gated_signup && !inviteToken && ( {error && }

- By signing up you become the admin of a new organization. You can invite teammates later. + {invitePreview + ? `You'll join ${invitePreview.organization_name || 'the invited organization'} directly — no separate organization is created.` + : 'By signing up you become the admin of a new organization. You can invite teammates later.'}

)} diff --git a/frontend/src/pages/auth/LoginCallback.tsx b/frontend/src/pages/auth/LoginCallback.tsx index ab047d53..28cab67d 100644 --- a/frontend/src/pages/auth/LoginCallback.tsx +++ b/frontend/src/pages/auth/LoginCallback.tsx @@ -5,6 +5,7 @@ import Logo from '../../components/Logo' import { apiClient } from '../../lib/api' import { useAuthStore } from '../../store/authStore' import { exchangeAuthorizationCode, readPkceState } from '../../lib/oidc' +import { consumePendingInviteToken, getPendingInviteToken } from '../../lib/inviteToken' export default function LoginCallback() { const navigate = useNavigate() @@ -50,6 +51,23 @@ export default function LoginCallback() { const user = await apiClient.getMe() if (!active) return + const pendingInvite = getPendingInviteToken() + if (pendingInvite) { + try { + const accepted = await apiClient.acceptInvitationByToken(pendingInvite) + consumePendingInviteToken() + setSession(accepted.access_token, accepted.user, accepted.refresh_token) + navigate('/', { replace: true }) + return + } catch (inviteErr: any) { + if (!active) return + setError(inviteErr?.response?.data?.detail || 'Signed in, but could not accept the invitation') + setSession(accessToken, user) + navigate('/', { replace: true }) + return + } + } + setSession(accessToken, user) const profile = await apiClient.getProfile() diff --git a/frontend/src/pages/callImports/CallImportDetail.tsx b/frontend/src/pages/callImports/CallImportDetail.tsx index 45b8bcfa..71c48b28 100644 --- a/frontend/src/pages/callImports/CallImportDetail.tsx +++ b/frontend/src/pages/callImports/CallImportDetail.tsx @@ -1649,7 +1649,16 @@ export default function CallImportDetail() {
{data.id}
- + + {data.latest_evaluation_status ? ( + <> + Latest evaluation + + + ) : ( + + )} + Provider:{' '} diff --git a/frontend/src/pages/callImports/CallImports.tsx b/frontend/src/pages/callImports/CallImports.tsx index c4bf63e6..f35f09d1 100644 --- a/frontend/src/pages/callImports/CallImports.tsx +++ b/frontend/src/pages/callImports/CallImports.tsx @@ -438,7 +438,17 @@ export default function CallImports() { /> - + {item.latest_evaluation_status ? ( + + Latest eval + + + ) : ( + + )} { - const providerRecordingUrl = result?.call_data?.recording_url - if (providerRecordingUrl) { - setAudioUrl(providerRecordingUrl) - return - } - if (presignedUrl?.url) { - setAudioUrl(presignedUrl.url) - return - } - setAudioUrl(null) - }, [presignedUrl, result?.call_data?.recording_url]) + let cancelled = false + let objectUrl: string | null = null - useEffect(() => { - const callShortId = observabilityCallShortId - const hasS3Recording = !!audioS3Key - if (!callShortId || !hasS3Recording || result?.call_data?.recording_url) { - return + async function resolveAudioUrl() { + if (presignedUrl?.url) { + if (!cancelled) setAudioUrl(presignedUrl.url) + return + } + + const provider = (result?.provider_platform || '').toLowerCase() + const providerRecordingUrl = getProviderRecordingUrl(result?.call_data, provider) + const resultId = result?.result_id || id + + if (!audioS3Key && providerRecordingUrl && provider !== 'elevenlabs') { + if (!cancelled) setAudioUrl(providerRecordingUrl) + return + } + + if (!audioS3Key && resultId && (AUTH_GATED_PROVIDERS.has(provider) || provider === 'vapi')) { + try { + objectUrl = await apiClient.getEvaluatorResultAudioUrl(resultId) + if (!cancelled) setAudioUrl(objectUrl) + return + } catch { + // fall through + } + } + + if (!cancelled) setAudioUrl(null) } - let cancelled = false - apiClient.getObservabilityCallAudioUrl(callShortId) - .then((url) => { - if (!cancelled) setAudioUrl(url) - }) - .catch(() => { - if (!cancelled && presignedUrl?.url) setAudioUrl(presignedUrl.url) - }) + void resolveAudioUrl() return () => { cancelled = true + if (objectUrl) URL.revokeObjectURL(objectUrl) } - }, [observabilityCallShortId, audioS3Key, result?.call_data?.recording_url, presignedUrl?.url]) + }, [ + presignedUrl, + audioS3Key, + result?.call_data, + result?.provider_platform, + result?.result_id, + id, + ]) useEffect(() => { const audio = audioRef.current @@ -763,9 +778,9 @@ export default function EvaluatorResultDetailPage({ const displayStatus = displayEvaluatorResultStatus(resultData) const statusConfig = getStatusConfig(displayStatus) const vobizPhoneNumbers = getVobizPhoneNumbers(resultData.call_data) + const providerRecordingUrl = getProviderRecordingUrl(resultData.call_data, resultData.provider_platform) const hasCallMediaOrTranscript = Boolean( - audioS3Key || - resultData.call_data?.recording_url || + hasEvaluatorResultRecording(resultData) || resultData.transcription || resultData.speaker_segments?.length || (Array.isArray(resultData.call_data?.messages) && resultData.call_data.messages.length > 0) @@ -1121,7 +1136,11 @@ export default function EvaluatorResultDetailPage({ )} - +
)} @@ -1354,12 +1373,12 @@ export default function EvaluatorResultDetailPage({ )} - {(resultData.call_data?.recording_url || audioS3Key) && ( + {(providerRecordingUrl || audioS3Key) && (

Recording

- {resultData.call_data?.recording_url ? ( + {providerRecordingUrl ? ( (null) + const [lastInviteUrl, setLastInviteUrl] = useState(null) const { data: organization, isLoading: orgLoading } = useQuery({ queryKey: ['iam', 'organization'], @@ -93,12 +96,22 @@ export default function IAM() { const inviteMutation = useMutation({ mutationFn: (data: InvitationCreate) => apiClient.inviteUser(data), - onSuccess: () => { + onSuccess: (invitation) => { queryClient.invalidateQueries({ queryKey: ['iam'] }) setShowInviteModal(false) setInviteEmail('') setInviteRole(Role.READER) - showToast('Invitation sent', 'success') + if (invitation.invite_path || invitation.invite_url) { + const shareUrl = buildInviteShareUrl(invitation) + if (shareUrl) { + setLastInviteUrl(shareUrl) + showToast('Invitation created — copy the link below to share it', 'success') + } else { + showToast('Invitation created', 'success') + } + } else { + showToast('Invitation created', 'success') + } }, onError: (error: unknown) => { showToast(getApiErrorMessage(error, 'Failed to send invitation'), 'error') @@ -174,6 +187,15 @@ export default function IAM() { inviteMutation.mutate({ email: inviteEmail, role: inviteRole }) } + const handleCopyInviteLink = async (invitation: Invitation) => { + const shareUrl = buildInviteShareUrl(invitation) + if (!shareUrl) return + await navigator.clipboard.writeText(shareUrl) + setCopiedInviteId(invitation.id) + setTimeout(() => setCopiedInviteId(null), 2000) + showToast('Invite link copied', 'success') + } + const closeResetPasswordModal = () => { setShowResetPasswordModal(false) setMemberToResetPassword(null) @@ -541,6 +563,30 @@ export default function IAM() {
+ {lastInviteUrl && ( +
+ {lastInviteUrl} + + +
+ )} {invitationsLoading ? (
Loading invitations...
) : invitations && invitations.length > 0 ? ( @@ -558,6 +604,20 @@ export default function IAM() {
{getInvitationStatusBadge(invitation.status)} + {isAdmin && invitation.status === 'pending' && buildInviteShareUrl(invitation) && ( + + )} {isAdmin && invitation.status === 'pending' && (
diff --git a/frontend/src/pages/metrics/components/MetricsStudioAudioPlayer.tsx b/frontend/src/pages/metrics/components/MetricsStudioAudioPlayer.tsx index d56e659b..8ca15a21 100644 --- a/frontend/src/pages/metrics/components/MetricsStudioAudioPlayer.tsx +++ b/frontend/src/pages/metrics/components/MetricsStudioAudioPlayer.tsx @@ -8,6 +8,8 @@ type MetricsStudioAudioPlayerProps = { metadata: Record } +const AUTH_GATED_PROVIDERS = new Set(['elevenlabs', 'vapi']) + function pickString(value: unknown): string | null { return typeof value === 'string' && value.trim() ? value.trim() : null } @@ -31,12 +33,6 @@ export default function MetricsStudioAudioPlayer({ setAudioUrl(null) try { - const providerUrl = pickString(metadata.recording_url) - if (providerUrl) { - if (!cancelled) setAudioUrl(providerUrl) - return - } - const s3Key = pickString(metadata.audio_s3_key) ?? pickString(metadata.recording_s3_key) if (s3Key) { @@ -63,10 +59,24 @@ export default function MetricsStudioAudioPlayer({ if (!cancelled) setAudioUrl(url) return } + + const provider = (result?.provider_platform || '').toLowerCase() + if (AUTH_GATED_PROVIDERS.has(provider)) { + objectUrl = await apiClient.getEvaluatorResultAudioUrl(sourceRef) + if (!cancelled) setAudioUrl(objectUrl) + return + } + const callDataUrl = pickString(result?.call_data?.recording_url) if (callDataUrl && !cancelled) { setAudioUrl(callDataUrl) } + return + } + + const providerUrl = pickString(metadata.recording_url) + if (providerUrl && !cancelled) { + setAudioUrl(providerUrl) } } catch { if (!cancelled) setError('Could not load recording') diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index d0ccfe79..cbe252ea 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -260,6 +260,18 @@ export interface Invitation { expires_at: string created_at: string organization_name?: string | null + invite_path?: string | null + invite_url?: string | null +} + +export interface InvitationPreview { + organization_name?: string | null + email: string + role: string + expires_at: string + status: string + user_exists: boolean + has_password: boolean } export interface InvitationCreate { @@ -1178,6 +1190,8 @@ export interface CallImport { failed_rows: number status: CallImportStatus error_message: string | null + /** Status of the most recent evaluation run, when any evaluation exists. */ + latest_evaluation_status?: string | null created_at: string updated_at: string created_by_email?: string | null @@ -1314,6 +1328,8 @@ export interface CallImportEvaluation { * Production and Diarised. Empty array on all other reads. */ sibling_evaluation_ids: string[] + /** Distinct LLM API calls per evaluation row (one per unique model/config). */ + expected_llm_calls_per_row?: number | null started_at: string | null finished_at: string | null created_at: string diff --git a/tests/conftest.py b/tests/conftest.py index 4fa7220a..ed46d3bf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -182,6 +182,15 @@ def _bind_runtime_database_url(database_url: str) -> None: settings.DATABASE_URL = database_url +def _drop_postgres_schema(engine) -> None: + """Drop all objects in public schema (handles migration-only tables).""" + with engine.begin() as conn: + conn.execute(text("DROP SCHEMA public CASCADE")) + conn.execute(text("CREATE SCHEMA public")) + conn.execute(text("GRANT ALL ON SCHEMA public TO public")) + conn.execute(text("GRANT ALL ON SCHEMA public TO CURRENT_USER")) + + @pytest.fixture(scope="session") def test_engine(worker_id): """ @@ -225,7 +234,11 @@ def test_engine(worker_id): yield engine finally: if drop_schema_on_teardown: - Base.metadata.drop_all(bind=engine) + parsed = make_url(str(engine.url)) + if parsed.drivername.startswith("postgresql"): + _drop_postgres_schema(engine) + else: + Base.metadata.drop_all(bind=engine) engine.dispose() diff --git a/tests/test_api/test_agents_routes.py b/tests/test_api/test_agents_routes.py index a7fca7cd..37c2014f 100644 --- a/tests/test_api/test_agents_routes.py +++ b/tests/test_api/test_agents_routes.py @@ -107,10 +107,28 @@ def test_list_and_get_agent(authenticated_client, make_agent): list_response = authenticated_client.get("/api/v1/agents") assert list_response.status_code == 200 assert len(list_response.json()) == 1 + assert list_response.json()[0]["silence_hangup_secs"] == 15 get_response = authenticated_client.get(f"/api/v1/agents/{agent.agent_id}") assert get_response.status_code == 200 - assert get_response.json()["id"] == str(agent.id) + body = get_response.json() + assert body["id"] == str(agent.id) + assert body["silence_hangup_secs"] == 15 + + +def test_update_agent_persists_silence_hangup_secs(authenticated_client, make_agent): + agent = make_agent(name="Silence Agent", agent_id="666666") + + update_response = authenticated_client.put( + f"/api/v1/agents/{agent.agent_id}", + json={"silence_hangup_secs": 60}, + ) + assert update_response.status_code == 200 + assert update_response.json()["silence_hangup_secs"] == 60 + + get_response = authenticated_client.get(f"/api/v1/agents/{agent.agent_id}") + assert get_response.status_code == 200 + assert get_response.json()["silence_hangup_secs"] == 60 def test_agent_delete_impact_without_dependencies(authenticated_client, make_agent): diff --git a/tests/test_api/test_auth_routes.py b/tests/test_api/test_auth_routes.py index 215c2589..224c12e6 100644 --- a/tests/test_api/test_auth_routes.py +++ b/tests/test_api/test_auth_routes.py @@ -20,11 +20,14 @@ from app.core.password import hash_password from app.models.database import ( APIKey, + Invitation, + InvitationStatus, Organization, OrganizationMember, RoleEnum, User, Workspace, + WorkspaceMember, ) from app.services.organization_provisioning import provision_default_workspace @@ -777,3 +780,197 @@ def test_logout_revokes_access_and_refresh_tokens( json={"refresh_token": refresh_token}, ) assert refresh.status_code == 401 + + +# --------------------------------------------------------------------------- +# Organization invite token flow +# --------------------------------------------------------------------------- + +def _make_invitation( + db_session, + *, + email, + invited_by_id, + organization_id, + role=RoleEnum.READER.value, + token=None, + expires_in_days=7, +): + invitation = Invitation( + id=uuid4(), + organization_id=organization_id, + invited_by_id=invited_by_id, + email=email, + role=role, + status=InvitationStatus.PENDING.value, + token=token or f"tok-{uuid4()}", + expires_at=datetime.now(timezone.utc) + timedelta(days=expires_in_days), + ) + db_session.add(invitation) + db_session.commit() + db_session.refresh(invitation) + return invitation + + +def test_preview_invitation_returns_org_details(client, db_session, user_context, org_id): + invitation = _make_invitation( + db_session, + email="invitee@example.com", + invited_by_id=user_context["user"].id, + organization_id=org_id, + role=RoleEnum.WRITER.value, + token="preview-token-123", + ) + + response = client.get(f"/api/v1/auth/invitations/preview/{invitation.token}") + + assert response.status_code == 200 + body = response.json() + assert body["email"] == "invitee@example.com" + assert body["role"] == RoleEnum.WRITER.value + assert body["status"] == InvitationStatus.PENDING.value + assert body["user_exists"] is False + assert body["has_password"] is False + + +def test_preview_invitation_not_found(client): + response = client.get("/api/v1/auth/invitations/preview/missing-token") + assert response.status_code == 404 + + +def test_signup_with_invite_token_joins_invited_org( + client, db_session, enable_local_password, user_context, org_id, seed_org +): + invitation = _make_invitation( + db_session, + email="new-invitee@example.com", + invited_by_id=user_context["user"].id, + organization_id=org_id, + role=RoleEnum.READER.value, + token="signup-invite-token", + ) + + response = client.post( + "/api/v1/auth/signup", + json={ + "email": "new-invitee@example.com", + "password": "Correct1!Horse", + "invite_token": invitation.token, + }, + ) + + assert response.status_code == 200 + body = response.json() + assert body["user"]["organization_id"] == str(org_id) + assert body["user"]["role"] == RoleEnum.READER.value + + user = db_session.query(User).filter(User.email == "new-invitee@example.com").one() + memberships = ( + db_session.query(OrganizationMember) + .filter(OrganizationMember.user_id == user.id) + .all() + ) + assert len(memberships) == 1 + assert memberships[0].organization_id == org_id + + personal_orgs = ( + db_session.query(Organization) + .filter(Organization.name.like("%new-invitee@example.com%")) + .count() + ) + assert personal_orgs == 0 + + db_session.refresh(invitation) + assert invitation.status == InvitationStatus.ACCEPTED.value + + +def test_signup_with_invite_token_rejects_email_mismatch( + client, db_session, enable_local_password, user_context, org_id +): + invitation = _make_invitation( + db_session, + email="invitee@example.com", + invited_by_id=user_context["user"].id, + organization_id=org_id, + token="mismatch-token", + ) + + response = client.post( + "/api/v1/auth/signup", + json={ + "email": "other@example.com", + "password": "Correct1!Horse", + "invite_token": invitation.token, + }, + ) + + assert response.status_code == 400 + assert "email" in response.json()["detail"].lower() + + +def test_accept_invitation_by_token_issues_org_scoped_session( + client, db_session, enable_local_password, user_context, org_id +): + invited_org = Organization(id=uuid4(), name="Target Org") + db_session.add(invited_org) + db_session.commit() + + invitation = _make_invitation( + db_session, + email="existing@example.com", + invited_by_id=user_context["user"].id, + organization_id=invited_org.id, + role=RoleEnum.WRITER.value, + token="accept-by-token", + ) + existing_user = User( + email="existing@example.com", + password_hash=hash_password(TEST_PASSWORD), + is_active=True, + ) + home_org = Organization(id=uuid4(), name="Home Org") + db_session.add(home_org) + db_session.add(existing_user) + db_session.flush() + db_session.add( + OrganizationMember( + organization_id=home_org.id, + user_id=existing_user.id, + role=RoleEnum.ADMIN.value, + ) + ) + provision_default_workspace( + db_session, + organization_id=invited_org.id, + created_by_user_id=user_context["user"].id, + ) + db_session.commit() + + login = client.post( + "/api/v1/auth/login", + json={"email": "existing@example.com", "password": TEST_PASSWORD}, + ) + assert login.status_code == 200 + access_token = login.json()["access_token"] + + response = client.post( + "/api/v1/auth/invitations/accept-by-token", + json={"token": invitation.token}, + headers={"Authorization": f"Bearer {access_token}"}, + ) + + assert response.status_code == 200 + body = response.json() + assert body["user"]["organization_id"] == str(invited_org.id) + assert body["user"]["role"] == RoleEnum.WRITER.value + + workspace_membership = ( + db_session.query(WorkspaceMember) + .join(Workspace, Workspace.id == WorkspaceMember.workspace_id) + .filter( + WorkspaceMember.user_id == existing_user.id, + Workspace.organization_id == invited_org.id, + ) + .count() + ) + assert workspace_membership >= 1 diff --git a/tests/test_api/test_call_import_evaluations.py b/tests/test_api/test_call_import_evaluations.py index 886c47cc..94eb2614 100644 --- a/tests/test_api/test_call_import_evaluations.py +++ b/tests/test_api/test_call_import_evaluations.py @@ -472,10 +472,7 @@ def test_create_evaluation_marks_completed_when_no_rows( authenticated_client, db_session, org_id, seed_org ): metric = _make_metric(db_session, org_id) - # Use PENDING rows so none qualify. - call_import, _ = _make_call_import( - db_session, org_id, rows=2, row_status=CallImportRowStatus.PENDING - ) + call_import, _ = _make_call_import(db_session, org_id, rows=0) response = authenticated_client.post( f"/api/v1/call-imports/{call_import.id}/evaluations", @@ -1459,3 +1456,82 @@ def test_update_evaluation_name_stamps_last_updated_by_email( assert body["name"] == "Renamed run" assert body["created_by_email"] == "owner@example.com" assert body["last_updated_by_email"] == "owner@example.com" + + +def test_diarized_rerun_after_production_eval_enqueues_materialize( + authenticated_client, db_session, org_id, seed_org, monkeypatch +): + """Pending import rows from a production run must not instant-complete + a later diarized re-run with zero rows.""" + from unittest.mock import MagicMock + + metric = _make_metric(db_session, org_id) + call_import, rows = _make_call_import( + db_session, + org_id, + rows=3, + row_status=CallImportRowStatus.PENDING, + ) + for row in rows: + row.recording_url = f"https://example.com/{row.row_index}.mp3" + row.transcript = f"transcript-{row.row_index}" + call_import.status = CallImportStatus.PROCESSING + call_import.completed_rows = 0 + db_session.commit() + + prod = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations", + json=_eval_body( + [metric.id], + transcript_sources=["production"], + auto_transcribe=False, + ), + ) + assert prod.status_code == 202, prod.text + + materialize_delay = MagicMock(return_value=types.SimpleNamespace(id="mat-task")) + fake_bulk_ops = types.ModuleType("app.workers.tasks.call_import_bulk_ops") + fake_bulk_ops.materialize_call_import_evaluation_task = types.SimpleNamespace( + delay=materialize_delay + ) + monkeypatch.setitem( + sys.modules, "app.workers.tasks.call_import_bulk_ops", fake_bulk_ops + ) + + diarized = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations", + json=_eval_body([metric.id], transcript_sources=["diarised"]), + ) + assert diarized.status_code == 202, diarized.text + body = diarized.json() + assert body["status"] == "pending" + assert body["transcript_source"] == "diarised" + assert body["total_rows"] == 3 + materialize_delay.assert_called_once() + + +def test_call_import_detail_includes_latest_evaluation_status( + authenticated_client, db_session, org_id, seed_org +): + metric = _make_metric(db_session, org_id) + call_import, _rows = _make_call_import(db_session, org_id, rows=1) + + created = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations", + json=_eval_body([metric.id], transcript_sources=["production"], auto_transcribe=False), + ) + assert created.status_code == 202, created.text + eval_id = UUID(created.json()["id"]) + + evaluation = ( + db_session.query(CallImportEvaluation) + .filter(CallImportEvaluation.id == eval_id) + .one() + ) + evaluation.status = "completed" + evaluation.completed_rows = 1 + db_session.commit() + + response = authenticated_client.get(f"/api/v1/call-imports/{call_import.id}") + assert response.status_code == 200, response.text + assert response.json()["latest_evaluation_status"] == "completed" diff --git a/tests/test_api/test_evaluator_results_routes.py b/tests/test_api/test_evaluator_results_routes.py index 08cc2cf7..9f84a04d 100644 --- a/tests/test_api/test_evaluator_results_routes.py +++ b/tests/test_api/test_evaluator_results_routes.py @@ -1,6 +1,14 @@ """API tests for evaluator results routes.""" +def _blob_storage_service(): + """Resolve the blob storage singleton exposed as s3_service.""" + import importlib + + s3_module = importlib.import_module("app.services.storage.s3_service") + return s3_module.s3_service + + def test_derive_speaker_segments_supports_smallest_payload(): from app.api.v1.routes.evaluator_results import _derive_speaker_segments_from_call_data @@ -234,3 +242,239 @@ def test_evaluator_results_overview_and_aggregate( agg = aggregate.json() assert agg["total_rows"] == 1 assert agg["completed_rows"] == 1 + + +def test_stream_evaluator_result_audio_from_s3( + authenticated_client, make_evaluator_result, monkeypatch +): + storage = _blob_storage_service() + + make_evaluator_result( + result_id="991122", + audio_s3_key="audio/organizations/test/evaluations/call-1/recording.mp3", + provider_platform="elevenlabs", + call_data={ + "recording_url": "https://api.elevenlabs.io/v1/convai/conversations/x/audio", + }, + ) + + monkeypatch.setattr(storage, "is_enabled", lambda: True) + monkeypatch.setattr( + storage, + "download_file_by_key", + lambda _key: b"fake-audio-bytes", + ) + + response = authenticated_client.get("/api/v1/evaluator-results/991122/audio") + + assert response.status_code == 200 + assert response.content == b"fake-audio-bytes" + assert response.headers["content-type"] == "audio/mpeg" + + +def test_stream_evaluator_result_audio_proxies_elevenlabs( + authenticated_client, + make_agent, + make_integration, + make_evaluator_result, + monkeypatch, +): + from app.models.enums import IntegrationPlatform + + integration = make_integration( + platform=IntegrationPlatform.ELEVENLABS.value, + api_key="enc-key", + ) + agent = make_agent(integration=integration) + make_evaluator_result( + result_id="992233", + agent_id=agent.id, + audio_s3_key=None, + provider_platform="elevenlabs", + provider_call_id="conv-123", + call_data={ + "recording_urls": { + "conversation_audio": ( + "https://api.elevenlabs.io/v1/convai/conversations/conv-123/audio" + ), + }, + }, + ) + + monkeypatch.setattr("app.core.encryption.decrypt_api_key", lambda _key: "test-xi-key") + + captured = {} + + class FakeResponse: + status_code = 200 + headers = {"content-type": "audio/mpeg"} + + def iter_content(self, chunk_size=8192): + yield b"proxied-audio" + + def fake_get(url, headers=None, stream=False, timeout=60): + captured["url"] = url + captured["headers"] = headers + return FakeResponse() + + monkeypatch.setattr("requests.get", fake_get) + + response = authenticated_client.get("/api/v1/evaluator-results/992233/audio") + + assert response.status_code == 200 + assert response.content == b"proxied-audio" + assert captured["headers"]["xi-api-key"] == "test-xi-key" + + +def test_stream_evaluator_result_audio_proxies_vapi_with_bearer( + authenticated_client, + make_agent, + make_integration, + make_evaluator_result, + monkeypatch, +): + from app.models.enums import IntegrationPlatform + + integration = make_integration( + platform=IntegrationPlatform.VAPI.value, + api_key="enc-vapi-key", + ) + agent = make_agent(integration=integration) + make_evaluator_result( + result_id="994455", + agent_id=agent.id, + audio_s3_key=None, + provider_platform="vapi", + call_data={"recordingUrl": "https://storage.vapi.ai/recording.wav"}, + ) + + monkeypatch.setattr("app.core.encryption.decrypt_api_key", lambda _key: "vapi-private-key") + + captured = {} + + class FakeResponse: + status_code = 200 + headers = {"content-type": "audio/wav"} + + def iter_content(self, chunk_size=8192): + yield b"vapi-audio" + + def fake_get(url, headers=None, stream=False, timeout=60): + captured["headers"] = headers + return FakeResponse() + + monkeypatch.setattr("requests.get", fake_get) + + response = authenticated_client.get("/api/v1/evaluator-results/994455/audio") + + assert response.status_code == 200 + assert response.content == b"vapi-audio" + assert captured["headers"]["Authorization"] == "Bearer vapi-private-key" + + +def test_stream_evaluator_result_audio_redirects_vapi_presigned_url( + authenticated_client, + make_evaluator_result, +): + signed_url = ( + "https://hipaa-recordings.example/recording.wav?" + "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=abc" + ) + make_evaluator_result( + result_id="994466", + audio_s3_key=None, + provider_platform="vapi", + call_data={ + "artifact": { + "presignedMonoUrl": signed_url, + "recordingUrl": "https://raw.example/recording.wav", + } + }, + ) + + response = authenticated_client.get( + "/api/v1/evaluator-results/994466/audio", + follow_redirects=False, + ) + + assert response.status_code in {302, 307} + assert response.headers["location"] == signed_url + + +def test_stream_evaluator_result_audio_not_found( + authenticated_client, make_evaluator_result +): + make_evaluator_result( + result_id="993344", + audio_s3_key=None, + provider_platform="elevenlabs", + call_data={}, + ) + + response = authenticated_client.get("/api/v1/evaluator-results/993344/audio") + + assert response.status_code == 404 + + +def test_re_evaluate_downloads_vapi_audio_with_bearer( + authenticated_client, + make_agent, + make_integration, + make_evaluator, + make_evaluator_result, + monkeypatch, +): + from app.models.enums import IntegrationPlatform + + integration = make_integration( + platform=IntegrationPlatform.VAPI.value, + api_key="enc", + ) + agent = make_agent(integration=integration) + evaluator = make_evaluator(agent_id=agent.id, evaluator_id="881122") + result = make_evaluator_result( + result_id="665544", + evaluator_id=evaluator.id, + agent_id=agent.id, + audio_s3_key=None, + provider_platform="vapi", + provider_call_id="call-vapi-1", + transcription="hello world", + call_data={"recordingUrl": "https://storage.vapi.ai/recording.wav"}, + ) + + captured = {} + + class FakeResp: + status_code = 200 + content = b"audio-bytes" + headers = {"content-type": "audio/mpeg"} + + def fake_get(url, headers=None, timeout=120): + captured["headers"] = headers + return FakeResp() + + monkeypatch.setattr("requests.get", fake_get) + monkeypatch.setattr("app.core.encryption.decrypt_api_key", lambda _key: "vapi-secret") + storage = _blob_storage_service() + monkeypatch.setattr( + storage, + "upload_file_by_key", + lambda *_args, **_kwargs: None, + ) + + class FakeTask: + id = "task-1" + + monkeypatch.setattr( + "app.workers.celery_app.process_evaluator_result_task.delay", + lambda *_args, **_kwargs: FakeTask(), + ) + + response = authenticated_client.post( + f"/api/v1/evaluator-results/{result.result_id}/re-evaluate" + ) + + assert response.status_code == 200 + assert captured["headers"]["Authorization"] == "Bearer vapi-secret" + diff --git a/tests/test_api/test_iam_routes.py b/tests/test_api/test_iam_routes.py index a3328201..7cf8d6c0 100644 --- a/tests/test_api/test_iam_routes.py +++ b/tests/test_api/test_iam_routes.py @@ -33,11 +33,19 @@ def test_invite_and_list_invitations(iam_admin_override, authenticated_client): json={"email": "invitee@example.com", "role": "reader"}, ) assert invite_response.status_code == 201 - assert invite_response.json()["email"] == "invitee@example.com" + body = invite_response.json() + assert body["email"] == "invitee@example.com" + assert body["invite_path"] + assert body["invite_path"].startswith("/invite/") + assert body["invite_url"] + assert "/invite/" in body["invite_url"] list_response = authenticated_client.get("/api/v1/iam/invitations") assert list_response.status_code == 200 - assert len(list_response.json()) == 1 + listed = list_response.json() + assert len(listed) == 1 + assert listed[0]["invite_path"] + assert listed[0]["invite_url"] def test_list_invitations_excludes_accepted( diff --git a/tests/test_api/test_manual_evaluations_routes.py b/tests/test_api/test_manual_evaluations_routes.py index 25dac621..2bbb87fd 100644 --- a/tests/test_api/test_manual_evaluations_routes.py +++ b/tests/test_api/test_manual_evaluations_routes.py @@ -1,5 +1,7 @@ """API tests for manual-evaluations routes.""" +import importlib +import sys from uuid import uuid4 @@ -7,10 +9,27 @@ def _org_key(org_id, filename: str = "audio.wav", prefix: str = "audio/") -> str return f"{prefix}organizations/{org_id}/audio/{filename}" +def _ensure_real_s3_service(monkeypatch): + """Rebind route storage after worker tests replace module singletons.""" + from app.api.v1.routes import manual_evaluations as manual_routes + + for module_name in ( + "app.services.storage.blob_storage_service", + "app.services.storage.s3_service", + ): + sys.modules.pop(module_name, None) + importlib.import_module(module_name) + + s3_module = importlib.import_module("app.services.storage.s3_service") + monkeypatch.setattr(manual_routes, "s3_service", s3_module.s3_service) + return manual_routes.s3_service + + def test_list_audio_files_and_presigned_url(authenticated_client, monkeypatch, org_id): from app.api.v1.routes import manual_evaluations as manual_routes from app.config import settings + _ensure_real_s3_service(monkeypatch) own_key = _org_key(org_id) monkeypatch.setattr(settings, "S3_PREFIX", "audio", raising=False) monkeypatch.setattr(manual_routes.s3_service, "is_enabled", lambda: True) @@ -55,6 +74,7 @@ def test_manual_evaluations_presigned_url_rejects_cross_tenant_key( from app.api.v1.routes import manual_evaluations as manual_routes from app.config import settings + _ensure_real_s3_service(monkeypatch) monkeypatch.setattr(settings, "S3_PREFIX", "audio", raising=False) monkeypatch.setattr(manual_routes.s3_service, "is_enabled", lambda: True) monkeypatch.setattr( @@ -74,6 +94,7 @@ def test_manual_evaluations_presigned_url_rejects_cross_tenant_key( def test_transcribe_audio_creates_transcription(authenticated_client, monkeypatch): from app.api.v1.routes import manual_evaluations as manual_routes + _ensure_real_s3_service(monkeypatch) monkeypatch.setattr(manual_routes.s3_service, "is_enabled", lambda: True) monkeypatch.setattr( manual_routes.transcription_service, diff --git a/tests/test_api/test_profile_routes.py b/tests/test_api/test_profile_routes.py index 26f8e277..35ab35e2 100644 --- a/tests/test_api/test_profile_routes.py +++ b/tests/test_api/test_profile_routes.py @@ -15,6 +15,8 @@ Organization, OrganizationMember, RoleEnum, + Workspace, + WorkspaceMember, ) @@ -112,6 +114,13 @@ def test_accept_pending_invitation_creates_membership( target_org = Organization(id=uuid4(), name="Invited Org") db_session.add(target_org) db_session.commit() + from app.services.organization_provisioning import provision_default_workspace + + provision_default_workspace( + db_session, + organization_id=target_org.id, + created_by_user_id=user_context["user"].id, + ) invitation = _make_invitation( db_session, @@ -145,8 +154,20 @@ def test_accept_pending_invitation_creates_membership( or membership.role == RoleEnum.WRITER ) + workspace = db_session.query(Workspace).filter(Workspace.organization_id == target_org.id).first() + if workspace is not None: + ws_member = ( + db_session.query(WorkspaceMember) + .filter( + WorkspaceMember.workspace_id == workspace.id, + WorkspaceMember.user_id == user_context["user"].id, + ) + .first() + ) + assert ws_member is not None + -def test_accept_expired_invitation_returns_400( +def test_accept_expired_invitation_returns_410( authenticated_client, user_context, db_session ): target_org = Organization(id=uuid4(), name="Stale Org") @@ -165,7 +186,7 @@ def test_accept_expired_invitation_returns_400( f"/api/v1/profile/invitations/{invitation.id}/accept" ) - assert response.status_code == 400 + assert response.status_code == 410 assert "expired" in response.json()["detail"].lower() diff --git a/tests/test_models/test_schemas.py b/tests/test_models/test_schemas.py index 5700f779..c540f81d 100644 --- a/tests/test_models/test_schemas.py +++ b/tests/test_models/test_schemas.py @@ -84,6 +84,8 @@ def test_agent_response_converts_legacy_enum_strings(): voice_ai_integration_id=uuid4(), voice_ai_agent_id="agent_123", provider_prompt=None, + prompt_variables={"company": "Acme"}, + silence_hangup_secs=45, provider_prompt_synced_at=None, created_at=datetime.now(UTC), updated_at=datetime.now(UTC), @@ -92,6 +94,8 @@ def test_agent_response_converts_legacy_enum_strings(): assert response.language.value == "en" assert response.call_type.value == "outbound" assert response.call_medium.value == "phone_call" + assert response.silence_hangup_secs == 45 + assert response.prompt_variables == {"company": "Acme"} def test_voice_bundle_response_converts_provider_enum_from_uppercase_name(): diff --git a/tests/test_services/test_call_import_bulk_ops_unified.py b/tests/test_services/test_call_import_bulk_ops_unified.py index 3a496bef..95fa4735 100644 --- a/tests/test_services/test_call_import_bulk_ops_unified.py +++ b/tests/test_services/test_call_import_bulk_ops_unified.py @@ -21,6 +21,7 @@ execute_bulk_diarization, execute_call_import_materialization, materialize_and_enqueue_evaluation, + rollup_call_import_batch_status, ) @@ -381,3 +382,94 @@ def test_materialize_production_evaluation_only_creates_transcript_rows( assert evaluation.total_rows == 1 assert len(eval_rows) == 1 assert eval_rows[0].call_import_row_id == row_with_text.id + + +def test_rollup_eval_primary_pending_imports_reflects_successful_eval(db_session): + call_import, _rows = _seed_import_with_rows(db_session, completed=0, pending=3) + + evaluation = CallImportEvaluation( + id=uuid4(), + call_import_id=call_import.id, + organization_id=call_import.organization_id, + workspace_id=call_import.workspace_id, + selected_metric_ids=[], + status="completed", + total_rows=3, + completed_rows=3, + failed_rows=0, + transcript_source="production", + ) + db_session.add(evaluation) + db_session.commit() + + rollup_call_import_batch_status(db_session, call_import) + assert call_import.status == CallImportStatus.COMPLETED + + +def test_rollup_active_diarized_eval_keeps_batch_processing(db_session): + call_import, _rows = _seed_import_with_rows(db_session, completed=0, pending=3) + + evaluation = CallImportEvaluation( + id=uuid4(), + call_import_id=call_import.id, + organization_id=call_import.organization_id, + workspace_id=call_import.workspace_id, + selected_metric_ids=[], + status="running", + total_rows=3, + completed_rows=0, + failed_rows=0, + transcript_source="diarised", + ) + db_session.add(evaluation) + db_session.commit() + + rollup_call_import_batch_status(db_session, call_import) + assert call_import.status == CallImportStatus.PROCESSING + + +def test_rollup_legacy_import_failure_stays_partial_when_some_rows_failed(db_session): + org = Organization(id=uuid4(), name="Legacy Rollup Org") + ws = Workspace( + id=uuid4(), + organization_id=org.id, + name="Default", + slug="default", + is_default=True, + ) + db_session.add_all([org, ws]) + db_session.flush() + call_import = CallImport( + id=uuid4(), + organization_id=org.id, + workspace_id=ws.id, + status=CallImportStatus.PROCESSING, + total_rows=2, + ) + db_session.add(call_import) + db_session.flush() + db_session.add_all( + [ + CallImportRow( + id=uuid4(), + call_import_id=call_import.id, + organization_id=org.id, + row_index=0, + conversation_id="ok", + status=CallImportRowStatus.COMPLETED, + recording_s3_key="ok.mp3", + ), + CallImportRow( + id=uuid4(), + call_import_id=call_import.id, + organization_id=org.id, + row_index=1, + conversation_id="bad", + status=CallImportRowStatus.FAILED, + ), + ] + ) + db_session.commit() + + rollup_call_import_batch_status(db_session, call_import) + assert call_import.status == CallImportStatus.PARTIAL diff --git a/tests/test_services/test_metric_failure_policy.py b/tests/test_services/test_metric_failure_policy.py index e20952fc..b9c57a5a 100644 --- a/tests/test_services/test_metric_failure_policy.py +++ b/tests/test_services/test_metric_failure_policy.py @@ -43,6 +43,16 @@ def test_suggest_failure_policy_prefers_no_over_yes(): assert policy.failure_values == ["no"] +def test_suggest_failure_policy_single_choice_yes_no_does_not_auto_flag_no(): + parent = _metric(name="AI reveal", selection_mode="single_choice") + policy = suggest_failure_policy( + parent, + observed_labels=["Yes", "No"], + child_names=["Yes", "No"], + ) + assert policy.failure_values == [] + + def test_prune_policy_when_no_rows_match_failure_label(): metric = _metric() raw = suggest_failure_policy(metric, observed_labels=["Yes", "No"]) diff --git a/tests/test_services/test_voice_providers/test_vapi_recording.py b/tests/test_services/test_voice_providers/test_vapi_recording.py new file mode 100644 index 00000000..f14989fc --- /dev/null +++ b/tests/test_services/test_voice_providers/test_vapi_recording.py @@ -0,0 +1,36 @@ +"""Unit tests for Vapi recording URL helpers.""" + +from app.services.voice_providers.vapi_recording import ( + extract_vapi_recording_url, + is_presigned_storage_url, +) + + +def test_is_presigned_storage_url_detects_r2_query_params(): + assert is_presigned_storage_url( + "https://example.r2.cloudflarestorage.com/file.wav?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=abc" + ) + assert not is_presigned_storage_url("https://example.r2.cloudflarestorage.com/file.wav") + + +def test_extract_vapi_recording_url_prefers_presigned_mono(): + call_data = { + "recordingUrl": "https://raw.example/recording.wav", + "artifact": { + "presignedMonoUrl": "https://signed.example/recording.wav?X-Amz-Signature=abc", + "recordingUrl": "https://raw.example/artifact-recording.wav", + "recording": {"mono": {"combinedUrl": "https://raw.example/combined.wav"}}, + }, + } + assert ( + extract_vapi_recording_url(call_data) + == "https://signed.example/recording.wav?X-Amz-Signature=abc" + ) + + +def test_extract_vapi_recording_url_falls_back_to_recording_url(): + call_data = { + "recordingUrl": "https://raw.example/recording.wav", + "artifact": {"recording": {"mono": {"combinedUrl": "https://raw.example/combined.wav"}}}, + } + assert extract_vapi_recording_url(call_data) == "https://raw.example/recording.wav" diff --git a/tests/test_workers/test_celery_task_workflows.py b/tests/test_workers/test_celery_task_workflows.py index 2719c846..a74aca2c 100644 --- a/tests/test_workers/test_celery_task_workflows.py +++ b/tests/test_workers/test_celery_task_workflows.py @@ -819,24 +819,20 @@ def test_evaluate_llm_metrics_grouped_passes_parent_for_categorization_children( db_session.add_all([parent, child_yes, child_no]) db_session.commit() - parent_calls: list = [] - flat_calls: list = [] - - def fake_evaluate_with_llm(*, llm_metrics, parent_metric=None, **kwargs): - if parent_metric is not None: - parent_calls.append((parent_metric, list(llm_metrics))) - return ( - { - str(parent.id): { - "type": "category", - "metric_name": parent.name, - "value": "Yes", - } - }, - 0.1, - ) - flat_calls.append(list(llm_metrics)) - return {}, 0.1 + grouped_calls: list = [] + + def fake_evaluate_with_llm(*, llm_metrics, metric_groups=None, parent_metric=None, **kwargs): + grouped_calls.append((metric_groups, list(llm_metrics))) + return ( + { + str(parent.id): { + "type": "category", + "metric_name": parent.name, + "value": "Yes", + } + }, + 0.1, + ) monkeypatch.setattr(task_module, "evaluate_with_llm", fake_evaluate_with_llm) @@ -853,9 +849,11 @@ def fake_evaluate_with_llm(*, llm_metrics, parent_metric=None, **kwargs): scenario=None, ) - assert len(parent_calls) == 1 - assert parent_calls[0][0].id == parent.id - assert {m.id for m in parent_calls[0][1]} == {child_yes.id, child_no.id} - assert flat_calls == [] + assert len(grouped_calls) == 1 + metric_groups, metrics = grouped_calls[0] + assert metric_groups is not None + assert len(metric_groups) == 1 + assert metric_groups[0].parent_metric.id == parent.id + assert {m.id for m in metrics} == {child_yes.id, child_no.id} assert str(parent.id) in scores assert scores[str(parent.id)]["type"] == "category" diff --git a/tests/test_workers/test_evaluate_call_import_row.py b/tests/test_workers/test_evaluate_call_import_row.py index 632b1311..e79c9369 100644 --- a/tests/test_workers/test_evaluate_call_import_row.py +++ b/tests/test_workers/test_evaluate_call_import_row.py @@ -730,6 +730,7 @@ def _capture(*_args, **kwargs): "transcription": kwargs.get("transcription"), "comparison_pair": kwargs.get("comparison_pair"), "metric_ids": [str(m.id) for m in kwargs["llm_metrics"]], + "metric_groups": kwargs.get("metric_groups"), } ) return ( @@ -755,10 +756,8 @@ def _capture(*_args, **kwargs): assert len(captured) == 1 invocation = captured[0] assert invocation["metric_ids"] == [str(metric.id)] - # Comparison metrics intentionally pass an empty transcription - # because the prompt-builder reads the pair via ``comparison_pair``. - assert invocation["transcription"] == "" assert invocation["comparison_pair"] == ("PROD speak", "DIAR speak") + assert invocation.get("metric_groups") is not None def test_evaluate_call_import_row_records_comparison_skip_when_diarised_missing( @@ -849,18 +848,15 @@ def _capture(*_args, **kwargs): ) assert result["status"] == "completed" - # Two LLM invocations: one per kind (comparison, transcript). - # Ordering is implementation-defined so key by metric. - assert len(invocations) == 2 - by_metric = {inv["metric_ids"][0]: inv for inv in invocations} - - cmp_call = by_metric[str(comparison_metric.id)] - assert cmp_call["transcription"] == "" - assert cmp_call["comparison_pair"] == ("PROD text", "DIAR text") - - transcript_call = by_metric[str(transcript_metric.id)] - assert transcript_call["comparison_pair"] is None - assert transcript_call["transcription"] == "DIAR text" + # One LLM invocation: comparison + transcript metrics share the same config. + assert len(invocations) == 1 + invocation = invocations[0] + assert set(invocation["metric_ids"]) == { + str(transcript_metric.id), + str(comparison_metric.id), + } + assert invocation["comparison_pair"] == ("PROD text", "DIAR text") + assert invocation["transcription"] == "DIAR text" db_session.refresh(eval_rows[0]) # Both metrics scored exactly once; the column-input bucket has @@ -1460,3 +1456,163 @@ def _counting_query(*args, **kwargs): # Incremental path should not query CallImportEvaluationRow statuses. row_status_queries = query_calls["count"] - queries_before assert row_status_queries <= 2 + + +def test_evaluate_call_import_row_batches_multiple_parent_groups_in_one_call( + db_session, monkeypatch +): + """Multiple categorization parents with the same run LLM → one API call.""" + org, _ci, _metrics, source_rows, evaluation, eval_rows = _seed( + db_session, metric_count=0 + ) + parent_a = Metric( + id=uuid4(), + organization_id=org.id, + workspace_id=evaluation.workspace_id, + name="Outcome A", + metric_type="boolean", + trigger="always", + enabled=True, + selection_mode="multi_label", + ) + parent_b = Metric( + id=uuid4(), + organization_id=org.id, + workspace_id=evaluation.workspace_id, + name="Outcome B", + metric_type="boolean", + trigger="always", + enabled=True, + selection_mode="multi_label", + ) + child_a = Metric( + id=uuid4(), + organization_id=org.id, + workspace_id=evaluation.workspace_id, + name="Child A1", + metric_type="boolean", + trigger="always", + enabled=True, + parent_metric_id=parent_a.id, + ) + child_b = Metric( + id=uuid4(), + organization_id=org.id, + workspace_id=evaluation.workspace_id, + name="Child B1", + metric_type="boolean", + trigger="always", + enabled=True, + parent_metric_id=parent_b.id, + ) + flat = Metric( + id=uuid4(), + organization_id=org.id, + workspace_id=evaluation.workspace_id, + name="Flat Metric", + metric_type="rating", + trigger="always", + enabled=True, + ) + db_session.add_all([parent_a, parent_b, child_a, child_b, flat]) + evaluation.selected_metric_ids = [ + str(child_a.id), + str(child_b.id), + str(flat.id), + ] + evaluation.llm_provider = "openai" + evaluation.llm_model = "gpt-4o" + db_session.commit() + + invocations: list = [] + + def _capture(*_args, **kwargs): + invocations.append(kwargs) + scores = {} + for m in kwargs["llm_metrics"]: + scores[str(m.id)] = { + "value": 1, + "type": "rating", + "metric_name": m.name, + } + return scores, 0.1 + + task_module = _patch_dependencies( + monkeypatch, db_session, evaluate_with_llm=_capture + ) + result = task_module.evaluate_call_import_row_task.run(str(eval_rows[0].id)) + + assert result["status"] == "completed" + assert len(invocations) == 1 + groups = invocations[0]["metric_groups"] + assert groups is not None + assert len(groups) == 3 + parent_ids = { + g.parent_metric.id for g in groups if g.parent_metric is not None + } + assert parent_ids == {parent_a.id, parent_b.id} + + +def test_build_llm_config_buckets_splits_on_metric_overrides(db_session): + from app.workers.tasks.evaluate_call_import_row_core import ( + build_llm_config_buckets, + count_distinct_llm_configs_for_metrics, + ) + + org = Organization(id=uuid4(), name="Bucket Org") + ws = Workspace( + id=uuid4(), + organization_id=org.id, + name="Default", + slug="default", + is_default=True, + ) + db_session.add_all([org, ws]) + db_session.flush() + global_metric = Metric( + id=uuid4(), + organization_id=org.id, + workspace_id=ws.id, + name="Global", + metric_type="rating", + trigger="always", + enabled=True, + ) + override_metric = Metric( + id=uuid4(), + organization_id=org.id, + workspace_id=ws.id, + name="Override", + metric_type="rating", + trigger="always", + enabled=True, + ) + db_session.add_all([global_metric, override_metric]) + db_session.commit() + + overrides = { + str(override_metric.id): { + "provider": "anthropic", + "model": "claude-3-5-sonnet-20241022", + } + } + metrics = [global_metric, override_metric] + assert count_distinct_llm_configs_for_metrics( + metrics, + overrides=overrides, + run_provider="openai", + run_model="gpt-4o", + run_llm_config=None, + run_credential_id=None, + ) == 2 + + buckets = build_llm_config_buckets( + db_session, + metrics, + overrides=overrides, + run_provider="openai", + run_model="gpt-4o", + run_llm_config=None, + run_credential_id=None, + ) + assert len(buckets) == 2 diff --git a/tests/test_workers/test_llm_evaluation.py b/tests/test_workers/test_llm_evaluation.py index fe16d003..fedf5550 100644 --- a/tests/test_workers/test_llm_evaluation.py +++ b/tests/test_workers/test_llm_evaluation.py @@ -661,3 +661,221 @@ def test_build_evaluation_prompt_without_comparison_pair_keeps_single_transcript assert "hello there" in prompt assert "## Transcripts to Compare" not in prompt assert "### Production Transcript" not in prompt + + +def test_build_evaluation_prompt_renders_multiple_parent_and_flat_groups(): + parent_a = _make_metric( + name="Outcome A", + metric_type="boolean", + description="Category A", + ) + parent_a.selection_mode = "multi_label" + parent_a.allow_discovery = False + child_a = _make_metric(name="Child A1", metric_type="boolean") + flat = _make_metric(name="Flat Score", metric_type="rating") + parent_b = _make_metric( + name="Outcome B", + metric_type="boolean", + description="Category B", + ) + parent_b.selection_mode = "multi_label" + parent_b.allow_discovery = False + child_b = _make_metric(name="Child B1", metric_type="boolean") + + groups = [ + llm_evaluation.MetricPromptGroup(parent_a, [child_a], None), + llm_evaluation.MetricPromptGroup(parent_b, [child_b], None), + llm_evaluation.MetricPromptGroup(None, [flat], None), + ] + prompt = llm_evaluation.build_evaluation_prompt( + "transcript body", + [], + metric_groups=groups, + ) + assert "### Category: Outcome A" in prompt + assert "### Category: Outcome B" in prompt + assert '"flat_score"' in prompt + assert len(llm_evaluation.flatten_metric_groups(groups)) == 3 + + +def test_map_evaluation_to_metrics_handles_metric_groups(): + parent = _make_metric(name="Outcome", metric_type="boolean", metric_id="p1") + parent.selection_mode = "multi_label" + parent.allow_discovery = False + child = _make_metric(name="Yes Path", metric_type="boolean", metric_id="c1") + flat = _make_metric(name="Quality", metric_type="rating", metric_id="f1") + groups = [ + llm_evaluation.MetricPromptGroup(parent, [child], None), + llm_evaluation.MetricPromptGroup(None, [flat], None), + ] + response = { + "yes_path": True, + "outcome__sequence": ["yes_path"], + "quality": 0.9, + } + scores = llm_evaluation._map_evaluation_to_metrics( + response, + llm_evaluation.flatten_metric_groups(groups), + metric_groups=groups, + ) + assert "c1" in scores + assert "p1" in scores + assert "f1" in scores + assert scores["f1"]["value"] == 0.9 + + +# --------------------------------------------------------------------------- +# Namespaced categorization child keys (batched metric_groups) +# --------------------------------------------------------------------------- + + +def test_batched_metric_groups_prompt_uses_namespaced_child_keys(): + parent_a = _make_metric(name="AI reveal", metric_type="boolean") + parent_a.selection_mode = "single_choice" + parent_a.allow_discovery = False + yes_a = _make_metric(name="Yes", metric_type="boolean") + no_a = _make_metric(name="No", metric_type="boolean") + parent_b = _make_metric(name="Bot gibberish", metric_type="boolean") + parent_b.selection_mode = "single_choice" + parent_b.allow_discovery = False + yes_b = _make_metric(name="Yes", metric_type="boolean") + no_b = _make_metric(name="No", metric_type="boolean") + + groups = [ + llm_evaluation.MetricPromptGroup(parent_a, [yes_a, no_a], None), + llm_evaluation.MetricPromptGroup(parent_b, [yes_b, no_b], None), + ] + prompt = llm_evaluation.build_evaluation_prompt( + "transcript", + [], + metric_groups=groups, + ) + assert '"ai_reveal__yes"' in prompt + assert '"ai_reveal__no"' in prompt + assert '"bot_gibberish__yes"' in prompt + assert '"bot_gibberish__no"' in prompt + # Bare yes/no must not appear as child boolean keys in batched mode. + assert '\n- "yes" (true/false)' not in prompt + + +def test_batched_yes_no_parents_parse_independently_with_namespaced_keys(): + parent_a = _make_metric( + name="AI reveal", metric_type="boolean", metric_id="parent-a" + ) + parent_a.selection_mode = "single_choice" + parent_a.allow_discovery = False + yes_a = _make_metric(name="Yes", metric_type="boolean", metric_id="yes-a") + no_a = _make_metric(name="No", metric_type="boolean", metric_id="no-a") + parent_b = _make_metric( + name="Bot gibberish", metric_type="boolean", metric_id="parent-b" + ) + parent_b.selection_mode = "single_choice" + parent_b.allow_discovery = False + yes_b = _make_metric(name="Yes", metric_type="boolean", metric_id="yes-b") + no_b = _make_metric(name="No", metric_type="boolean", metric_id="no-b") + + groups = [ + llm_evaluation.MetricPromptGroup(parent_a, [yes_a, no_a], None), + llm_evaluation.MetricPromptGroup(parent_b, [yes_b, no_b], None), + ] + response = { + "ai_reveal": "no", + "ai_reveal__yes": False, + "ai_reveal__no": True, + "ai_reveal__sequence": ["no"], + "bot_gibberish": "yes", + "bot_gibberish__yes": True, + "bot_gibberish__no": False, + "bot_gibberish__sequence": ["yes"], + } + scores = llm_evaluation._map_evaluation_to_metrics( + response, + llm_evaluation.flatten_metric_groups(groups), + metric_groups=groups, + ) + assert scores["parent-a"]["value"] == "No" + assert scores["parent-b"]["value"] == "Yes" + assert scores["no-a"]["value"] is True + assert scores["yes-b"]["value"] is True + + +def test_batched_outcome_detection_namespaced_children(): + parent = _make_metric( + name="Outcome detection for ticket", + metric_type="boolean", + metric_id="outcome-parent", + ) + parent.selection_mode = "single_choice" + parent.allow_discovery = False + children = [ + _make_metric( + name="Product_not_delivered", + metric_type="boolean", + metric_id="c-prod", + ), + _make_metric( + name="Pincode_unserviceable", + metric_type="boolean", + metric_id="c-pin", + ), + ] + groups = [llm_evaluation.MetricPromptGroup(parent, children, None)] + prompt = llm_evaluation.build_evaluation_prompt( + "transcript", [], metric_groups=groups + ) + assert '"outcome_detection_for_ticket__product_not_delivered"' in prompt + assert '"outcome_detection_for_ticket__pincode_unserviceable"' in prompt + + response = { + "outcome_detection_for_ticket": "pincode_unserviceable", + "outcome_detection_for_ticket__product_not_delivered": False, + "outcome_detection_for_ticket__pincode_unserviceable": True, + "outcome_detection_for_ticket__sequence": ["pincode_unserviceable"], + } + scores = llm_evaluation._map_evaluation_to_metrics( + response, + llm_evaluation.flatten_metric_groups(groups), + metric_groups=groups, + ) + assert scores["outcome-parent"]["value"] == "Pincode_unserviceable" + assert scores["c-pin"]["value"] is True + assert scores["c-prod"]["value"] is False + + +def test_legacy_single_parent_path_still_parses_bare_child_keys(): + parent = _make_metric(name="Call Outcome", metric_type="boolean", metric_id="p1") + parent.selection_mode = "single_choice" + parent.allow_discovery = False + happy = _make_metric(name="happy_completion", metric_type="boolean", metric_id="c1") + angry = _make_metric(name="angry_hangup", metric_type="boolean", metric_id="c2") + response = { + "call_outcome": "angry_hangup", + "happy_completion": False, + "angry_hangup": True, + "call_outcome__sequence": ["angry_hangup"], + } + scores = llm_evaluation._map_evaluation_to_metrics( + response, + [happy, angry], + parent_metric=parent, + ) + assert scores["p1"]["value"] == "angry_hangup" + assert scores["c2"]["value"] is True + + +def test_metric_groups_falls_back_to_bare_child_keys_when_namespaced_missing(): + parent = _make_metric(name="Outcome", metric_type="boolean", metric_id="p1") + parent.selection_mode = "multi_label" + parent.allow_discovery = False + child = _make_metric(name="Yes Path", metric_type="boolean", metric_id="c1") + groups = [llm_evaluation.MetricPromptGroup(parent, [child], None)] + response = { + "yes_path": True, + "outcome__sequence": ["yes_path"], + } + scores = llm_evaluation._map_evaluation_to_metrics( + response, + llm_evaluation.flatten_metric_groups(groups), + metric_groups=groups, + ) + assert scores["c1"]["value"] is True diff --git a/tests/test_workers/test_process_call_import_row.py b/tests/test_workers/test_process_call_import_row.py index fc464a47..9c598414 100644 --- a/tests/test_workers/test_process_call_import_row.py +++ b/tests/test_workers/test_process_call_import_row.py @@ -174,6 +174,15 @@ def upload_file_by_key(self, file_content, key, content_type="audio/mpeg"): self.uploads.append({"key": key, "size": len(file_content), "content_type": content_type}) return key + def get_organization_root_prefix(self, organization_id: str) -> str: + return f"{self.prefix}organizations/{organization_id}/" + + def list_audio_files(self, **_kwargs): + return [] + + def generate_presigned_url_by_key(self, key, expiration=3600): + return f"https://example.com/{key}?exp={expiration}" + class _NonClosingSession: """Proxy that forwards everything to the underlying session but ignores .close().