From 801800511d0eaedc99f7d4935aa1c3bff5c3da54 Mon Sep 17 00:00:00 2001 From: M Sami Date: Fri, 7 Aug 2026 19:02:08 +0530 Subject: [PATCH 01/32] feat: added call-imports metadata --- .../components/callImports/AuditMetaChips.tsx | 175 +++ .../pages/callImports/CallImportDetail.tsx | 30 +- .../CallImportEvaluationDetail.tsx | 23 +- .../src/pages/callImports/CallImports.tsx | 1128 +++++++++-------- 4 files changed, 773 insertions(+), 583 deletions(-) create mode 100644 frontend/src/components/callImports/AuditMetaChips.tsx diff --git a/frontend/src/components/callImports/AuditMetaChips.tsx b/frontend/src/components/callImports/AuditMetaChips.tsx new file mode 100644 index 00000000..a7bae2e4 --- /dev/null +++ b/frontend/src/components/callImports/AuditMetaChips.tsx @@ -0,0 +1,175 @@ +import type { ReactNode } from 'react' + +/** Bordered metadata chips for call-import audit fields. */ + +const CHIP_CLASS = + 'inline-flex flex-col min-w-0 rounded-md border border-slate-200 bg-white px-2 py-1 text-[11px] leading-tight shadow-sm' + +export function formatMetaDateTime(iso: string | null | undefined): string { + if (!iso) return '—' + const parsed = new Date(iso) + if (Number.isNaN(parsed.getTime())) return '—' + return parsed.toLocaleString() +} + +type AuditMetaChipProps = { + label: string + value: string | null | undefined + wide?: boolean + /** Tighter chip for horizontal list rows */ + dense?: boolean + /** Fill a grid cell (2×2 activity column on list page) */ + stacked?: boolean + className?: string +} + +export function AuditMetaChip({ + label, + value, + wide, + dense, + stacked, + className = '', +}: AuditMetaChipProps) { + const display = value?.trim() || '—' + const sizeClass = dense + ? stacked + ? 'w-full min-w-0 px-1.5 py-0.5 text-[10px]' + : 'shrink-0 px-1.5 py-0.5 text-[10px] min-w-[4.75rem] max-w-[8.5rem]' + : wide + ? 'min-w-[7.5rem] max-w-[12rem]' + : 'min-w-[6.5rem] max-w-[10rem]' + return ( +
+ {label} + {display} +
+ ) +} + +type AuditMetaChipRowProps = { + className?: string + wide?: boolean + compact?: boolean + children: ReactNode +} + +function AuditMetaChipRow({ + className = '', + wide, + compact, + children, +}: AuditMetaChipRowProps) { + return ( +
+ {children} +
+ ) +} + +type CallImportAuditMetaProps = { + createdAt: string | null | undefined + updatedAt: string | null | undefined + createdByEmail?: string | null + lastUpdatedByEmail?: string | null + className?: string + wide?: boolean + compact?: boolean + /** 2×2 grid for table cells (no horizontal scroll) */ + stacked?: boolean +} + +/** Created / updated timestamps + actor emails for a call-import batch. */ +export function CallImportAuditMeta({ + createdAt, + updatedAt, + createdByEmail, + lastUpdatedByEmail, + className, + wide = true, + compact, + stacked, +}: CallImportAuditMetaProps) { + const chipCommon = { wide, dense: compact, stacked } + if (stacked) { + return ( +
+ + + + +
+ ) + } + return ( + + + + + + + ) +} + +type EvaluationAuditMetaProps = { + createdAt: string | null | undefined + updatedAt?: string | null | undefined + startedAt?: string | null + finishedAt?: string | null + /** User who started this evaluation run (API: created_by_email). */ + runByEmail?: string | null + /** @deprecated Use runByEmail */ + createdByEmail?: string | null + lastUpdatedByEmail?: string | null + formatDate?: (iso: string | null | undefined) => string + className?: string + wide?: boolean + compact?: boolean + showRunTimes?: boolean +} + +/** Evaluation run metadata (detail header or list card). */ +export function EvaluationAuditMeta({ + createdAt, + updatedAt, + startedAt, + finishedAt, + runByEmail, + createdByEmail, + lastUpdatedByEmail, + formatDate = formatMetaDateTime, + className, + wide = true, + compact, + showRunTimes = true, +}: EvaluationAuditMetaProps) { + const runner = runByEmail ?? createdByEmail + return ( + + + + + {updatedAt ? ( + + ) : null} + {showRunTimes && startedAt ? ( + + ) : null} + {showRunTimes && finishedAt ? ( + + ) : null} + + ) +} diff --git a/frontend/src/pages/callImports/CallImportDetail.tsx b/frontend/src/pages/callImports/CallImportDetail.tsx index bc8687b8..9270e14e 100644 --- a/frontend/src/pages/callImports/CallImportDetail.tsx +++ b/frontend/src/pages/callImports/CallImportDetail.tsx @@ -57,6 +57,10 @@ import Button from '../../components/Button' import ConfirmModal from '../../components/ConfirmModal' import Pagination from '../../components/Pagination' import StatusBadge from '../../components/shared/StatusBadge' +import { + CallImportAuditMeta, + EvaluationAuditMeta, +} from '../../components/callImports/AuditMetaChips' import DiariseStatusPill from '../../components/callImports/DiariseStatusPill' import ProviderModelPicker, { type ProviderModelValue, @@ -1680,12 +1684,12 @@ export default function CallImportDetail() { )} - - Created: {new Date(data.created_at).toLocaleString()} - - - Updated: {new Date(data.updated_at).toLocaleString()} - + @@ -3183,13 +3187,19 @@ export default function CallImportDetail() { to={`/call-imports/${id}/evaluations/${evaluation.id}`} className="flex-1 min-w-0 flex items-center justify-between gap-3" > -
+

{headerLabel}

-

- Created {new Date(evaluation.created_at).toLocaleString()} -

+
diff --git a/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx b/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx index 542fdd6c..0c42f91a 100644 --- a/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx +++ b/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx @@ -92,6 +92,7 @@ import ProviderModelPicker, { } from '../../components/providers/ProviderModelPicker' import { getActiveWorkspaceId, useWorkspaceStore } from '../../store/workspaceStore' import StatusBadge from '../../components/shared/StatusBadge' +import { EvaluationAuditMeta } from '../../components/callImports/AuditMetaChips' import DiariseStatusPill from '../../components/callImports/DiariseStatusPill' import CallImportProgressBar from './components/CallImportProgressBar' import MetricPromptImprovementsPanel from './components/MetricPromptImprovementsPanel' @@ -2499,19 +2500,15 @@ export default function CallImportEvaluationDetail() { ? 'Evaluated on Diarised transcript' : 'Evaluated on Production transcript'} - - Created: {formatDateTime(evaluation.created_at)} - - {evaluation.started_at && ( - - Started: {formatDateTime(evaluation.started_at)} - - )} - {evaluation.finished_at && ( - - Finished: {formatDateTime(evaluation.finished_at)} - - )} +
diff --git a/frontend/src/pages/callImports/CallImports.tsx b/frontend/src/pages/callImports/CallImports.tsx index c9ccaf26..b3d6295a 100644 --- a/frontend/src/pages/callImports/CallImports.tsx +++ b/frontend/src/pages/callImports/CallImports.tsx @@ -1,560 +1,568 @@ -import { useMemo, useState } from 'react' -import { Link, useNavigate } from 'react-router-dom' -import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { - ChevronLeft, - ChevronRight, - FileAudio, - FileSpreadsheet, - Layers, - Phone, - RefreshCw, - Trash2, - Upload, -} from 'lucide-react' -import { Tag as TagIcon } from 'lucide-react' -import { apiClient } from '../../lib/api' -import { getApiErrorMessage } from '../../lib/apiErrors' -import { useToast } from '../../hooks/useToast' -import { useWorkspaceStore } from '../../store/workspaceStore' -import type { CallImport, CallImportStatus, CallImportTag } from '../../types/api' -import Button from '../../components/Button' -import ConfirmModal from '../../components/ConfirmModal' -import StatusBadge from '../../components/shared/StatusBadge' -import CallImportProgressBar from './components/CallImportProgressBar' -import UploadAudioModal from './components/UploadAudioModal' -import UploadCsvModal from './components/UploadCsvModal' - -const PAGE_SIZE = 20 - -const STATUS_OPTIONS: Array<{ label: string; value: '' | CallImportStatus }> = [ - { label: 'All statuses', value: '' }, - { label: 'Uploaded', value: 'uploaded' }, - { label: 'Mapped', value: 'mapped' }, - { label: 'Pending', value: 'pending' }, - { label: 'Processing', value: 'processing' }, - { label: 'Completed', value: 'completed' }, - { label: 'Partial', value: 'partial' }, - { label: 'Failed', value: 'failed' }, - { label: 'Deleting', value: 'deleting' }, -] - -type UploadTab = 'datasets' | 'audio' - -export default function CallImports() { - const navigate = useNavigate() - const queryClient = useQueryClient() - const { showToast, ToastContainer } = useToast() - // Active workspace is part of every workspace-scoped queryKey so a - // workspace switch produces a clean cache miss instead of leaking - // rows from the previously-active workspace. - const activeWorkspaceId = useWorkspaceStore((s) => s.activeWorkspaceId) - const [page, setPage] = useState(1) - const [statusFilter, setStatusFilter] = useState<'' | CallImportStatus>('') - const [datasetFilter, setDatasetFilter] = useState('') - const [tagFilter, setTagFilter] = useState([]) - const [activeTab, setActiveTab] = useState('datasets') - const [showUpload, setShowUpload] = useState(false) - const [showAudioUpload, setShowAudioUpload] = useState(false) - const [pendingDelete, setPendingDelete] = useState(null) - const [deleteError, setDeleteError] = useState(null) - - const { data: datasets = [] } = useQuery({ - queryKey: ['call-import-datasets', activeWorkspaceId], - queryFn: () => apiClient.listCallImportDatasets(), - }) - - const { data: allTags = [] } = useQuery({ - queryKey: ['call-import-tags', activeWorkspaceId], - queryFn: () => apiClient.listCallImportTags(), - }) - - const deleteMutation = useMutation({ - mutationFn: (id: string) => apiClient.deleteCallImport(id), - onSuccess: (result) => { - queryClient.invalidateQueries({ queryKey: ['call-imports'] }) - setPendingDelete(null) - setDeleteError(null) - if (result.status === 'accepted') { - showToast( - 'Deletion started — large imports may take a minute.', - 'success', - ) - } - }, - onError: (err: unknown) => { - const message = getApiErrorMessage(err, 'Failed to delete import.') - setDeleteError(message) - showToast(message, 'error') - }, - }) - - const queryParams = useMemo( - () => ({ - page, - page_size: PAGE_SIZE, - ...(statusFilter ? { status: statusFilter } : {}), - ...(datasetFilter ? { dataset: datasetFilter } : {}), - ...(tagFilter.length > 0 ? { tag_id: tagFilter } : {}), - source_format: activeTab === 'audio' ? 'audio' : '__non_audio__', - }), - [page, statusFilter, datasetFilter, tagFilter, activeTab], - ) - - const { data, isLoading, isFetching, refetch } = useQuery({ - queryKey: ['call-imports', activeWorkspaceId, queryParams], - queryFn: () => apiClient.listCallImports(queryParams), - refetchInterval: (query) => { - const items = query.state.data?.items ?? [] - const hasActive = items.some( - (i: CallImport) => i.status === 'pending' || i.status === 'processing', - ) - const hasDeleting = items.some( - (i: CallImport) => i.status === 'deleting', - ) - if (hasDeleting) return 3000 - return hasActive ? 5000 : false - }, - }) - - const items = data?.items ?? [] - const total = data?.total ?? 0 - const totalPages = Math.max(1, Math.ceil(total / PAGE_SIZE)) - - return ( -
- -
-
-

Call Imports

-

- Upload datasets from CSV / Excel, or add manual call recordings - directly and use the same diarisation and evaluation tools. -

-
-
- - - - - - - - -
-
- -
- - -
- - {/* - High-level dataset segregation lives at the top of the page so users can - scope all filtering/searching that follows to a specific dataset. We - intentionally render this above the main card to make it visually - distinct from the in-card status/tag filters. - */} -
- - - {datasetFilter && ( - - Showing imports tagged with dataset “{datasetFilter}”. - - )} -
- -
-
-
-
- - -
- {allTags.length > 0 && ( -
- Tags: - {allTags.map((tag: CallImportTag) => { - const active = tagFilter.includes(tag.id) - return ( - - ) - })} - {tagFilter.length > 0 && ( - - )} -
- )} -
-

- {total} {activeTab === 'audio' ? 'manual upload' : 'dataset import'} - {total === 1 ? '' : 's'} -

-
- - {isLoading ? ( -
- -

Loading imports...

-
- ) : items.length === 0 ? ( -
- -

- {statusFilter - ? 'No imports match this filter.' - : activeTab === 'audio' - ? 'No manual audio uploads yet.' - : 'No dataset uploads yet.'} -

- {!statusFilter && ( - - )} -
- ) : ( -
- - - - - - - - - - - - - - {items.map((item: CallImport) => { - const isDeleting = item.status === 'deleting' - return ( - { - if (isDeleting) return - navigate(`/call-imports/${item.id}`) - }} - > - - - - - - - - - ) - })} - -
- Filename - - Provider - - Dataset / Tags - - Progress - - Status - - Created - - Actions -
-
- {item.original_filename || '(unnamed)'} -
-
- {item.id.slice(0, 8)} -
- 		- {item.source_format === 'audio' ? ( - - - Manual upload - - ) : item.provider || ( - - — - - )} - -
- {item.dataset ? ( - - {item.dataset} - - ) : ( - - no dataset - - )} - {item.tags.length > 0 && ( -
- {item.tags.map((tag) => ( - - {tag.name} - - ))} -
- )} -
- 		- - - - - {new Date(item.created_at).toLocaleString()} - e.stopPropagation()} - > -
- - View - - -
-
- - {totalPages > 1 && ( -
-

- Page {page} of {totalPages} -

-
- - -
-
- )} -
- )} -
- - setShowUpload(false)} /> - setShowAudioUpload(false)} - /> - - { - if (!pendingDelete) return '' - const name = pendingDelete.original_filename || '(unnamed)' - const total = pendingDelete.total_rows - const completed = pendingDelete.completed_rows - const inFlight = - pendingDelete.status === 'pending' || - pendingDelete.status === 'processing' || - pendingDelete.status === 'deleting' - const lines = [ - `“${name}” will be permanently deleted, along with all ${total} row record${total === 1 ? '' : 's'} and ${completed} stored recording${completed === 1 ? '' : 's'} in S3.`, - inFlight - ? 'This batch is still processing — pending tasks will be revoked before deletion.' - : '', - 'This cannot be undone.', - deleteError ? `Error: ${deleteError}` : '', - ] - return lines.filter(Boolean).join('\n\n') - })()} - confirmLabel="Delete" - cancelLabel="Cancel" - variant="danger" - isLoading={deleteMutation.isPending} - onConfirm={() => { - if (pendingDelete) deleteMutation.mutate(pendingDelete.id) - }} - onCancel={() => { - if (deleteMutation.isPending) return - setPendingDelete(null) - setDeleteError(null) - }} - /> -
- ) -} +import { useMemo, useState } from 'react' +import { Link, useNavigate } from 'react-router-dom' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { + ChevronLeft, + ChevronRight, + FileAudio, + FileSpreadsheet, + Layers, + Phone, + RefreshCw, + Trash2, + Upload, +} from 'lucide-react' +import { Tag as TagIcon } from 'lucide-react' +import { apiClient } from '../../lib/api' +import { getApiErrorMessage } from '../../lib/apiErrors' +import { useToast } from '../../hooks/useToast' +import { useWorkspaceStore } from '../../store/workspaceStore' +import type { CallImport, CallImportStatus, CallImportTag } from '../../types/api' +import Button from '../../components/Button' +import ConfirmModal from '../../components/ConfirmModal' +import StatusBadge from '../../components/shared/StatusBadge' +import { CallImportAuditMeta } from '../../components/callImports/AuditMetaChips' +import CallImportProgressBar from './components/CallImportProgressBar' +import UploadAudioModal from './components/UploadAudioModal' +import UploadCsvModal from './components/UploadCsvModal' + +const PAGE_SIZE = 20 + +const STATUS_OPTIONS: Array<{ label: string; value: '' | CallImportStatus }> = [ + { label: 'All statuses', value: '' }, + { label: 'Uploaded', value: 'uploaded' }, + { label: 'Mapped', value: 'mapped' }, + { label: 'Pending', value: 'pending' }, + { label: 'Processing', value: 'processing' }, + { label: 'Completed', value: 'completed' }, + { label: 'Partial', value: 'partial' }, + { label: 'Failed', value: 'failed' }, + { label: 'Deleting', value: 'deleting' }, +] + +type UploadTab = 'datasets' | 'audio' + +export default function CallImports() { + const navigate = useNavigate() + const queryClient = useQueryClient() + const { showToast, ToastContainer } = useToast() + // Active workspace is part of every workspace-scoped queryKey so a + // workspace switch produces a clean cache miss instead of leaking + // rows from the previously-active workspace. + const activeWorkspaceId = useWorkspaceStore((s) => s.activeWorkspaceId) + const [page, setPage] = useState(1) + const [statusFilter, setStatusFilter] = useState<'' | CallImportStatus>('') + const [datasetFilter, setDatasetFilter] = useState('') + const [tagFilter, setTagFilter] = useState([]) + const [activeTab, setActiveTab] = useState('datasets') + const [showUpload, setShowUpload] = useState(false) + const [showAudioUpload, setShowAudioUpload] = useState(false) + const [pendingDelete, setPendingDelete] = useState(null) + const [deleteError, setDeleteError] = useState(null) + + const { data: datasets = [] } = useQuery({ + queryKey: ['call-import-datasets', activeWorkspaceId], + queryFn: () => apiClient.listCallImportDatasets(), + }) + + const { data: allTags = [] } = useQuery({ + queryKey: ['call-import-tags', activeWorkspaceId], + queryFn: () => apiClient.listCallImportTags(), + }) + + const deleteMutation = useMutation({ + mutationFn: (id: string) => apiClient.deleteCallImport(id), + onSuccess: (result) => { + queryClient.invalidateQueries({ queryKey: ['call-imports'] }) + setPendingDelete(null) + setDeleteError(null) + if (result.status === 'accepted') { + showToast( + 'Deletion started — large imports may take a minute.', + 'success', + ) + } + }, + onError: (err: unknown) => { + const message = getApiErrorMessage(err, 'Failed to delete import.') + setDeleteError(message) + showToast(message, 'error') + }, + }) + + const queryParams = useMemo( + () => ({ + page, + page_size: PAGE_SIZE, + ...(statusFilter ? { status: statusFilter } : {}), + ...(datasetFilter ? { dataset: datasetFilter } : {}), + ...(tagFilter.length > 0 ? { tag_id: tagFilter } : {}), + source_format: activeTab === 'audio' ? 'audio' : '__non_audio__', + }), + [page, statusFilter, datasetFilter, tagFilter, activeTab], + ) + + const { data, isLoading, isFetching, refetch } = useQuery({ + queryKey: ['call-imports', activeWorkspaceId, queryParams], + queryFn: () => apiClient.listCallImports(queryParams), + refetchInterval: (query) => { + const items = query.state.data?.items ?? [] + const hasActive = items.some( + (i: CallImport) => i.status === 'pending' || i.status === 'processing', + ) + const hasDeleting = items.some( + (i: CallImport) => i.status === 'deleting', + ) + if (hasDeleting) return 3000 + return hasActive ? 5000 : false + }, + }) + + const items = data?.items ?? [] + const total = data?.total ?? 0 + const totalPages = Math.max(1, Math.ceil(total / PAGE_SIZE)) + + return ( +
+ +
+
+

Call Imports

+

+ Upload datasets from CSV / Excel, or add manual call recordings + directly and use the same diarisation and evaluation tools. +

+
+
+ + + + + + + + +
+
+ +
+ + +
+ + {/* + High-level dataset segregation lives at the top of the page so users can + scope all filtering/searching that follows to a specific dataset. We + intentionally render this above the main card to make it visually + distinct from the in-card status/tag filters. + */} +
+ + + {datasetFilter && ( + + Showing imports tagged with dataset “{datasetFilter}”. + + )} +
+ +
+
+
+
+ + +
+ {allTags.length > 0 && ( +
+ Tags: + {allTags.map((tag: CallImportTag) => { + const active = tagFilter.includes(tag.id) + return ( + + ) + })} + {tagFilter.length > 0 && ( + + )} +
+ )} +
+

+ {total} {activeTab === 'audio' ? 'manual upload' : 'dataset import'} + {total === 1 ? '' : 's'} +

+
+ + {isLoading ? ( +
+ +

Loading imports...

+
+ ) : items.length === 0 ? ( +
+ +

+ {statusFilter + ? 'No imports match this filter.' + : activeTab === 'audio' + ? 'No manual audio uploads yet.' + : 'No dataset uploads yet.'} +

+ {!statusFilter && ( + + )} +
+ ) : ( +
+ + + + + + + + + + + + + + {items.map((item: CallImport) => { + const isDeleting = item.status === 'deleting' + return ( + { + if (isDeleting) return + navigate(`/call-imports/${item.id}`) + }} + > + + + + + + + + + ) + })} + +
+ Filename + + Provider + + Dataset / Tags + + Progress + + Status + + Activity + + Actions +
+
+ {item.original_filename || '(unnamed)'} +
+
+ {item.id.slice(0, 8)} +
+ 		+ {item.source_format === 'audio' ? ( + + + Manual upload + + ) : item.provider || ( + + — + + )} + +
+ {item.dataset ? ( + + {item.dataset} + + ) : ( + + no dataset + + )} + {item.tags.length > 0 && ( +
+ {item.tags.map((tag) => ( + + {tag.name} + + ))} +
+ )} +
+ 		+ + + + + + e.stopPropagation()} + > +
+ + View + + +
+
+ + {totalPages > 1 && ( +
+

+ Page {page} of {totalPages} +

+
+ + +
+
+ )} +
+ )} +
+ + setShowUpload(false)} /> + setShowAudioUpload(false)} + /> + + { + if (!pendingDelete) return '' + const name = pendingDelete.original_filename || '(unnamed)' + const total = pendingDelete.total_rows + const completed = pendingDelete.completed_rows + const inFlight = + pendingDelete.status === 'pending' || + pendingDelete.status === 'processing' || + pendingDelete.status === 'deleting' + const lines = [ + `“${name}” will be permanently deleted, along with all ${total} row record${total === 1 ? '' : 's'} and ${completed} stored recording${completed === 1 ? '' : 's'} in S3.`, + inFlight + ? 'This batch is still processing — pending tasks will be revoked before deletion.' + : '', + 'This cannot be undone.', + deleteError ? `Error: ${deleteError}` : '', + ] + return lines.filter(Boolean).join('\n\n') + })()} + confirmLabel="Delete" + cancelLabel="Cancel" + variant="danger" + isLoading={deleteMutation.isPending} + onConfirm={() => { + if (pendingDelete) deleteMutation.mutate(pendingDelete.id) + }} + onCancel={() => { + if (deleteMutation.isPending) return + setPendingDelete(null) + setDeleteError(null) + }} + /> +
+ ) +} From 6d46894fd54222537ff8314df962a91223ba6705 Mon Sep 17 00:00:00 2001 From: M Sami Date: Fri, 7 Aug 2026 19:14:52 +0530 Subject: [PATCH 02/32] feat: added import-calls user metadata --- app/api/v1/routes/call_import_evaluations.py | 17804 ++++++++-------- app/api/v1/routes/call_imports.py | 7963 +++---- app/config.py | 2 +- app/migrations/059_call_import_audit_users.py | 69 + app/models/database.py | 5464 ++--- app/models/schemas.py | 4 + app/services/call_imports/audit.py | 99 + env.example | 2 +- frontend/src/types/api.ts | 4268 ++-- tests/test_api/test_call_import_audit.py | 78 + .../test_api/test_call_import_evaluations.py | 40 + 11 files changed, 18106 insertions(+), 17687 deletions(-) create mode 100644 app/migrations/059_call_import_audit_users.py create mode 100644 app/services/call_imports/audit.py create mode 100644 tests/test_api/test_call_import_audit.py diff --git a/app/api/v1/routes/call_import_evaluations.py b/app/api/v1/routes/call_import_evaluations.py index f73744ad..95222d6d 100644 --- a/app/api/v1/routes/call_import_evaluations.py +++ b/app/api/v1/routes/call_import_evaluations.py @@ -1,8877 +1,8927 @@ -"""Evaluation routes scoped to a Call Import batch.""" - -from __future__ import annotations - -import asyncio -import csv -import base64 -import io -import json -import math -import re -import statistics -from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple -from uuid import UUID - -from datetime import date, datetime, timedelta, timezone - -from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status -from fastapi.responses import StreamingResponse -from loguru import logger -from pydantic import BaseModel, Field, field_validator -from sqlalchemy import desc, func, or_, text -from sqlalchemy.orm import Session -from sqlalchemy.orm.attributes import flag_modified - -from app.core.auth import Principal, get_principal -from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message -from app.database import get_db -from app.dependencies import ( - get_api_key, - get_organization_id, - get_workspace_id, - require_enterprise_feature, -) -from app.services.workspace_rbac import resolve_workspace_capabilities -from app.models.database import ( - AIProvider, - CallImport, - CallImportEvaluation, - CallImportEvaluationReportSnapshot, - CallImportEvaluationRow, - CallImportRow, - Metric, - PromptPartial, - Workspace, -) -from app.models.enums import CallImportRowStatus, ModelProvider -from app.models.schemas import ( - CallImportEvaluationAggregateResponse, - CallImportEvaluationBulkDelete, - CallImportEvaluationBulkActionResponse, - CallImportEvaluationCreate, - CallImportEvaluationListResponse, - CallImportEvaluationResponse, - CallImportEvaluationRetryRequest, - CallImportEvaluationRetryResponse, - CallImportEvaluationRetrySkippedItem, - CallImportEvaluationRowListResponse, - CallImportEvaluationRowResponse, - CallImportEvaluationUpdate, - CallImportMetricAggregate, - CallImportMetricHistogramBucket, - CallImportMetricLabelPair, - CallImportMetricSummary, - CallImportMetricValueCount, - DiscoveredLabelDeleteRequest, - DiscoveredLabelItem, - DiscoveredLabelMergeRequest, - DiscoveredLabelsResponse, - DiscoveredMetricDeleteRequest, - DiscoveredMetricItem, - DiscoveredMetricMergeRequest, - DiscoveredMetricsResponse, - EvaluationInsightsRequest, - EvaluationTldrSummary, - EvaluationMetricClustersRequest, - EvaluationMetricClustersState, - EvaluationPromptImprovementsRequest, - EvaluationPromptImprovementsState, - MetricFailurePoliciesResponse, - MetricFailurePoliciesSaveRequest, - MetricFailurePolicy, - MetricClusterEligibleRow, - MetricClusterEligibleRowsResponse, - EvaluationUserInsightsRequest, - EvaluationUserInsightsState, - MetricFlowEdge, - MetricPeriodDelta, - MetricFlowNode, - MetricFlowResponse, -) -from app.services.reporting.call_import_evaluation_pdf_report import ( - call_import_evaluation_pdf_report_service, -) -from app.services.call_import_metric_clusters import ( - METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, - estimate_metric_clusters_llm_calls, - filter_completed_row_pairs, - list_eligible_cluster_rows, - metric_clusters_raw_is_cancelled, - metric_clusters_state_from_raw, - metric_clusters_state_to_db, -) -from app.services.metric_failure_policy import ( - aggregate_primary_percent, - build_failure_policy_previews, - effective_policies, - failure_rate_percent_from_rows, - failure_policies_to_db, - has_clusterable_metrics, - merge_clustering_policies, - merge_failure_policies_into_raw, - policies_from_evaluation_raw, - validate_failure_policies_for_metrics, -) -from app.services.call_import_user_insights import ( - normalize_max_llm_calls, - total_llm_calls_for_rows, - user_insights_state_from_raw, -) - -router = APIRouter( - prefix="/call-imports/{call_import_id}/evaluations", - tags=["Call Import Evaluations"], - dependencies=[Depends(require_enterprise_feature("call_imports"))], -) - - -class CallImportEvaluationPdfReportRequest(BaseModel): - vendor_name: str = Field(..., min_length=1, max_length=120) - report_type: Literal["external", "internal"] = "external" - include_weekly_delta: bool = False - include_period_delta: bool = False - baseline_evaluation_id: Optional[str] = None - period_label: Optional[str] = Field(default=None, max_length=64) - use_case: Optional[str] = Field(default=None, max_length=120) - internal_brand_image_id: Optional[str] = None - external_brand_image_id: Optional[str] = None - report_config: Dict[str, Any] = Field(default_factory=dict) - platform_base_url: Optional[str] = Field( - default=None, - max_length=512, - description="Frontend origin for deep links to example calls in internal PDFs.", - ) - - @field_validator("vendor_name") - @classmethod - def _clean_vendor_name(cls, value: str) -> str: - cleaned = value.strip() - if not cleaned: - raise ValueError("Vendor name is required.") - return cleaned - - -class CallImportEvaluationBaselineCandidate(BaseModel): - evaluation_id: str - name: str - dataset: str - period_label: Optional[str] = None - period_start: Optional[date] = None - period_end: Optional[date] = None - period_display: str - completed_rows: int - created_at: datetime - is_default: bool = False - - -class CallImportEvaluationBaselineCandidatesResponse(BaseModel): - items: List[CallImportEvaluationBaselineCandidate] - default_evaluation_id: Optional[str] = None - - -def _require_import( - db: Session, - call_import_id: UUID, - organization_id: UUID, -) -> CallImport: - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException(status_code=404, detail="Call import not found") - return call_import - - -def require_call_import_capability(capability: str): - """Ensure the caller has *capability* in the call import's workspace (not just the header).""" - - def _dep( - call_import_id: UUID, - principal: Principal = Depends(get_principal), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), - ) -> CallImport: - call_import = _require_import(db, call_import_id, organization_id) - caps, _, role = resolve_workspace_capabilities( - db, - principal=principal, - workspace_id=call_import.workspace_id, - organization_id=organization_id, - ) - if capability not in caps: - raise HTTPException( - status_code=403, - detail=capability_denied_message( - capability, - role_name=role.name if role else None, - workspace_label="the active workspace", - ), - ) - return call_import - - return _dep - - -def _flatten_transcript(text: Optional[str]) -> str: - """Collapse a multi-line transcript onto a single line for spreadsheet export. - - The diarised transcript is stored as ``: `` lines joined - by ``\\n`` because the in-app ``TranscriptView`` parses those line - breaks to render chat bubbles. In Excel / Google Sheets that same - newline-per-turn formatting causes each cell to balloon vertically, - which the user reads as "lots of empty space on top of the cell". - Flattening at export time keeps the DB shape intact while giving the - spreadsheet a single-line cell per row. - """ - if not text: - return "" - parts = [ - segment.strip() - for segment in text.replace("\r\n", "\n").replace("\r", "\n").split("\n") - ] - return " ".join(p for p in parts if p) - - -def _evaluated_transcript_source_label( - evaluation: CallImportEvaluation, - source_row: CallImportRow, -) -> str: - """Label which transcript source this row was scored against.""" - source = (evaluation.transcript_source or "diarised").strip().lower() - if source == "production": - if not (source_row.transcript or "").strip(): - return "" - return "Production" - if not (source_row.diarised_transcript or "").strip(): - return "" - return "Diarised" - - -def _pick_evaluation_row_transcript( - source_row: Optional[CallImportRow], - evaluation: Optional[CallImportEvaluation] = None, -) -> Optional[str]: - """Transcript shown in evaluation row detail for the run's source.""" - if source_row is None: - return None - source = ( - (evaluation.transcript_source or "diarised").strip().lower() - if evaluation is not None - else "diarised" - ) - if source == "production": - raw = (source_row.transcript or "").strip() - return raw or None - diarised = (source_row.diarised_transcript or "").strip() - if diarised: - return diarised - raw = (source_row.transcript or "").strip() - return raw or None - - -def _to_evaluation_row_response( - eval_row_obj: CallImportEvaluationRow, - source_row: Optional[CallImportRow], - evaluation: Optional[CallImportEvaluation] = None, -) -> CallImportEvaluationRowResponse: - """Serialize one evaluation row plus joined source-row metadata.""" - return CallImportEvaluationRowResponse( - id=eval_row_obj.id, - evaluation_id=eval_row_obj.evaluation_id, - call_import_row_id=eval_row_obj.call_import_row_id, - row_index=source_row.row_index if source_row else None, - conversation_id=source_row.conversation_id if source_row else None, - transcript=_pick_evaluation_row_transcript(source_row, evaluation), - raw_columns=source_row.raw_columns if source_row else None, - recording_url=source_row.recording_url if source_row else None, - recording_date=source_row.recording_date if source_row else None, - recording_s3_key=source_row.recording_s3_key if source_row else None, - diarised_transcript_status=( - source_row.diarised_transcript_status if source_row else None - ), - diarised_transcript_error=( - source_row.diarised_transcript_error if source_row else None - ), - status=eval_row_obj.status, - metric_scores=eval_row_obj.metric_scores or {}, - error_message=eval_row_obj.error_message, - started_at=eval_row_obj.started_at, - finished_at=eval_row_obj.finished_at, - created_at=eval_row_obj.created_at, - updated_at=eval_row_obj.updated_at, - ) - - -def _serialize_selected_metric_ids(value) -> List[UUID]: - result: List[UUID] = [] - if not isinstance(value, list): - return result - for item in value: - try: - result.append(UUID(str(item))) - except (TypeError, ValueError): - continue - return result - - -def _metrics_for_ids(db: Session, org_id: UUID, ids: List[UUID]) -> List[Metric]: - if not ids: - return [] - rows = ( - db.query(Metric) - .filter( - Metric.organization_id == org_id, - Metric.id.in_(ids), - ) - .all() - ) - by_id = {row.id: row for row in rows} - return [by_id[mid] for mid in ids if mid in by_id] - - -def _expand_metric_selection( - db: Session, - org_id: UUID, - selected_ids: List[UUID], -) -> Tuple[List[Metric], Dict[UUID, List[Metric]]]: - """Resolve user-supplied metric ids into actual leaves + parent grouping. - - Rules: - * If a parent id is in ``selected_ids`` and no specific children of - that parent are also listed, include EVERY enabled child of that - parent. - * If a parent id AND some of its children are listed, include only - the listed children (treat the parent selection as the - "container" so users can deselect labels). - * Standalone metrics (no parent, no children) pass through - unchanged. - * Disabled metrics are filtered out at this layer so the caller - doesn't have to repeat the check. - - Returns: - (effective_metrics, parent_to_children) - - ``effective_metrics`` is the deduplicated list of metrics the - worker will actually score (children + standalone). Order is - preserved from ``selected_ids`` for display stability. - - ``parent_to_children`` maps each parent metric id (UUID) to the - list of its selected children. Useful for grouping in the LLM - prompt builder. - """ - if not selected_ids: - return [], {} - - requested = list(selected_ids) - initial_rows = ( - db.query(Metric) - .filter( - Metric.organization_id == org_id, - Metric.id.in_(requested), - ) - .all() - ) - initial_by_id = {row.id: row for row in initial_rows} - - parent_ids_requested = { - m.id for m in initial_rows if m.selection_mode and not m.parent_metric_id - } - # Map parent id -> children explicitly requested by the user. - explicit_children_by_parent: Dict[UUID, List[Metric]] = {} - for m in initial_rows: - if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: - explicit_children_by_parent.setdefault( - m.parent_metric_id, [] - ).append(m) - - # For parents without explicit children, hydrate every enabled child. - parents_needing_full_expansion = [ - pid - for pid in parent_ids_requested - if pid not in explicit_children_by_parent - ] - auto_expanded_children: Dict[UUID, List[Metric]] = {} - if parents_needing_full_expansion: - for pid in parents_needing_full_expansion: - child_rows = ( - db.query(Metric) - .filter( - Metric.organization_id == org_id, - Metric.parent_metric_id == pid, - Metric.enabled.is_(True), - ) - .order_by(Metric.created_at.asc()) - .all() - ) - auto_expanded_children[pid] = child_rows - - parent_to_children: Dict[UUID, List[Metric]] = {} - for pid in parent_ids_requested: - children = explicit_children_by_parent.get( - pid - ) or auto_expanded_children.get(pid, []) - # Drop disabled children so the worker doesn't waste a slot on - # them. Empty parents (no enabled children) are still tracked - # because the UI may want to show "0 of 0" rather than swallow - # them silently. - parent_to_children[pid] = [c for c in children if c.enabled] - - effective: List[Metric] = [] - seen: set[UUID] = set() - for mid in requested: - m = initial_by_id.get(mid) - if m is None: - continue - if m.selection_mode and not m.parent_metric_id: - # Parent row itself is not scored — only its children. - for child in parent_to_children.get(m.id, []): - if child.id in seen or not child.enabled: - continue - seen.add(child.id) - effective.append(child) - continue - if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: - # Already accounted for via the parent expansion above. - continue - if not m.enabled: - continue - if m.id in seen: - continue - seen.add(m.id) - effective.append(m) - - return effective, parent_to_children - - -def _evaluation_bulk_operation_for_response( - evaluation_id: UUID, -) -> Optional[str]: - from app.services.call_imports.evaluation_bulk_op import ( - get_evaluation_bulk_operation, - ) - - return get_evaluation_bulk_operation(evaluation_id) - - -def _serialize_eval( - db: Session, - row: CallImportEvaluation, - *, - sibling_evaluation_ids: Optional[List[UUID]] = None, -) -> CallImportEvaluationResponse: - selected_ids = _serialize_selected_metric_ids(row.selected_metric_ids) - - # Pull every metric referenced anywhere in the run's grouping (leaves, - # standalone, AND parents from selected_metric_groups) so the UI can - # render parent labels even when only children were materialized into - # selected_metric_ids. - groups_raw: Dict[str, List[str]] = {} - if isinstance(row.selected_metric_groups, dict): - for parent_str, children in row.selected_metric_groups.items(): - if not isinstance(children, list): - continue - cleaned: List[str] = [] - for c in children: - try: - UUID(str(c)) - cleaned.append(str(c)) - except (TypeError, ValueError): - continue - try: - UUID(parent_str) - groups_raw[parent_str] = cleaned - except (TypeError, ValueError): - continue - - metric_ids_for_lookup: List[UUID] = list(selected_ids) - for parent_str in groups_raw.keys(): - try: - pid = UUID(parent_str) - if pid not in metric_ids_for_lookup: - metric_ids_for_lookup.append(pid) - except (TypeError, ValueError): - continue - - metrics = _metrics_for_ids( - db, row.organization_id, metric_ids_for_lookup - ) - - from app.services.call_imports.progress_counters import merge_eval_counters_for_ui - - ui_completed_raw, ui_failed_raw = merge_eval_counters_for_ui(row) - total = int(row.total_rows or 0) - ui_completed = ( - min(ui_completed_raw, total) if total else ui_completed_raw - ) - ui_failed = min(ui_failed_raw, total) if total else ui_failed_raw - - return CallImportEvaluationResponse( - id=row.id, - call_import_id=row.call_import_id, - organization_id=row.organization_id, - name=row.name, - selected_metric_ids=selected_ids, - selected_metric_groups=groups_raw or None, - metrics=[ - CallImportMetricSummary( - id=metric.id, - name=metric.name, - metric_type=metric.metric_type, - description=metric.description, - parent_metric_id=metric.parent_metric_id, - selection_mode=metric.selection_mode, - # Required by the Flow tab to know whether a parent - # opted into discovery; without it the - # DiscoveredLabelsPanel stays hidden even when the - # worker is actively producing discovered_labels. - allow_discovery=bool( - getattr(metric, "allow_discovery", False) - ), - ) - for metric in metrics - ], - status=row.status, - total_rows=row.total_rows, - completed_rows=ui_completed, - failed_rows=ui_failed, - error_message=row.error_message, - llm_provider=row.llm_provider, - llm_model=row.llm_model, - llm_credential_id=row.llm_credential_id, - llm_config=( - row.llm_config if isinstance(getattr(row, "llm_config", None), dict) else None - ), - metric_llm_overrides=( - row.metric_llm_overrides - if isinstance(row.metric_llm_overrides, dict) - else None - ), - stt_provider=row.stt_provider, - stt_model=row.stt_model, - stt_credential_id=row.stt_credential_id, - diarisation_llm_provider=getattr(row, "diarisation_llm_provider", None), - diarisation_llm_model=getattr(row, "diarisation_llm_model", None), - diarisation_llm_credential_id=getattr( - row, "diarisation_llm_credential_id", None - ), - diarisation_prompt=getattr(row, "diarisation_prompt", None), - transcribe_mode=( - (getattr(row, "transcribe_mode", None) or "stt_llm") - ), - transcript_source=(row.transcript_source or "diarised"), - sibling_evaluation_ids=list(sibling_evaluation_ids or []), - started_at=row.started_at, - finished_at=row.finished_at, - created_at=row.created_at, - updated_at=row.updated_at, - tldr_summary=_tldr_summary_payload(row), - user_insights=_user_insights_payload(row), - metric_clusters=_metric_clusters_payload(row), - discover_new_metrics=bool( - getattr(row, "discover_new_metrics", False) - ), - bulk_operation=_evaluation_bulk_operation_for_response(row.id), - ) - - -def _normalize_name(value: Optional[str]) -> Optional[str]: - """Trim user-supplied name; empty string becomes ``NULL``.""" - if value is None: - return None - trimmed = value.strip() - return trimmed or None - - -def _rollup_evaluation_status(evaluation: CallImportEvaluation, db: Session) -> None: - """Recompute counters + terminal status after rows are added/removed. - - Uses a single aggregate query instead of loading every row status. - """ - from app.workers.tasks.evaluate_call_import_row_core import ( - _apply_parent_status_from_counters, - reconcile_evaluation_counters, - ) - - reconcile_evaluation_counters(db, evaluation) - _apply_parent_status_from_counters(evaluation) - db.flush() - - if evaluation.status in {"completed", "failed", "partial"}: - from app.models.database import CallImport - from app.services.call_imports.bulk_ops import rollup_call_import_batch_status - - call_import = ( - db.query(CallImport) - .filter(CallImport.id == evaluation.call_import_id) - .first() - ) - if call_import is not None: - rollup_call_import_batch_status(db, call_import) - - -@router.post( - "", - response_model=CallImportEvaluationResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="createCallImportEvaluation", -) -async def create_call_import_evaluation( - call_import_id: UUID, - payload: CallImportEvaluationCreate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationResponse: - del api_key - call_import = _require_import(db, call_import_id, organization_id) - - metric_ids = payload.metric_ids - if not metric_ids: - raise HTTPException( - status_code=400, - detail="Select at least one metric to run the evaluation against.", - ) - - org_metrics = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.id.in_(metric_ids), - ) - .all() - ) - by_id = {metric.id: metric for metric in org_metrics} - unknown_ids = [mid for mid in metric_ids if mid not in by_id] - if unknown_ids: - raise HTTPException( - status_code=400, - detail=( - "These metric ids do not exist in your organization: " - f"{', '.join(str(mid) for mid in unknown_ids)}. " - "Refresh the metrics list and try again." - ), - ) - # Parents themselves are containers, not scored rows, so a disabled - # parent shouldn't block the run as long as it has enabled children. - # We only reject disabled rows that the worker will actually try to - # evaluate (children + standalone leaves). - disabled_leaves = [ - metric - for metric in org_metrics - if not metric.enabled - and not (metric.selection_mode and not metric.parent_metric_id) - ] - if disabled_leaves: - names = ", ".join(metric.name for metric in disabled_leaves) - raise HTTPException( - status_code=400, - detail=( - f"These metrics are disabled and cannot be evaluated: {names}. " - "Enable them on the Metrics page (or pick different ones) and " - "try again." - ), - ) - - # Expand hierarchical selection: parents auto-include their enabled - # children, mixed parent+child selections respect the user's subset. - effective_metrics, parent_to_children = _expand_metric_selection( - db, organization_id, metric_ids - ) - if not effective_metrics: - raise HTTPException( - status_code=400, - detail=( - "None of the selected metrics yielded an enabled leaf to " - "evaluate. Check that parent categories have enabled " - "children, then try again." - ), - ) - - # The effective list (children + standalone leaves) is what gets - # persisted to ``selected_metric_ids`` and scored by the worker. - # The original parents are preserved in ``selected_metric_groups`` - # so the UI can rebuild the tree later. - leaf_metric_ids: List[UUID] = [m.id for m in effective_metrics] - selected_metric_groups: Dict[str, List[str]] = { - str(pid): [str(c.id) for c in children] - for pid, children in parent_to_children.items() - } - metric_rows = effective_metrics - valid_metric_id_strs = {str(m.id) for m in metric_rows} - - # ----- Validate run-level + per-metric LLM config ----- - llm_provider_norm: Optional[str] = None - llm_model_norm: Optional[str] = None - if payload.llm_provider or payload.llm_model: - if not (payload.llm_provider and payload.llm_model): - raise HTTPException( - status_code=400, - detail="Both llm_provider and llm_model are required when overriding the run LLM.", - ) - try: - llm_provider_norm = ModelProvider( - payload.llm_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Unknown LLM provider '{payload.llm_provider}'. " - "Valid keys are documented in ModelProvider." - ), - ) - llm_model_norm = payload.llm_model.strip() or None - if not llm_model_norm: - raise HTTPException( - status_code=400, detail="llm_model cannot be empty." - ) - - if payload.llm_credential_id is not None: - cred = ( - db.query(AIProvider) - .filter( - AIProvider.id == payload.llm_credential_id, - AIProvider.organization_id == organization_id, - ) - .first() - ) - if not cred: - raise HTTPException( - status_code=400, - detail=( - "The provided llm_credential_id does not exist in this " - "organization." - ), - ) - - # Per-metric overrides: keys can be either a leaf metric id (applies - # to that metric only) or a parent metric id (applies to every - # child of that parent). Parent keys are expanded to their - # children so the worker only sees concrete leaf ids. - metric_overrides_payload: Optional[Dict[str, Dict[str, Any]]] = None - if payload.metric_llm_overrides: - metric_overrides_payload = {} - for metric_id, override in payload.metric_llm_overrides.items(): - target_leaf_ids: List[str] = [] - if metric_id in valid_metric_id_strs: - target_leaf_ids = [metric_id] - else: - # Maybe it's a parent id — expand to the children that - # are part of THIS run. - try: - parent_uuid = UUID(metric_id) - except (TypeError, ValueError): - raise HTTPException( - status_code=400, - detail=( - "metric_llm_overrides references metric " - f"{metric_id} which is not a valid UUID." - ), - ) - children_for_parent = parent_to_children.get(parent_uuid) - if not children_for_parent: - raise HTTPException( - status_code=400, - detail=( - "metric_llm_overrides references metric " - f"{metric_id} which is not in metric_ids." - ), - ) - target_leaf_ids = [str(c.id) for c in children_for_parent] - - override_dict: Dict[str, Any] = {} - if override.provider is not None: - if not override.model: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} has a provider " - "but no model." - ), - ) - try: - override_dict["provider"] = ModelProvider( - override.provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} uses unknown " - f"provider '{override.provider}'." - ), - ) - override_dict["model"] = override.model.strip() - elif override.model: - # Model without provider doesn't make sense — treat as 400 - # so the UI can fix it instead of silently falling back. - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} has a model but " - "no provider." - ), - ) - if override.credential_id is not None: - override_dict["credential_id"] = str(override.credential_id) - if override.llm_config is not None: - override_dict["llm_config"] = override.llm_config - if override_dict: - for leaf_id in target_leaf_ids: - metric_overrides_payload[leaf_id] = override_dict - - # ----- Validate auto-transcribe settings ----- - # Diarised runs auto-diarise rows missing a diarised transcript and - # require STT + diariser LLM config. Production runs score the CSV - # transcript directly and skip diarisation entirely. - use_diarised = payload.transcript_sources[0] == "diarised" - auto_transcribe = use_diarised - - transcribe_mode_norm: Optional[str] = None - stt_provider_norm: Optional[str] = None - stt_model_norm: Optional[str] = None - diarisation_llm_provider_norm: Optional[str] = None - diarisation_llm_model_norm: Optional[str] = None - diarisation_prompt_norm: Optional[str] = None - - if use_diarised: - transcribe_mode_norm = (payload.transcribe_mode or "stt_llm").strip().lower() - if transcribe_mode_norm not in {"stt_llm", "llm_only"}: - raise HTTPException( - status_code=400, - detail=( - f"Unknown transcribe_mode '{payload.transcribe_mode}'. " - "Expected 'stt_llm' or 'llm_only'." - ), - ) - - if transcribe_mode_norm == "stt_llm": - if not payload.stt_provider: - raise HTTPException( - status_code=400, - detail=( - "stt_provider is required when " - "transcribe_mode='stt_llm': every evaluation run " - "auto-diarises rows that are missing a diarised " - "transcript." - ), - ) - if not payload.stt_model: - raise HTTPException( - status_code=400, - detail=( - "stt_model is required when transcribe_mode='stt_llm'." - ), - ) - try: - stt_provider_norm = ModelProvider( - payload.stt_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=f"Unknown STT provider '{payload.stt_provider}'.", - ) - stt_model_norm = payload.stt_model.strip() or None - if not stt_model_norm: - raise HTTPException( - status_code=400, detail="stt_model cannot be empty." - ) - else: - # llm_only — explicitly reject lingering STT inputs so the - # contract is unambiguous (the worker would ignore them but - # silent acceptance hides accidental misconfiguration). - if (payload.stt_provider or "").strip() or ( - payload.stt_model or "" - ).strip(): - raise HTTPException( - status_code=400, - detail=( - "stt_provider / stt_model must be omitted when " - "transcribe_mode='llm_only'; the LLM consumes the " - "audio directly." - ), - ) - - # --- Validate LLM diariser settings ----- - if not payload.diarization_llm_provider: - raise HTTPException( - status_code=400, - detail=( - "diarization_llm_provider is required: every evaluation " - "run diarises STT output with an LLM." - ), - ) - if not payload.diarization_llm_model: - raise HTTPException( - status_code=400, - detail=( - "diarization_llm_model is required: every evaluation " - "run diarises STT output with an LLM." - ), - ) - try: - diarisation_llm_provider_norm = ModelProvider( - payload.diarization_llm_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Unknown diarisation LLM provider " - f"'{payload.diarization_llm_provider}'." - ), - ) - diarisation_llm_model_norm = ( - payload.diarization_llm_model.strip() or None - ) - if not diarisation_llm_model_norm: - raise HTTPException( - status_code=400, - detail="diarization_llm_model cannot be empty.", - ) - diarisation_prompt_norm = ( - payload.diarization_prompt.strip() - if isinstance(payload.diarization_prompt, str) - else None - ) or None - - from app.models.enums import CallImportParameterType, CallImportStatus - from app.services.call_imports.bulk_ops import ( - count_all_source_rows, - count_completed_source_rows, - count_source_rows_with_production_transcript, - ) - - starting_from_mapped = False - if call_import.status == CallImportStatus.MAPPED: - if not call_import.source_s3_key or not call_import.source_format: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This batch has no staged source file. Upload and map " - "a CSV/Excel file before running evaluation." - ), - ) - if not call_import.schema_id: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="Cannot run evaluation without a mapped schema.", - ) - from app.api.v1.routes.call_imports import ( - _ensure_blob_storage_enabled, - _resolve_schema, - _resolve_telephony_integration, - _validate_direct_url_import_ready, - ) - - workspace_id = call_import.workspace_id - schema = _resolve_schema( - db, organization_id, workspace_id, call_import.schema_id - ) - parameters = list(schema.parameters) - if not use_diarised: - transcript_mapped = any( - param.type == CallImportParameterType.TRANSCRIPT - and (call_import.parameter_mapping or {}).get(param.name) - for param in parameters - ) - if not transcript_mapped: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "No transcript column is mapped in this batch. " - "Map a schema transcript parameter to a CSV column, " - "or choose 'Diarize then evaluate'." - ), - ) - if payload.telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, - organization_id, - payload.telephony_integration_id, - payload.provider or "", - ) - else: - _validate_direct_url_import_ready( - parameters, dict(call_import.parameter_mapping or {}) - ) - integration = None - - _ensure_blob_storage_enabled() - - if integration is not None: - call_import.provider = integration.provider - call_import.telephony_integration_id = integration.id - else: - call_import.provider = None - call_import.telephony_integration_id = None - - call_import.total_rows = 0 - call_import.completed_rows = 0 - call_import.failed_rows = 0 - call_import.error_message = None - call_import.status = CallImportStatus.PROCESSING - db.commit() - db.refresh(call_import) - starting_from_mapped = True - - if use_diarised: - total_row_count = count_completed_source_rows(db, call_import.id) - else: - # Production runs score CSV text — rows need not wait for - # recording fetch to finish before they are evaluable. - total_row_count = count_source_rows_with_production_transcript( - db, call_import.id - ) - - requested_sources: List[str] = list(payload.transcript_sources) - - if ( - not use_diarised - and not starting_from_mapped - and count_all_source_rows(db, call_import.id) > 0 - and total_row_count == 0 - ): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "No rows have a production transcript. " - "Choose 'Diarize then evaluate' or import rows with " - "a transcript column." - ), - ) - - base_name = _normalize_name(payload.name) - - def _name_for_source(source: str) -> Optional[str]: - # Single-source runs preserve the user's chosen name verbatim. - del source - return base_name - - created_evaluations: List[CallImportEvaluation] = [] - - for source in requested_sources: - evaluation = CallImportEvaluation( - call_import_id=call_import.id, - organization_id=organization_id, - # Mirror the parent CallImport's workspace so listings can - # filter on workspace_id directly without joining. - workspace_id=call_import.workspace_id, - name=_name_for_source(source), - selected_metric_ids=[ - str(metric_id) for metric_id in leaf_metric_ids - ], - selected_metric_groups=selected_metric_groups or None, - status="pending", - total_rows=total_row_count, - completed_rows=0, - failed_rows=0, - llm_provider=llm_provider_norm, - llm_model=llm_model_norm, - llm_credential_id=payload.llm_credential_id, - llm_config=payload.llm_config, - metric_llm_overrides=metric_overrides_payload, - stt_provider=stt_provider_norm, - stt_model=stt_model_norm, - stt_credential_id=( - payload.stt_credential_id if auto_transcribe else None - ), - diarisation_llm_provider=diarisation_llm_provider_norm, - diarisation_llm_model=diarisation_llm_model_norm, - diarisation_llm_credential_id=( - payload.diarization_llm_credential_id if auto_transcribe else None - ), - diarisation_prompt=diarisation_prompt_norm, - transcribe_mode=transcribe_mode_norm, - transcript_source=source, - discover_new_metrics=bool( - getattr(payload, "discover_new_metrics", False) - ), - ) - db.add(evaluation) - db.flush() - created_evaluations.append(evaluation) - - db.commit() - for evaluation in created_evaluations: - db.refresh(evaluation) - - primary_evaluation = created_evaluations[0] - sibling_ids = [e.id for e in created_evaluations[1:]] - - if not total_row_count and not starting_from_mapped: - for evaluation in created_evaluations: - evaluation.status = "completed" - db.commit() - for evaluation in created_evaluations: - db.refresh(evaluation) - return _serialize_eval( - db, primary_evaluation, sibling_evaluation_ids=sibling_ids - ) - - if starting_from_mapped: - from app.workers.tasks.call_import_bulk_ops import ( - materialize_mapped_call_import_evaluation_task, - ) - - for evaluation in created_evaluations: - materialize_mapped_call_import_evaluation_task.delay( - str(call_import.id), - str(organization_id), - str(call_import.workspace_id), - str(evaluation.id), - transcribe_overwrite=payload.transcribe_overwrite, - ) - else: - from app.workers.tasks.call_import_bulk_ops import ( - materialize_call_import_evaluation_task, - ) - - for evaluation in created_evaluations: - materialize_call_import_evaluation_task.delay( - str(evaluation.id), - transcribe_overwrite=payload.transcribe_overwrite, - ) - - for evaluation in created_evaluations: - db.refresh(evaluation) - - return _serialize_eval( - db, primary_evaluation, sibling_evaluation_ids=sibling_ids - ) - - -@router.get( - "", - response_model=CallImportEvaluationListResponse, - operation_id="listCallImportEvaluations", -) -async def list_call_import_evaluations( - call_import_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationListResponse: - del api_key - _require_import(db, call_import_id, organization_id) - rows = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .order_by(desc(CallImportEvaluation.created_at)) - .all() - ) - return CallImportEvaluationListResponse( - items=[_serialize_eval(db, row) for row in rows], - total=len(rows), - ) - - -@router.get( - "/{eval_id}", - response_model=CallImportEvaluationResponse, - operation_id="getCallImportEvaluation", -) -async def get_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationResponse: - del api_key - _require_import(db, call_import_id, organization_id) - row = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not row: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - return _serialize_eval(db, row) - - -@router.get( - "/{eval_id}/rows", - response_model=CallImportEvaluationRowListResponse, - operation_id="listCallImportEvaluationRows", -) -async def list_call_import_evaluation_rows( - call_import_id: UUID, - eval_id: UUID, - page: int = Query(1, ge=1), - page_size: int = Query(100, ge=1, le=500), - q: Optional[str] = Query( - None, - description=( - "Free-text search across conversation_id and transcript " - "(case-insensitive substring match)." - ), - ), - metric_id: Optional[UUID] = Query( - None, - description=( - "If set, only return rows whose ``metric_scores[metric_id].value`` " - "exactly matches ``metric_value`` (string-compared). " - "Use together with ``metric_value``." - ), - ), - metric_value: Optional[str] = Query( - None, - description="Value to match against metric_id (string compare).", - ), - status_filter: Optional[str] = Query( - None, - alias="status", - description="Restrict to rows with this evaluation row status.", - ), - flow_parent_id: Optional[UUID] = Query( - None, - description=( - "Parent (category) metric whose ``sequence`` array should be " - "checked against ``flow_node`` and ``flow_edge_target``. Used " - "to drill into the calls behind a flow-chart node or edge." - ), - ), - flow_node: Optional[str] = Query( - None, - description=( - "If set together with ``flow_parent_id``, only return rows " - "whose sequence under that parent contains this step. Accepts " - "either a child metric UUID (resolved to slug(name)), a " - "``disc:`` discovered-label id, or a raw slug." - ), - ), - flow_edge_target: Optional[str] = Query( - None, - description=( - "Optional companion to ``flow_node``: when set, restrict to " - "rows whose sequence contains the directed transition " - "``flow_node -> flow_edge_target`` (immediately adjacent). " - "Same id format as ``flow_node``." - ), - ), - discovered_parent_id: Optional[UUID] = Query( - None, - description=( - "Parent (category) metric that defines the discovery scope " - "for ``discovered_label_key`` / ``has_discovered``." - ), - ), - discovered_label_key: Optional[str] = Query( - None, - description=( - "If set together with ``discovered_parent_id``, only return " - "rows whose ``metric_scores[parent].discovered_labels`` " - "list contains an entry with this slug (after applying " - "evaluation-level merge aliases)." - ), - ), - has_discovered: Optional[bool] = Query( - None, - description=( - "If true together with ``discovered_parent_id``, only return " - "rows that have at least one LLM-discovered label for the " - "parent. Useful to triage which calls produced novel labels." - ), - ), - sort_by: Optional[str] = Query( - None, - description=( - "Column to sort by. Accepted values: ``row_index`` (default " - "when omitted), ``conversation_id``, ``status`` (the " - "evaluation-row status), or ``metric:`` to sort " - "by ``metric_scores[].value``. Metric sorts compare " - "the extracted JSON text — adequate for booleans, enum " - "labels, and 0-1 ratings; large integer values may sort " - "lexicographically (10 before 2)." - ), - ), - sort_dir: Optional[str] = Query( - "asc", - description="Sort direction: ``asc`` (default) or ``desc``.", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationRowListResponse: - del api_key - _require_import(db, call_import_id, organization_id) - - eval_row = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not eval_row: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - query = ( - db.query(CallImportEvaluationRow, CallImportRow) - .join(CallImportRow, CallImportRow.id == CallImportEvaluationRow.call_import_row_id) - .filter(CallImportEvaluationRow.evaluation_id == eval_id) - ) - - # --- Filters ---------------------------------------------------------- - if q and q.strip(): - needle = f"%{q.strip()}%" - # Search across both transcript columns so a hit in either the - # production or the diarised version surfaces the row, - # independent of which source the evaluation actually scored. - query = query.filter( - or_( - CallImportRow.conversation_id.ilike(needle), - CallImportRow.transcript.ilike(needle), - CallImportRow.diarised_transcript.ilike(needle), - ) - ) - - if status_filter: - # The CallImportEvaluationRow.status column is a string in PG so a - # plain == filter works; we lowercase to match the stored values. - query = query.filter( - CallImportEvaluationRow.status == status_filter.strip().lower() - ) - - if metric_id is not None and metric_value is not None: - # ``metric_scores`` is a JSONB column shaped like - # ``{"": {"value": , "type": "boolean", ...}}``. We - # extract the nested ``value`` as text and compare to the user - # input as a string — that handles bool/int/enum without needing - # per-type casts. ``metric_value`` is matched case-insensitively - # so chart clicks on labels like "True" survive any casing drift - # between worker output and the chart label. - path_value = func.json_extract_path_text( - CallImportEvaluationRow.metric_scores, - str(metric_id), - "value", - ) - query = query.filter(func.lower(path_value) == metric_value.strip().lower()) - - # --- Flow chart drilldown filter ------------------------------------- - # Translates a clicked node (or edge) on the flow chart into a - # SQL filter against ``metric_scores[].sequence``. The - # frontend sends either a child UUID, a ``disc:`` discovered - # node id, or a raw slug — we normalize all three to the slug that - # actually appears in stored ``sequence`` arrays. - if flow_parent_id is not None and flow_node and flow_node.strip(): - parent_id_str_local = str(flow_parent_id) - alias_map_flow = _alias_map_for_parent(eval_row, flow_parent_id) - - def _flow_node_to_slug(raw: str) -> Optional[str]: - raw_clean = raw.strip() - if not raw_clean: - return None - if raw_clean == _FLOW_START_NODE_ID: - # The synthetic START node isn't a real sequence entry; - # filtering on it is meaningless so we skip silently. - return None - if raw_clean.startswith(_DISCOVERED_NODE_PREFIX): - return _resolve_alias( - alias_map_flow, - _slug_label(raw_clean[len(_DISCOVERED_NODE_PREFIX) :]), - ) - # Try to interpret as a child metric UUID first; fall back - # to treating it as a slug. - try: - child_uuid = UUID(raw_clean) - except (TypeError, ValueError): - return _resolve_alias(alias_map_flow, _slug_label(raw_clean)) - child = ( - db.query(Metric.name) - .filter( - Metric.id == child_uuid, - Metric.organization_id == organization_id, - ) - .first() - ) - if child and child[0]: - return _resolve_alias(alias_map_flow, _slug_label(child[0])) - return _resolve_alias(alias_map_flow, _slug_label(raw_clean)) - - from_slug = _flow_node_to_slug(flow_node) - target_slug: Optional[str] = None - if flow_edge_target and flow_edge_target.strip(): - target_slug = _flow_node_to_slug(flow_edge_target) - - if from_slug: - # The ``metric_scores`` column is declared as ``Column(JSON)`` - # in the model so on databases where the table was created - # from the model (rather than the migration) the physical - # type is ``json``, not ``jsonb``. The JSONB-only operators - # below (``jsonb_exists``, ``jsonb_array_elements_text``, - # ``@>``) require a JSONB input — we cast once up front so - # the same SQL works regardless of which path created the - # table. - scores_jsonb = ( - "(call_import_evaluation_rows.metric_scores)::jsonb" - ) - if target_slug: - # Edge filter: rows whose sequence under this parent - # contains ``from_slug`` immediately followed by - # ``target_slug``. Implemented as a correlated EXISTS - # over ``jsonb_array_elements_text`` with ORDINALITY, - # which is the portable way to express "next array - # index" against a JSONB array in Postgres. - edge_filter_sql = text( - f""" - EXISTS ( - SELECT 1 - FROM jsonb_array_elements_text( - COALESCE( - {scores_jsonb} -> :p_id -> 'sequence', - '[]'::jsonb - ) - ) WITH ORDINALITY AS s1(elem, ord) - JOIN jsonb_array_elements_text( - COALESCE( - {scores_jsonb} -> :p_id -> 'sequence', - '[]'::jsonb - ) - ) WITH ORDINALITY AS s2(elem, ord) - ON s2.ord = s1.ord + 1 - WHERE s1.elem = :from_slug - AND s2.elem = :to_slug - ) - """ - ).bindparams( - p_id=parent_id_str_local, - from_slug=from_slug, - to_slug=target_slug, - ) - query = query.filter(edge_filter_sql) - else: - # Node filter: rows whose ``metric_scores -> parent -> - # 'sequence'`` array contains ``from_slug``. We use the - # function form ``jsonb_exists`` rather than the ``?`` - # operator to avoid psycopg2 mistaking the question - # mark for a parameter placeholder. - node_filter_sql = text( - f""" - jsonb_exists( - COALESCE( - {scores_jsonb} -> :p_id -> 'sequence', - '[]'::jsonb - ), - :slug - ) - """ - ).bindparams(p_id=parent_id_str_local, slug=from_slug) - query = query.filter(node_filter_sql) - - # --- Discovered label filters --------------------------------------- - # Surfaces "which calls produced THIS LLM-discovered label" and the - # broader "which calls produced ANY LLM-discovered label". Both - # operate on ``metric_scores[].discovered_labels`` (a list - # of dicts) plus the same ``sequence`` array — covering both legacy - # rows where the slug only made it into ``sequence`` and newer - # rows where it landed in both. - if discovered_parent_id is not None and ( - discovered_label_key or has_discovered - ): - d_parent_str = str(discovered_parent_id) - alias_map_disc = _alias_map_for_parent(eval_row, discovered_parent_id) - # See note above: cast once so the JSONB operators don't reject - # the column when it's typed as ``json`` in the database. - scores_jsonb = "(call_import_evaluation_rows.metric_scores)::jsonb" - if discovered_label_key and discovered_label_key.strip(): - target = _resolve_alias( - alias_map_disc, _slug_label(discovered_label_key) - ) - if target: - # Match rows whose discovered_labels list has an entry - # ``{"key": }`` OR whose sequence array still - # contains the slug. The latter covers older rows that - # were rewritten by a merge in the discovered_labels - # blob but whose sequence may have lagged. - contains_json = json.dumps( - {d_parent_str: {"discovered_labels": [{"key": target}]}} - ) - disc_filter_sql = text( - f""" - ( - {scores_jsonb} @> CAST(:contains AS JSONB) - OR - jsonb_exists( - COALESCE( - {scores_jsonb} -> :p_id -> 'sequence', - '[]'::jsonb - ), - :slug - ) - ) - """ - ).bindparams( - contains=contains_json, - p_id=d_parent_str, - slug=target, - ) - query = query.filter(disc_filter_sql) - elif has_discovered: - # No specific slug — just rows that surfaced any candidate - # under this parent. We coalesce missing paths to ``[]`` so - # ``jsonb_array_length`` always sees an array (it raises on - # non-array inputs, but our shape guarantees a list when - # the key is present). - has_disc_sql = text( - f""" - jsonb_array_length( - COALESCE( - {scores_jsonb} -> :p_id -> 'discovered_labels', - '[]'::jsonb - ) - ) > 0 - """ - ).bindparams(p_id=d_parent_str) - query = query.filter(has_disc_sql) - - # --- Sorting ---------------------------------------------------------- - # Column-click sorting from the UI. Falls back to ``row_index`` so - # paging stays stable when the user clears the sort. We always add a - # secondary ``row_index`` tiebreaker so duplicate sort keys (e.g. - # many rows with ``status = 'completed'``) keep a deterministic - # order across page boundaries — without this, pagination can - # double-show or skip rows when Postgres picks a different physical - # order on each query. - direction_desc = (sort_dir or "asc").strip().lower() == "desc" - - def _apply_direction(column_expr): - return column_expr.desc() if direction_desc else column_expr.asc() - - # Whether the caller's ``sort_by`` resolved to a known column. We - # use this flag to decide whether ``sort_dir`` is honoured on the - # fallback path: unrecognized columns (typos, stale UI state) fall - # back to the implicit ``row_index ASC`` default and intentionally - # ignore ``sort_dir`` so users don't get a surprise reverse order - # from a typo'd column name. - sort_recognized = False - sort_by_clean = (sort_by or "").strip() - primary_sort = None - metric_uuid: Optional[UUID] = None - if sort_by_clean == "row_index": - sort_recognized = True - # Falls through to the default ``order_by`` below with - # ``primary_sort`` still None — but ``sort_recognized=True`` - # tells the fallback branch to apply the requested direction. - elif sort_by_clean == "conversation_id": - sort_recognized = True - primary_sort = _apply_direction(CallImportRow.conversation_id) - elif sort_by_clean == "status": - sort_recognized = True - primary_sort = _apply_direction(CallImportEvaluationRow.status) - elif sort_by_clean.startswith("metric:"): - raw_metric_id = sort_by_clean.split(":", 1)[1].strip() - try: - metric_uuid = UUID(raw_metric_id) - except (TypeError, ValueError): - metric_uuid = None - if metric_uuid is not None: - sort_recognized = True - # ``metric_scores`` is JSON-typed but the helper functions - # for path extraction differ between Postgres (production) - # and SQLite (default test backend). Branch on the active - # dialect so we can use the right primitive: - # * Postgres → ``json_extract_path_text(col, key, "value")`` - # which returns the value as TEXT for both ``json`` and - # ``jsonb`` columns. - # * SQLite → ``json_extract(col, '$."".value')`` - # using JSONPath syntax. ``metric_uuid`` is already - # validated above (``UUID(raw_metric_id)``), so the - # interpolated path is safe from injection. - # NULL values (rows where the metric wasn't scored) sort - # to the END regardless of direction so un-scored rows - # don't crowd the top of an ascending sort. - dialect_name = ( - db.bind.dialect.name if db.bind is not None else "postgresql" - ) - if dialect_name == "sqlite": - json_path = f'$."{metric_uuid}".value' - path_value = func.json_extract( - CallImportEvaluationRow.metric_scores, - json_path, - ) - else: - path_value = func.json_extract_path_text( - CallImportEvaluationRow.metric_scores, - str(metric_uuid), - "value", - ) - primary_sort = ( - path_value.desc().nullslast() - if direction_desc - else path_value.asc().nullslast() - ) - - if primary_sort is not None: - query = query.order_by(primary_sort, CallImportRow.row_index.asc()) - elif sort_recognized: - # Explicit ``sort_by=row_index`` request — honour direction. - query = query.order_by(_apply_direction(CallImportRow.row_index)) - else: - # No sort requested OR unrecognized column — safe default of - # ``row_index ASC``. We deliberately ignore ``sort_dir`` here - # so a typo'd / stale ``sort_by`` doesn't quietly invert the - # default order. - query = query.order_by(CallImportRow.row_index.asc()) - from app.db_sharding.eval_rows import fetch_evaluation_row_pairs_page - from app.db_sharding.sessions import is_sharding_enabled - - def _pair_row_index( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> int: - return int(pair[1].row_index or 0) - - def _directed_string(value: Optional[str], desc: bool) -> Tuple[int, ...]: - text = value or "" - if not desc: - return (0, *text.encode("utf-8")) - return (1, *(-byte for byte in text.encode("utf-8"))) - - if sort_by_clean == "conversation_id": - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[Any, ...]: - return ( - _directed_string(pair[1].conversation_id, direction_desc), - _pair_row_index(pair), - ) - elif sort_by_clean == "status": - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[Any, ...]: - return ( - _directed_string(pair[0].status, direction_desc), - _pair_row_index(pair), - ) - elif sort_by_clean.startswith("metric:") and metric_uuid is not None: - metric_id_str = str(metric_uuid) - - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[Any, ...]: - scores = pair[0].metric_scores or {} - entry = scores.get(metric_id_str, {}) - raw_value = entry.get("value") if isinstance(entry, dict) else None - null_rank = 1 if raw_value is None else 0 - return ( - null_rank, - _directed_string( - str(raw_value) if raw_value is not None else None, - direction_desc, - ), - _pair_row_index(pair), - ) - elif sort_recognized and sort_by_clean == "row_index": - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[int, ...]: - idx = _pair_row_index(pair) - return (-idx,) if direction_desc else (idx,) - else: - def _pair_sort_key( - pair: Tuple[CallImportEvaluationRow, CallImportRow], - ) -> Tuple[int, ...]: - return (_pair_row_index(pair),) - - if is_sharding_enabled(): - def _build_query(session: Session): - return query.with_session(session) - - total, rows = fetch_evaluation_row_pairs_page( - db, - _build_query, - page=page, - page_size=page_size, - sort_key=_pair_sort_key, - bounded_shard_fetch=( - not sort_recognized or sort_by_clean == "row_index" - ), - ) - else: - total = query.count() - rows = query.offset((page - 1) * page_size).limit(page_size).all() - - # Row detail shows the transcript for this run's chosen source. - items: List[CallImportEvaluationRowResponse] = [ - _to_evaluation_row_response(eval_row_obj, source_row, eval_row) - for eval_row_obj, source_row in rows - ] - - return CallImportEvaluationRowListResponse( - items=items, - total=total, - page=page, - page_size=page_size, - ) - - -@router.get( - "/{eval_id}/export", - operation_id="exportCallImportEvaluationCsv", -) -async def export_call_import_evaluation_csv( - call_import_id: UUID, - eval_id: UUID, - format: Literal["csv", "xlsx"] = Query( - "csv", - description=( - "Output format. ``csv`` returns a UTF-8 BOM CSV; ``xlsx`` " - "returns a native Excel workbook (single sheet)." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> StreamingResponse: - del api_key - call_import = _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - selected_metric_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) - # Include parent metric ids referenced in selected_metric_groups so - # the export shows a parent "Chosen Label" column next to its - # children's true/false columns. - lookup_ids: List[UUID] = list(selected_metric_ids) - groups_raw = ( - evaluation.selected_metric_groups - if isinstance(evaluation.selected_metric_groups, dict) - else {} - ) - for parent_str in groups_raw.keys(): - try: - pid = UUID(parent_str) - if pid not in lookup_ids: - lookup_ids.append(pid) - except (TypeError, ValueError): - continue - metrics = _metrics_for_ids(db, organization_id, lookup_ids) - metric_names = {str(metric.id): metric.name for metric in metrics} - metrics_by_id = {str(metric.id): metric for metric in metrics} - - # Two export-time modes depending on how the batch was uploaded: - # - # * Schema-driven (new): ``call_imports.schema_id`` is set, - # ``parameter_mapping`` records which CSV header fed each - # parameter, and ``raw_columns`` on each row is keyed by - # parameter NAME. Export headers are the parameter names. - # * Legacy (pre-schema): ``column_mapping`` / ``extra_columns`` / - # ``custom_column_mapping`` drive the columns and - # ``raw_columns`` is keyed by the original CSV header. - # - # We bucket entries into ``standard_export_headers`` (raw_columns - # key == export header) and ``custom_export`` (export header - # differs from the raw_columns key) so the row-projection loop - # below stays mode-agnostic. - standard_export_headers: List[str] = [] - custom_export: List[tuple[str, str]] = [] # [(export_header, raw_columns_key)] - - if call_import.schema_id is not None: - # Use the live schema parameter list for column ordering. Falls - # back to whatever's in ``parameter_mapping`` if the schema was - # deleted (defensive - the FK is ON DELETE RESTRICT, but tests - # / future cascades may still hit this branch). - from app.models.database import CallImportSchema as _ImportSchema - - schema_obj = ( - db.query(_ImportSchema) - .filter(_ImportSchema.id == call_import.schema_id) - .first() - ) - if schema_obj is not None: - params_sorted = sorted( - schema_obj.parameters, key=lambda p: p.ordering or 0 - ) - for param in params_sorted: - if param.name and param.name not in standard_export_headers: - standard_export_headers.append(param.name) - else: - for param_name in (call_import.parameter_mapping or {}).keys(): - if param_name and param_name not in standard_export_headers: - standard_export_headers.append(param_name) - else: - mapping = call_import.column_mapping or {} - mapped_headers = [ - mapping.get("external_call_id"), - mapping.get("transcript"), - mapping.get("recording_url"), - ] - for header in [*mapped_headers, *(call_import.extra_columns or [])]: - if ( - isinstance(header, str) - and header - and header not in standard_export_headers - ): - standard_export_headers.append(header) - - custom_mapping = call_import.custom_column_mapping or {} - if isinstance(custom_mapping, dict): - for name, csv_header in custom_mapping.items(): - if not isinstance(name, str) or not isinstance(csv_header, str): - continue - if not name or not csv_header: - continue - if name in standard_export_headers: - continue # would clobber a real column - custom_export.append((name, csv_header)) - - if ( - call_import.source_format == "audio" - and "conversation_id" not in standard_export_headers - ): - standard_export_headers.insert(0, "conversation_id") - - # Build the metric columns: each parent (if any) gets a value column - # and (when capture_rationale=true) a " - LLM Rationale" - # column. The per-child boolean columns are intentionally suppressed - # — categorization metrics now collapse to exactly two columns in - # the export, mirroring the in-app table. - child_ids_in_groups: set[str] = set() - for parent_str, child_strs in groups_raw.items(): - for child_str in child_strs: - if isinstance(child_str, str): - child_ids_in_groups.add(child_str) - - metric_headers: List[str] = [] - rationale_headers: Dict[str, str] = {} # metric_id_str -> rationale column name - seen_metric_ids: set[str] = set() - - def _add_metric_column(metric: Metric) -> None: - mid_str = str(metric.id) - if mid_str in seen_metric_ids: - return - # Skip any child whose parent is part of this run — the parent - # column above already shows the chosen child name as its - # value. - if mid_str in child_ids_in_groups: - return - seen_metric_ids.add(mid_str) - header = metric_names[mid_str] - metric_headers.append(header) - if bool(getattr(metric, "capture_rationale", False)): - rationale_header = f"{header} - LLM Rationale" - metric_headers.append(rationale_header) - rationale_headers[mid_str] = rationale_header - - for parent_str in groups_raw.keys(): - parent = metrics_by_id.get(parent_str) - if parent: - _add_metric_column(parent) - # Children of an in-run parent are deliberately not emitted — - # the ``child_ids_in_groups`` guard inside ``_add_metric_column`` - # is what enforces this. We still iterate the keys above (not - # ``.items()``) so the parent-only emission is explicit. - # Append anything left over (standalone metrics not in any group, or - # legacy runs without ``selected_metric_groups``). - for metric in metrics: - if metric.selection_mode and not metric.parent_metric_id: - continue # already handled above - if str(metric.id) in seen_metric_ids: - continue - _add_metric_column(metric) - - # Three new fixed columns surface the two transcript fields and the - # evaluation's transcript_source as live values pulled from the - # ``CallImportRow`` (not from the frozen ``raw_columns`` snapshot). - # The user can now compare "what was in the CSV" vs "what the - # diarisation worker produced" without round-tripping through the - # UI, and downstream tools can verify which transcript the metrics - # were computed against. - PRODUCTION_TRANSCRIPT_HEADER = "Production Transcript" - DIARISED_TRANSCRIPT_HEADER = "Diarised Transcript" - EVAL_SOURCE_HEADER = "Evaluated Transcript Source" - - fieldnames = [ - *standard_export_headers, - *[h for h, _ in custom_export], - PRODUCTION_TRANSCRIPT_HEADER, - DIARISED_TRANSCRIPT_HEADER, - EVAL_SOURCE_HEADER, - *metric_headers, - ] - - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - rows = sorted( - load_evaluation_row_pairs(db, eval_id), - key=lambda pair: int(pair[1].row_index or 0), - ) - else: - rows = ( - db.query(CallImportEvaluationRow, CallImportRow) - .join( - CallImportRow, - CallImportRow.id == CallImportEvaluationRow.call_import_row_id, - ) - .filter(CallImportEvaluationRow.evaluation_id == eval_id) - .order_by(CallImportRow.row_index.asc()) - .all() - ) - - def _project_rows() -> Iterator[Dict[str, str]]: - for eval_row, source_row in rows: - row_out: Dict[str, str] = {} - raw = ( - source_row.raw_columns - if isinstance(source_row.raw_columns, dict) - else {} - ) - for header in standard_export_headers: - value = raw.get(header) - if value is None and header == "conversation_id": - value = source_row.conversation_id - row_out[header] = "" if value is None else str(value) - for export_header, csv_header in custom_export: - value = raw.get(csv_header) - row_out[export_header] = "" if value is None else str(value) - - # Live transcripts pulled from the row, NOT from raw_columns, - # so re-diarised values are always reflected in the export. - # Both transcript columns are flattened to a single line so the - # spreadsheet cell doesn't balloon vertically — the in-app - # ``TranscriptView`` still has the DB copy with line breaks - # intact for chat-bubble rendering. - row_out[PRODUCTION_TRANSCRIPT_HEADER] = _flatten_transcript( - source_row.transcript - ) - row_out[DIARISED_TRANSCRIPT_HEADER] = _flatten_transcript( - source_row.diarised_transcript - ) - row_out[EVAL_SOURCE_HEADER] = _evaluated_transcript_source_label( - evaluation, - source_row, - ) - - scores = ( - eval_row.metric_scores - if isinstance(eval_row.metric_scores, dict) - else {} - ) - for metric in metrics: - metric_score = ( - scores.get(str(metric.id)) - if isinstance(scores, dict) - else None - ) - value = ( - metric_score.get("value") - if isinstance(metric_score, dict) - else None - ) - # Parent metrics (selection_mode set) render the chosen - # child name for single_choice or the ";"-joined list of - # true child names for multi_label. - if ( - metric.selection_mode - and not metric.parent_metric_id - and isinstance(metric_score, dict) - ): - if metric.selection_mode == "multi_label": - selected = metric_score.get("selected_child_names") - if isinstance(selected, list): - value = ";".join(str(s) for s in selected) - else: - value = ( - metric_score.get("chosen_child_name") - or metric_score.get("value") - ) - row_out[metric.name] = "" if value is None else str(value) - rationale_header = rationale_headers.get(str(metric.id)) - if rationale_header is not None: - rationale = ( - metric_score.get("rationale") - if isinstance(metric_score, dict) - else None - ) - row_out[rationale_header] = ( - "" if rationale is None else str(rationale) - ) - yield row_out - - base_filename = f"call-import-{call_import_id}-evaluation-{eval_id}" - - if format == "xlsx": - # xlsx is unicode-native (Hindi/Devanagari, emoji, etc.) so the - # UTF-8-BOM dance isn't needed here. ``write_only`` mode keeps - # peak memory bounded for large evaluations because openpyxl - # only buffers the current row. - try: - from openpyxl import Workbook # type: ignore - from openpyxl.cell import WriteOnlyCell # type: ignore - from openpyxl.styles import Font # type: ignore - except ImportError as exc: # pragma: no cover - exercised by pyproject lock - raise HTTPException( - status_code=500, - detail=( - "Excel export requires the 'openpyxl' package which is " - "not installed." - ), - ) from exc - - workbook = Workbook(write_only=True) - worksheet = workbook.create_sheet(title="Evaluation") - - bold_font = Font(bold=True) - header_cells = [] - for header in fieldnames: - cell = WriteOnlyCell(worksheet, value=header) - cell.font = bold_font - header_cells.append(cell) - worksheet.append(header_cells) - - for row_dict in _project_rows(): - worksheet.append([row_dict.get(h, "") for h in fieldnames]) - - buffer = io.BytesIO() - workbook.save(buffer) - xlsx_bytes = buffer.getvalue() - filename = f"{base_filename}.xlsx" - return StreamingResponse( - iter([xlsx_bytes]), - media_type=( - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ), - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, - ) - - output = io.StringIO() - writer = csv.DictWriter(output, fieldnames=fieldnames, extrasaction="ignore") - writer.writeheader() - for row_dict in _project_rows(): - writer.writerow(row_dict) - - # Excel on Windows defaults to the system ANSI codepage (Windows-1252) - # when a CSV has no encoding marker, which turns UTF-8 Hindi/Devanagari - # / any non-ASCII text into mojibake (e.g. ``ठीक`` → ``ठीक``). - # A UTF-8 BOM tells Excel to switch to UTF-8 decoding and is silently - # skipped by every other UTF-8-aware reader (pandas, LibreOffice, - # Google Sheets, etc.), so the data round-trips correctly everywhere. - csv_text = output.getvalue() - # ``utf-8-sig`` adds the UTF-8 BOM so Excel on Windows decodes the file - # as UTF-8 instead of the system codepage. We also declare the same - # codec in the Content-Type header so well-behaved HTTP clients (incl. - # ``httpx`` / ``requests`` in our tests) strip the BOM during decode. - csv_bytes = csv_text.encode("utf-8-sig") - filename = f"{base_filename}.csv" - return StreamingResponse( - iter([csv_bytes]), - media_type="text/csv; charset=utf-8-sig", - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, - ) - - -def _report_filename_slug(value: str) -> str: - slug = re.sub(r"[^a-zA-Z0-9]+", "-", value.strip().lower()).strip("-") - return slug or "client" - - -def _report_branding_for_import_workspace( - db: Session, - organization_id: UUID, - workspace_id: UUID, - *, - internal_brand_image_id: Optional[str] = None, - external_brand_image_id: Optional[str] = None, -) -> tuple[dict[str, str] | list[str], Optional[str]]: - workspace = ( - db.query(Workspace) - .filter( - Workspace.id == workspace_id, - Workspace.organization_id == organization_id, - ) - .first() - ) - raw = workspace.report_branding if workspace and isinstance(workspace.report_branding, dict) else {} - images = raw.get("images") if isinstance(raw.get("images"), list) else [] - loaded_images: list[dict[str, str]] = [] - for item in images: - if not isinstance(item, dict) or not item.get("s3_key"): - continue - content_type = str(item.get("content_type") or "image/png") - try: - from app.services.storage.s3_service import s3_service - - image_bytes = s3_service.download_file_by_key(str(item["s3_key"])) - except Exception as exc: # noqa: BLE001 - logger.warning( - "Unable to load report branding image for workspace {}: {}", - workspace_id, - exc, - ) - continue - encoded = base64.b64encode(image_bytes).decode("ascii") - role = str(item.get("role") or "generic") - if role not in {"internal", "external", "generic"}: - role = "generic" - loaded_images.append( - { - "id": str(item.get("id") or ""), - "role": role, - "data_uri": f"data:{content_type};base64,{encoded}", - } - ) - - def _pick(role: str, selected_id: Optional[str]) -> Optional[str]: - if selected_id: - for loaded in loaded_images: - if loaded["id"] == selected_id: - return loaded["data_uri"] - for loaded in loaded_images: - if loaded["role"] == role: - return loaded["data_uri"] - return None - - logo_data_uris: dict[str, str] = {} - internal_uri = _pick("internal", internal_brand_image_id) - external_uri = _pick("external", external_brand_image_id) - if internal_uri: - logo_data_uris["internal"] = internal_uri - if external_uri: - logo_data_uris["external"] = external_uri - if ( - not logo_data_uris - and not internal_brand_image_id - and not external_brand_image_id - ): - # Backward compatibility for workspaces that only had a generic logo - # library before the two-slot report header existed. - generic_uris = [ - loaded["data_uri"] - for loaded in loaded_images - if loaded.get("data_uri") - ] - if generic_uris: - heading = raw.get("heading") if isinstance(raw.get("heading"), str) else None - return generic_uris[:4], heading - heading = raw.get("heading") if isinstance(raw.get("heading"), str) else None - return logo_data_uris, heading - - -def _display_metrics_for_pdf_report( - db: Session, - organization_id: UUID, - evaluation: CallImportEvaluation, -) -> list[Metric]: - selected_metric_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) - lookup_ids: List[UUID] = list(selected_metric_ids) - groups_raw = ( - evaluation.selected_metric_groups - if isinstance(evaluation.selected_metric_groups, dict) - else {} - ) - for parent_str in groups_raw.keys(): - try: - parent_id = UUID(parent_str) - except (TypeError, ValueError): - continue - if parent_id not in lookup_ids: - lookup_ids.append(parent_id) - - metrics = _metrics_for_ids(db, organization_id, lookup_ids) - child_ids_in_groups: set[str] = set() - for child_strs in groups_raw.values(): - if not isinstance(child_strs, list): - continue - child_ids_in_groups.update(str(child_id) for child_id in child_strs) - - metrics_by_id = {str(metric.id): metric for metric in metrics} - display: list[Metric] = [] - seen: set[str] = set() - - for parent_str in groups_raw.keys(): - parent = metrics_by_id.get(str(parent_str)) - if parent and str(parent.id) not in seen: - display.append(parent) - seen.add(str(parent.id)) - - for metric in metrics: - metric_id = str(metric.id) - if metric_id in seen or metric_id in child_ids_in_groups: - continue - if metric.selection_mode and not metric.parent_metric_id: - continue - display.append(metric) - seen.add(metric_id) - - return display - - -def _metrics_for_clustering( - db: Session, - evaluation: CallImportEvaluation, - eval_rows: List[CallImportEvaluationRow], -) -> List[Metric]: - """All enabled quality metrics scored in this run, normalized for clustering. - - Hierarchical children are collapsed to their parent metric so cluster - groups render at the category level (e.g. ``AI reveal``) instead of the - child label level (e.g. ``Yes`` / ``No``). - """ - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - aggregate_metric_ids: List[UUID] = [] - for agg in aggregates: - if (agg.metric_category or "quality") == "user_insight": - continue - try: - aggregate_metric_ids.append(UUID(agg.metric_id)) - except (TypeError, ValueError): - continue - if not aggregate_metric_ids: - return [] - - aggregate_metrics = _metrics_for_ids( - db, evaluation.organization_id, aggregate_metric_ids - ) - by_id = {metric.id: metric for metric in aggregate_metrics} - - normalized_ids: List[UUID] = [] - seen: set[UUID] = set() - for metric_id in aggregate_metric_ids: - metric = by_id.get(metric_id) - target_id = ( - metric.parent_metric_id - if metric is not None and metric.parent_metric_id - else metric_id - ) - if target_id in seen: - continue - seen.add(target_id) - normalized_ids.append(target_id) - - metrics = _metrics_for_ids(db, evaluation.organization_id, normalized_ids) - return [ - metric - for metric in metrics - if getattr(metric, "enabled", True) and not _metric_is_user_insight(metric) - ] - - -def _metric_is_user_insight(metric: Metric) -> bool: - if (getattr(metric, "metric_category", "quality") or "quality") == "user_insight": - return True - text_value = " ".join( - str(part or "").lower() - for part in (getattr(metric, "name", ""), getattr(metric, "description", "")) - ) - normalized = text_value.replace("-", " ").replace("_", " ") - phrases = ( - "call context", - "caller context", - "product identification", - "out of scope", - "identity match", - "user identity", - "caller identity", - "frustration trigger", - "video call offer", - "video call reception", - ) - return any(phrase in normalized for phrase in phrases) - - -def _evaluation_rows_for_period( - db: Session, - evaluation_id: UUID, -) -> list[tuple[CallImportEvaluationRow, CallImportRow]]: - return ( - db.query(CallImportEvaluationRow, CallImportRow) - .join(CallImportRow, CallImportRow.id == CallImportEvaluationRow.call_import_row_id) - .filter(CallImportEvaluationRow.evaluation_id == evaluation_id) - .order_by(CallImportRow.row_index.asc()) - .all() - ) - - -def _baseline_candidate_evaluations( - db: Session, - organization_id: UUID, - workspace_id: UUID, - current_evaluation: CallImportEvaluation, - current_period_start: Optional[date], - *, - limit: int = 20, -) -> list[dict[str, Any]]: - candidates = ( - db.query(CallImportEvaluation, CallImport) - .join(CallImport, CallImport.id == CallImportEvaluation.call_import_id) - .filter( - CallImportEvaluation.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - CallImportEvaluation.id != current_evaluation.id, - CallImportEvaluation.status == "completed", - CallImportEvaluation.completed_rows > 0, - ) - .order_by(desc(CallImportEvaluation.created_at)) - .limit(limit * 3) - .all() - ) - items: list[dict[str, Any]] = [] - for candidate_eval, candidate_import in candidates: - rows = _evaluation_rows_for_period(db, candidate_eval.id) - period_start, period_end, period_label, period_display = _report_period_from_rows(rows) - if current_period_start and period_start and period_start >= current_period_start: - continue - dataset = ( - (candidate_import.dataset or "").strip() - or (candidate_import.original_filename or candidate_import.filename or "").strip() - or "Unknown dataset" - ) - evaluation_name = ( - (candidate_eval.name or "").strip() - or str(candidate_eval.id)[:8] - ) - items.append( - { - "evaluation_id": str(candidate_eval.id), - "name": evaluation_name, - "dataset": dataset, - "period_label": period_label, - "period_start": period_start, - "period_end": period_end, - "period_display": period_display, - "completed_rows": int(candidate_eval.completed_rows or 0), - "created_at": candidate_eval.created_at, - "is_default": False, - } - ) - if len(items) >= limit: - break - items.sort( - key=lambda item: ( - item["period_start"] or date.min, - item["created_at"] or datetime.min.replace(tzinfo=timezone.utc), - ), - reverse=True, - ) - if items: - items[0]["is_default"] = True - return items - - -def _resolve_baseline_evaluation( - db: Session, - organization_id: UUID, - workspace_id: UUID, - current_evaluation: CallImportEvaluation, - current_period_start: Optional[date], - baseline_evaluation_id: Optional[str], -) -> Optional[CallImportEvaluation]: - candidates = _baseline_candidate_evaluations( - db, - organization_id, - workspace_id, - current_evaluation, - current_period_start, - ) - allowed_ids = {item["evaluation_id"] for item in candidates} - if baseline_evaluation_id: - baseline_id = str(baseline_evaluation_id).strip() - if baseline_id not in allowed_ids: - raise HTTPException( - status_code=400, - detail="Selected baseline evaluation is not a valid prior run for this report.", - ) - return ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == UUID(baseline_id), - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not candidates: - return None - default_id = candidates[0]["evaluation_id"] - return ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == UUID(default_id), - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - - -def _benchmark_context_for_evaluation( - db: Session, - baseline_evaluation: Optional[CallImportEvaluation], -) -> Optional[dict[str, str]]: - if baseline_evaluation is None: - return None - baseline_import = ( - db.query(CallImport) - .filter(CallImport.id == baseline_evaluation.call_import_id) - .first() - ) - rows = _evaluation_rows_for_period(db, baseline_evaluation.id) - period_start, _period_end, period_label, _period_display = _report_period_from_rows(rows) - dataset = ( - (baseline_import.dataset or "").strip() - if baseline_import and baseline_import.dataset - else None - ) - filename = ( - (baseline_import.original_filename or baseline_import.filename or "").strip() - if baseline_import - else None - ) - evaluation_label = ( - (baseline_evaluation.name or "").strip() - if baseline_evaluation.name - else str(baseline_evaluation.id)[:8] - ) - period = period_label or ( - period_start.isoformat() if period_start else "previous report" - ) - return { - "dataset": dataset or filename or "Unknown dataset", - "evaluation": evaluation_label, - "evaluation_id": str(baseline_evaluation.id), - "period": period, - } - - -def _period_deltas_from_evaluation( - db: Session, - baseline_evaluation: CallImportEvaluation, - current_metric_aggregates: list[dict[str, Any]], - current_evaluation: CallImportEvaluation, - current_eval_rows: List[CallImportEvaluationRow], -) -> dict[str, dict[str, str]]: - baseline_rows = _evaluation_rows_for_period(db, baseline_evaluation.id) - baseline_eval_rows = [eval_row for eval_row, _source_row in baseline_rows] - baseline_aggregate_models = _compute_metric_aggregates( - db, - baseline_evaluation, - baseline_eval_rows, - ) - baseline_metric_aggregates = [ - _aggregate_to_dict(aggregate) for aggregate in baseline_aggregate_models - ] - _metrics, _aggs, policies, _source, _child_map = _clustering_context( - db, current_evaluation, current_eval_rows - ) - metric_by_id = {str(m.id): m for m in _metrics} - current_by_id = { - str(item.get("metric_id")): item for item in current_metric_aggregates - } - previous_by_id = { - str(item.get("metric_id")): item - for item in baseline_metric_aggregates - if isinstance(item, dict) - } - deltas: dict[str, dict[str, str]] = {} - for metric_id, current in current_by_id.items(): - metric = metric_by_id.get(metric_id) - policy = policies.get(metric_id) - previous_raw = previous_by_id.get(metric_id) - if metric is None or policy is None: - deltas[metric_id] = { - "label": "No previous-week baseline", - "detail": "No comparable prior report snapshot was found.", - } - continue - current_pct = failure_rate_percent_from_rows( - current_eval_rows, metric, policy - ) - previous_pct = failure_rate_percent_from_rows( - baseline_eval_rows, metric, policy - ) - if current_pct is None or previous_pct is None: - current_pct = current_pct or _aggregate_primary_percent(current, policy) - previous_pct = ( - previous_pct or _aggregate_primary_percent(previous_raw, policy) - if previous_raw - else None - ) - if current_pct is None or previous_pct is None: - deltas[metric_id] = { - "label": "No previous-week baseline", - "detail": "No comparable prior report snapshot was found.", - } - continue - delta = current_pct - previous_pct - sign = "+" if delta >= 0 else "" - deltas[metric_id] = { - "label": f"{sign}{delta:.1f} pp", - "detail": ( - f"Current report {current_pct:.1f}% vs previous report " - f"{previous_pct:.1f}%" - ), - } - return deltas - - -_DELTA_EXPLANATION_SYSTEM_PROMPT = ( - "You are a senior conversation-analytics reviewer. You will receive " - "week-over-week metric failure-rate deltas plus reconciled failure " - "cluster context per metric.\n\n" - "Return STRICT JSON only:\n" - "{\n" - ' "explanations": {"": "<1-2 sentence explanation of why the delta likely occurred>"}\n' - "}\n\n" - "Constraints:\n" - "- Only include metrics supplied in the prompt.\n" - "- Cluster labels are generated independently each run and are NOT stable " - "IDs. Never compare an unmatched current label to 0% baseline.\n" - "- Use matched_theme_shifts for label-aligned comparisons, " - "gap_label_shifts for structural shifts, and new_themes_current_period " - "for themes that emerged without a baseline match.\n" - "- If reconciliation is uncertain, explain using the numeric delta and " - "gap_label_shifts only.\n" - "- Keep each explanation to 1-2 short sentences (~220 chars).\n" - "- Vendor-safe, factual language; no markdown." -) - - -def _period_delta_explanation_cache_key( - baseline_evaluation_id: UUID, - *, - completed_rows: int, - baseline_completed_rows: int, -) -> str: - return ( - f"{baseline_evaluation_id}:{completed_rows}:" - f"{baseline_completed_rows}:reconciled-v2" - ) - - -def _normalize_cluster_label(label: str) -> str: - return re.sub(r"[^a-z0-9]+", " ", (label or "").lower()).strip() - - -_CLUSTER_LABEL_STOPWORDS = frozenset( - { - "a", - "an", - "the", - "and", - "or", - "during", - "while", - "with", - "for", - "from", - "into", - "general", - "user", - "bot", - "agent", - } -) - - -def _cluster_label_tokens(label: str) -> set[str]: - return { - token - for token in _normalize_cluster_label(label).split() - if token and token not in _CLUSTER_LABEL_STOPWORDS and len(token) > 2 - } - - -def _cluster_label_similarity(left: str, right: str) -> float: - tokens_left = _cluster_label_tokens(left) - tokens_right = _cluster_label_tokens(right) - if not tokens_left or not tokens_right: - return 0.0 - intersection = tokens_left & tokens_right - if not intersection: - return 0.0 - union = tokens_left | tokens_right - jaccard = len(intersection) / len(union) - smaller = tokens_left if len(tokens_left) <= len(tokens_right) else tokens_right - overlap_ratio = len(intersection) / len(smaller) - return max(jaccard, overlap_ratio * 0.85) - - -def _group_clusters_by_gap_label( - clusters: list[dict[str, Any]], -) -> dict[str, list[dict[str, Any]]]: - grouped: dict[str, list[dict[str, Any]]] = {} - for cluster in clusters: - gap_label = str(cluster.get("gap_label") or "UNKNOWN") - grouped.setdefault(gap_label, []).append(cluster) - return grouped - - -def _append_matched_cluster_pair( - matched: list[dict[str, Any]], - current: dict[str, Any], - baseline: dict[str, Any], - *, - match_confidence: float, - match_method: str, -) -> None: - matched.append( - { - "current_label": current.get("label"), - "baseline_label": baseline.get("label"), - "gap_label": current.get("gap_label") or baseline.get("gap_label"), - "current_share_pct": current.get("share_pct"), - "baseline_share_pct": baseline.get("share_pct"), - "share_delta_pp": round( - float(current.get("share_pct") or 0.0) - - float(baseline.get("share_pct") or 0.0), - 1, - ), - "match_confidence": round(match_confidence, 2), - "match_method": match_method, - } - ) - - -def _aggregate_share_by_gap_label( - clusters: list[dict[str, Any]], -) -> dict[str, float]: - totals: dict[str, float] = {} - for cluster in clusters: - gap_label = str(cluster.get("gap_label") or "UNKNOWN") - totals[gap_label] = totals.get(gap_label, 0.0) + float( - cluster.get("share_pct") or 0.0 - ) - return {gap: round(share, 1) for gap, share in totals.items()} - - -def _reconcile_cluster_periods( - current_clusters: list[dict[str, Any]], - baseline_clusters: list[dict[str, Any]], - *, - similarity_threshold: float = 0.35, -) -> dict[str, Any]: - """Align independently-generated cluster labels before delta explanation.""" - matched: list[dict[str, Any]] = [] - current_unmatched = list(current_clusters) - remaining_baseline = list(baseline_clusters) - - current_by_gap = _group_clusters_by_gap_label(current_unmatched) - baseline_by_gap = _group_clusters_by_gap_label(remaining_baseline) - for gap_label in list(current_by_gap): - current_group = current_by_gap.get(gap_label) or [] - baseline_group = baseline_by_gap.get(gap_label) or [] - if len(current_group) != 1 or len(baseline_group) != 1: - continue - current = current_group[0] - baseline = baseline_group[0] - _append_matched_cluster_pair( - matched, - current, - baseline, - match_confidence=0.75, - match_method="single_cluster_per_gap_label", - ) - current_unmatched.remove(current) - remaining_baseline.remove(baseline) - current_by_gap[gap_label] = [] - baseline_by_gap[gap_label] = [] - - for current in list(current_unmatched): - best_idx: Optional[int] = None - best_score = 0.0 - for idx, baseline in enumerate(remaining_baseline): - score = _cluster_label_similarity( - str(current.get("label") or ""), - str(baseline.get("label") or ""), - ) - if current.get("gap_label") == baseline.get("gap_label"): - score += 0.1 - if score > best_score: - best_score = score - best_idx = idx - - if best_idx is not None and best_score >= similarity_threshold: - baseline = remaining_baseline.pop(best_idx) - _append_matched_cluster_pair( - matched, - current, - baseline, - match_confidence=best_score, - match_method="label_similarity", - ) - - matched_current_labels = { - str(item.get("current_label") or "") for item in matched - } - matched_baseline_labels = { - str(item.get("baseline_label") or "") for item in matched - } - current_unmatched = [ - cluster - for cluster in current_clusters - if str(cluster.get("label") or "") not in matched_current_labels - ] - remaining_baseline = [ - cluster - for cluster in baseline_clusters - if str(cluster.get("label") or "") not in matched_baseline_labels - ] - - new_themes = [ - { - "label": cluster.get("label"), - "gap_label": cluster.get("gap_label"), - "share_pct": cluster.get("share_pct"), - "note": "New theme in current period (no close baseline match).", - } - for cluster in current_unmatched - ] - - retired_themes = [ - { - "label": baseline.get("label"), - "gap_label": baseline.get("gap_label"), - "share_pct": baseline.get("share_pct"), - "note": "Theme present in baseline only (retired or renamed).", - } - for baseline in remaining_baseline - ] - - current_gap = _aggregate_share_by_gap_label(current_clusters) - baseline_gap = _aggregate_share_by_gap_label(baseline_clusters) - gap_label_shifts: dict[str, dict[str, float]] = {} - for gap_label in set(current_gap) | set(baseline_gap): - current_share = current_gap.get(gap_label, 0.0) - baseline_share = baseline_gap.get(gap_label, 0.0) - if abs(current_share - baseline_share) >= 0.5: - gap_label_shifts[gap_label] = { - "current_share_pct": current_share, - "baseline_share_pct": baseline_share, - "share_delta_pp": round(current_share - baseline_share, 1), - } - - return { - "matched_theme_shifts": matched, - "new_themes_current_period": new_themes, - "retired_themes_baseline_period": retired_themes, - "gap_label_shifts": gap_label_shifts, - "reconciliation_note": ( - "Cluster labels are generated independently each run and may " - "rename the same failure mode. Do not treat unmatched current " - "labels as 0% in the baseline period." - ), - } - - -def _load_period_delta_explanations_cache( - evaluation: CallImportEvaluation, - cache_key: str, -) -> Optional[dict[str, str]]: - raw = getattr(evaluation, "period_delta_explanations", None) - if not isinstance(raw, dict): - return None - entry = raw.get(cache_key) - if not isinstance(entry, dict): - return None - explanations_raw = entry.get("explanations") - if not isinstance(explanations_raw, dict): - return None - return { - str(metric_id): str(why).strip() - for metric_id, why in explanations_raw.items() - if str(metric_id).strip() and isinstance(why, str) and why.strip() - } - - -def _save_period_delta_explanations_cache( - db: Session, - evaluation: CallImportEvaluation, - cache_key: str, - explanations: dict[str, str], -) -> None: - raw = evaluation.period_delta_explanations - if not isinstance(raw, dict): - raw = {} - updated = dict(raw) - updated[cache_key] = { - "explanations": explanations, - "generated_at": datetime.now(timezone.utc).isoformat(), - } - evaluation.period_delta_explanations = updated - flag_modified(evaluation, "period_delta_explanations") - db.commit() - - -def _cluster_summary_for_metric( - state: Optional[EvaluationMetricClustersState], - metric_id: str, -) -> list[dict[str, Any]]: - if state is None or state.status != "completed": - return [] - for group in state.groups: - if str(group.metric_id) != metric_id: - continue - return [ - { - "label": cluster.label, - "gap_label": cluster.gap_label, - "share_pct": round(cluster.share_pct, 1), - "count": cluster.count, - } - for cluster in group.clusters[:5] - ] - return [] - - -def _merge_delta_why( - raw_deltas: dict[str, dict[str, str]], - explanations: dict[str, str], -) -> dict[str, dict[str, str]]: - if not explanations: - return raw_deltas - merged: dict[str, dict[str, str]] = {} - for metric_id, delta in raw_deltas.items(): - updated = dict(delta) - why = explanations.get(metric_id) - if why: - updated["why"] = why - merged[metric_id] = updated - return merged - - -def _explain_period_deltas( - db: Session, - organization_id: UUID, - evaluation: CallImportEvaluation, - baseline_evaluation: CallImportEvaluation, - raw_deltas: dict[str, dict[str, str]], - *, - min_delta_pp: float = 0.5, -) -> dict[str, dict[str, str]]: - """Attach ``why`` explanations to period deltas using cached LLM output.""" - if not raw_deltas: - return raw_deltas - - cache_key = _period_delta_explanation_cache_key( - baseline_evaluation.id, - completed_rows=evaluation.completed_rows, - baseline_completed_rows=baseline_evaluation.completed_rows, - ) - cached = _load_period_delta_explanations_cache(evaluation, cache_key) - if cached is not None: - return _merge_delta_why(raw_deltas, cached) - - current_clusters = _metric_clusters_payload(evaluation) - baseline_clusters = _metric_clusters_payload(baseline_evaluation) - metrics_for_prompt: list[dict[str, Any]] = [] - for metric_id, delta in raw_deltas.items(): - label = delta.get("label") or "" - if "No previous-week baseline" in label: - continue - match = re.search(r"([+-]?\d+(?:\.\d+)?)\s*pp", label) - if match and abs(float(match.group(1))) < min_delta_pp: - continue - current_summary = _cluster_summary_for_metric(current_clusters, metric_id) - baseline_summary = _cluster_summary_for_metric(baseline_clusters, metric_id) - if not current_summary and not baseline_summary: - continue - cluster_reconciliation = _reconcile_cluster_periods( - current_summary, - baseline_summary, - ) - metrics_for_prompt.append( - { - "metric_id": metric_id, - "delta_label": label, - "delta_detail": delta.get("detail") or "", - "cluster_reconciliation": cluster_reconciliation, - } - ) - - if not metrics_for_prompt: - return raw_deltas - - provider_hint: Optional[str] = None - model_hint: Optional[str] = None - tldr_raw = evaluation.tldr_summary - if isinstance(tldr_raw, dict): - if isinstance(tldr_raw.get("provider"), str): - provider_hint = tldr_raw["provider"] - if isinstance(tldr_raw.get("model"), str): - model_hint = tldr_raw["model"] - - from app.services.ai.llm_resolver import get_llm_provider_and_model - from app.services.call_import_user_insights import _call_llm, _parse_json_object - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, provider_hint, model_hint - ) - try: - text = _call_llm( - db, - organization_id, - provider_enum, - model_str, - [ - {"role": "system", "content": _DELTA_EXPLANATION_SYSTEM_PROMPT}, - { - "role": "user", - "content": json.dumps( - {"metrics": metrics_for_prompt}, - ensure_ascii=False, - default=str, - ), - }, - ], - temperature=0.3, - max_tokens=900, - ) - except Exception as exc: - logger.warning("[PeriodDeltaExplain] LLM call failed: {}", exc) - return raw_deltas - - parsed = _parse_json_object(text) - explanations_raw = parsed.get("explanations") - explanations: dict[str, str] = {} - if isinstance(explanations_raw, dict): - for metric_id, why in explanations_raw.items(): - if isinstance(why, str) and why.strip(): - explanations[str(metric_id)] = why.strip() - - if explanations: - _save_period_delta_explanations_cache( - db, evaluation, cache_key, explanations - ) - return _merge_delta_why(raw_deltas, explanations) - - -def _period_deltas_with_explanations( - db: Session, - organization_id: UUID, - evaluation: CallImportEvaluation, - baseline_evaluation: CallImportEvaluation, - raw_deltas: dict[str, dict[str, str]], -) -> dict[str, dict[str, str]]: - return _explain_period_deltas( - db, - organization_id, - evaluation, - baseline_evaluation, - raw_deltas, - ) - - -def _benchmark_context_for_snapshot( - db: Session, - previous_snapshot: Optional[CallImportEvaluationReportSnapshot], -) -> Optional[dict[str, str]]: - if previous_snapshot is None: - return None - previous_import = ( - db.query(CallImport) - .filter(CallImport.id == previous_snapshot.call_import_id) - .first() - ) - previous_eval = ( - db.query(CallImportEvaluation) - .filter(CallImportEvaluation.id == previous_snapshot.evaluation_id) - .first() - ) - dataset = ( - (previous_import.dataset or "").strip() - if previous_import and previous_import.dataset - else None - ) - filename = ( - (previous_import.original_filename or previous_import.filename or "").strip() - if previous_import - else None - ) - evaluation_label = ( - (previous_eval.name or "").strip() - if previous_eval and previous_eval.name - else str(previous_snapshot.evaluation_id)[:8] - ) - period = previous_snapshot.period_label or ( - previous_snapshot.period_start.isoformat() - if previous_snapshot.period_start - else "previous report" - ) - return { - "dataset": dataset or filename or "Unknown dataset", - "evaluation": evaluation_label, - "evaluation_id": str(previous_snapshot.evaluation_id), - "period": period, - } - - -def _clamp_prose_to_sentences( - text: str, - *, - max_sentences: int = 3, - max_chars: int = 300, -) -> str: - """Keep concise audit/TLDR prose within sentence and character limits.""" - cleaned = (text or "").strip() - if not cleaned: - return cleaned - cleaned = re.sub(r"\s*\n+\s*", " ", cleaned).strip() - sentences = [ - sentence.strip() - for sentence in re.split(r"(?<=[.!?])\s+", cleaned) - if sentence.strip() - ] - if sentences: - result = " ".join(sentences[:max_sentences]).strip() - else: - result = cleaned - if len(result) > max_chars: - trimmed = result[: max_chars - 3].rsplit(" ", 1)[0].rstrip(".,;:") - result = f"{trimmed}..." if trimmed else result[:max_chars] - return result - - -def _audit_summary_text_from_tldr( - summary: Optional[EvaluationTldrSummary], -) -> Optional[str]: - if summary is None: - return None - narrative = _clamp_prose_to_sentences(summary.narrative.strip()) - return narrative or None - - -def _metric_insights_from_tldr( - summary: Optional[EvaluationTldrSummary], -) -> dict[str, str]: - if summary is None: - return {} - return { - str(metric_id): insight.strip() - for metric_id, insight in summary.metric_insights.items() - if str(metric_id).strip() and insight.strip() - } - - -def _report_period_from_rows( - rows: list[tuple[CallImportEvaluationRow, CallImportRow]], -) -> tuple[Optional[date], Optional[date], Optional[str], str]: - dates = [ - source_row.recording_date - for eval_row, source_row in rows - if eval_row.status == "completed" and source_row.recording_date - ] - if not dates: - return None, None, None, "Not specified" - start = min(dates) - end = max(dates) - week_anchor = max(dates) - week_start = week_anchor - timedelta(days=week_anchor.weekday()) - week_end = week_start + timedelta(days=6) - iso_year, iso_week, _ = week_anchor.isocalendar() - label = f"{iso_year}-W{iso_week:02d}" - if week_start.year == week_end.year: - week_range = f"{week_start.strftime('%b %d')}–{week_end.strftime('%b %d, %Y')}" - else: - week_range = ( - f"{week_start.strftime('%b %d, %Y')}–{week_end.strftime('%b %d, %Y')}" - ) - display = f"W{iso_week:02d} · {week_range}" - return start, end, label, display - - -def _aggregate_to_dict(aggregate: CallImportMetricAggregate) -> dict[str, Any]: - if hasattr(aggregate, "model_dump"): - return aggregate.model_dump(mode="json") - return aggregate.dict() - - -def _aggregate_primary_percent( - raw: dict[str, Any], - policy: Optional[MetricFailurePolicy] = None, -) -> Optional[float]: - return aggregate_primary_percent(raw, policy) - - -def _child_names_by_parent( - db: Session, - organization_id: UUID, - parent_metric_ids: Sequence[UUID], -) -> Dict[str, List[str]]: - if not parent_metric_ids: - return {} - children = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.parent_metric_id.in_(list(parent_metric_ids)), - ) - .all() - ) - out: Dict[str, List[str]] = {} - for child in children: - pid = str(child.parent_metric_id) - out.setdefault(pid, []).append(child.name) - return out - - -def _clustering_context( - db: Session, - evaluation: CallImportEvaluation, - eval_rows: List[CallImportEvaluationRow], -) -> Tuple[ - List[Metric], - List[CallImportMetricAggregate], - Dict[str, MetricFailurePolicy], - Literal["inferred", "user"], - Dict[str, List[str]], -]: - metrics = _metrics_for_clustering(db, evaluation, eval_rows) - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - parent_ids = [ - m.id - for m in metrics - if getattr(m, "selection_mode", None) and not getattr(m, "parent_metric_id", None) - ] - child_names_by_parent = _child_names_by_parent( - db, evaluation.organization_id, parent_ids - ) - policies, source = effective_policies( - evaluation, - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - ) - return metrics, aggregates, policies, source, child_names_by_parent - - -def _period_deltas_from_aggregates( - previous_metric_aggregates: list[dict[str, Any]], - current_metric_aggregates: list[dict[str, Any]], - policies: Optional[Dict[str, MetricFailurePolicy]] = None, -) -> dict[str, dict[str, str]]: - current_by_id = {str(item.get("metric_id")): item for item in current_metric_aggregates} - previous_by_id = { - str(item.get("metric_id")): item - for item in previous_metric_aggregates - if isinstance(item, dict) - } - deltas: dict[str, dict[str, str]] = {} - for metric_id, current in current_by_id.items(): - previous_raw = previous_by_id.get(metric_id) - policy = (policies or {}).get(metric_id) - current_pct = _aggregate_primary_percent(current, policy) - previous_pct = ( - _aggregate_primary_percent(previous_raw, policy) - if previous_raw - else None - ) - if current_pct is None or previous_pct is None: - deltas[metric_id] = { - "label": "No previous-week baseline", - "detail": "No comparable prior report snapshot was found.", - } - continue - delta = current_pct - previous_pct - sign = "+" if delta >= 0 else "" - deltas[metric_id] = { - "label": f"{sign}{delta:.1f} pp", - "detail": f"Current report {current_pct:.1f}% vs previous report {previous_pct:.1f}%", - } - return deltas - - -def _period_deltas_from_snapshot( - previous: Optional[CallImportEvaluationReportSnapshot], - current_metric_aggregates: list[dict[str, Any]], -) -> dict[str, dict[str, str]]: - previous_items = ( - previous.metric_aggregates - if previous and isinstance(previous.metric_aggregates, list) - else [] - ) - return _period_deltas_from_aggregates(previous_items, current_metric_aggregates) - - -def _sample_evidence_for_metrics( - rows: list[tuple[CallImportEvaluationRow, CallImportRow]], - metric_ids: set[str], -) -> dict[str, list[dict[str, str]]]: - samples: dict[str, list[dict[str, str]]] = {metric_id: [] for metric_id in metric_ids} - for eval_row, source_row in rows: - scores = eval_row.metric_scores if isinstance(eval_row.metric_scores, dict) else {} - for metric_id in metric_ids: - if len(samples.get(metric_id, [])) >= 4: - continue - score = scores.get(metric_id) - if not isinstance(score, dict): - continue - rationale = score.get("rationale") - transcript = source_row.diarised_transcript or source_row.transcript or "" - quote = rationale if isinstance(rationale, str) and rationale.strip() else transcript[:350] - if quote: - samples.setdefault(metric_id, []).append( - { - "conversation_id": source_row.conversation_id, - "quote": str(quote).strip()[:500], - } - ) - return samples - - -def _fallback_report_narrative( - insight_aggregates: list[dict[str, Any]], - evidence_samples: dict[str, list[dict[str, str]]], -) -> dict[str, Any]: - observations: dict[str, str] = {} - evidence: dict[str, dict[str, str]] = {} - design_notes: list[str] = [] - for aggregate in insight_aggregates: - metric_id = str(aggregate.get("metric_id") or "") - name = str(aggregate.get("metric_name") or "Insight") - counts = aggregate.get("value_counts") if isinstance(aggregate.get("value_counts"), list) else [] - if counts: - top = counts[0] - total = int(aggregate.get("count") or 0) or sum( - int(item.get("count") or 0) for item in counts if isinstance(item, dict) - ) - pct = (int(top.get("count") or 0) / total) * 100 if total else 0 - observations[metric_id] = ( - f"{top.get('label')} is the dominant {name.lower()} category at {pct:.1f}% of classified calls." - ) - design_notes.append( - f"{name}: {top.get('label')} is the largest segment and should be reviewed for workflow or prompt improvements." - ) - sample = (evidence_samples.get(metric_id) or [{}])[0] - if sample: - evidence[metric_id] = sample - return { - "observations": observations, - "evidence": evidence, - "design_notes": design_notes[:7], - "audit_summary": None, - } - - -def _generate_report_narrative( - db: Session, - organization_id: UUID, - *, - metric_aggregates: list[dict[str, Any]], - insight_aggregates: list[dict[str, Any]], - period_delta_by_metric: dict[str, dict[str, str]], - evidence_samples: dict[str, list[dict[str, str]]], - report_config: dict[str, Any], -) -> dict[str, Any]: - if not insight_aggregates: - return {"observations": {}, "evidence": {}, "design_notes": [], "audit_summary": None} - try: - from app.services.ai.llm_resolver import get_llm_provider_and_model - from app.services.ai.llm_service import llm_service - - provider_enum, model_str = get_llm_provider_and_model(organization_id, db, None, None) - prompt = ( - "You are writing a vendor-safe external call quality audit report. " - "Return strict JSON with keys observations (object keyed by metric_id), " - "evidence (object keyed by metric_id with conversation_id and quote), " - "design_notes (array of concise numbered-note strings), and audit_summary (string). " - "Use only the supplied aggregates and evidence samples.\n\n" - + json.dumps( - { - "metric_aggregates": metric_aggregates[:30], - "insight_aggregates": insight_aggregates, - "period_deltas": period_delta_by_metric, - "evidence_samples": evidence_samples, - "report_config": report_config, - }, - default=str, - ) - ) - llm_result = llm_service.generate_response( - messages=[ - {"role": "system", "content": "Return JSON only. No markdown."}, - {"role": "user", "content": prompt}, - ], - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - temperature=0.2, - max_tokens=1200, - ) - parsed = json.loads(str(llm_result.content or "{}")) - if isinstance(parsed, dict): - fallback = _fallback_report_narrative(insight_aggregates, evidence_samples) - return { - "observations": parsed.get("observations") or fallback["observations"], - "evidence": parsed.get("evidence") or fallback["evidence"], - "design_notes": parsed.get("design_notes") or fallback["design_notes"], - "audit_summary": parsed.get("audit_summary") or fallback["audit_summary"], - } - except Exception as exc: # noqa: BLE001 - logger.warning("Report narrative LLM generation fell back to deterministic text: {}", exc) - return _fallback_report_narrative(insight_aggregates, evidence_samples) - - -@router.get( - "/{eval_id}/baseline-candidates", - response_model=CallImportEvaluationBaselineCandidatesResponse, - operation_id="listCallImportEvaluationBaselineCandidates", -) -async def list_call_import_evaluation_baseline_candidates( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationBaselineCandidatesResponse: - del api_key - call_import = _require_import(db, call_import_id, organization_id) - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - rows = _evaluation_rows_for_period(db, evaluation.id) - period_start, _period_end, _derived_period_label, _period_display = _report_period_from_rows( - rows - ) - candidates = _baseline_candidate_evaluations( - db, - organization_id, - call_import.workspace_id, - evaluation, - period_start, - ) - default_evaluation_id = next( - (item["evaluation_id"] for item in candidates if item.get("is_default")), - None, - ) - return CallImportEvaluationBaselineCandidatesResponse( - items=[CallImportEvaluationBaselineCandidate(**item) for item in candidates], - default_evaluation_id=default_evaluation_id, - ) - - -@router.post( - "/{eval_id}/pdf-report", - operation_id="generateCallImportEvaluationPdfReport", - dependencies=[Depends(require_call_import_capability(REPORTS_GENERATE))], -) -async def generate_call_import_evaluation_pdf_report( - call_import_id: UUID, - eval_id: UUID, - payload: CallImportEvaluationPdfReportRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> StreamingResponse: - del api_key - call_import = _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - is_internal = payload.report_type == "internal" - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - rows = sorted( - load_evaluation_row_pairs(db, eval_id), - key=lambda pair: int(pair[1].row_index or 0), - ) - else: - rows = ( - db.query(CallImportEvaluationRow, CallImportRow) - .join( - CallImportRow, - CallImportRow.id == CallImportEvaluationRow.call_import_row_id, - ) - .filter(CallImportEvaluationRow.evaluation_id == eval_id) - .order_by(CallImportRow.row_index.asc()) - .all() - ) - report_config = payload.report_config if isinstance(payload.report_config, dict) else {} - metrics = _display_metrics_for_pdf_report(db, organization_id, evaluation) - configured_quality_ids = { - str(item) - for item in report_config.get("quality_metric_ids", []) - if item - } - configured_insight_ids = { - str(item.get("metric_id") or item) - for item in report_config.get("insights", []) - if item - } - if configured_quality_ids or configured_insight_ids: - allowed_ids = configured_quality_ids | configured_insight_ids - metrics = [metric for metric in metrics if str(metric.id) in allowed_ids] - - eval_rows = [eval_row for eval_row, _source_row in rows] - aggregate_models = _compute_metric_aggregates(db, evaluation, eval_rows) - selected_report_metric_ids = {str(metric.id) for metric in metrics} - aggregate_dicts = [ - _aggregate_to_dict(aggregate) - for aggregate in aggregate_models - if aggregate.metric_id in selected_report_metric_ids - ] - insight_metric_ids = { - str(metric.id) - for metric in metrics - if _metric_is_user_insight(metric) - } - metric_aggregates = [ - item for item in aggregate_dicts if str(item.get("metric_id")) not in insight_metric_ids - ] - insight_aggregates = [ - item for item in aggregate_dicts if str(item.get("metric_id")) in insight_metric_ids - ] - period_start, period_end, derived_period_label, period_display = _report_period_from_rows(rows) - period_label = (payload.period_label or derived_period_label or "").strip() or None - include_period_delta = ( - payload.include_period_delta or payload.include_weekly_delta - ) - previous_snapshot = None - period_delta_by_metric: dict[str, dict[str, str]] = {} - baseline_evaluation: Optional[CallImportEvaluation] = None - if include_period_delta and period_start: - baseline_evaluation = _resolve_baseline_evaluation( - db, - organization_id, - call_import.workspace_id, - evaluation, - period_start, - payload.baseline_evaluation_id, - ) - if baseline_evaluation: - period_delta_by_metric = _period_deltas_from_evaluation( - db, - baseline_evaluation, - metric_aggregates, - evaluation, - [eval_row for eval_row, _ in rows], - ) - period_delta_by_metric = _period_deltas_with_explanations( - db, - organization_id, - evaluation, - baseline_evaluation, - period_delta_by_metric, - ) - benchmark_context = _benchmark_context_for_evaluation(db, baseline_evaluation) - evidence_samples = _sample_evidence_for_metrics(rows, insight_metric_ids) - cached_tldr_summary = _tldr_summary_payload(evaluation) - cached_user_insights = _user_insights_payload(evaluation) - cached_metric_clusters = _metric_clusters_payload(evaluation) - cached_prompt_improvements = _prompt_improvements_payload(evaluation) - generated_insights_for_pdf = _selected_generated_user_insights( - cached_user_insights, - report_config, - ) - metric_clusters_for_pdf = _selected_metric_clusters_for_pdf( - cached_metric_clusters, - report_config, - ) - prompt_improvements_for_pdf = _selected_prompt_improvements_for_pdf( - cached_prompt_improvements, - report_config, - ) - narrative = _generate_report_narrative( - db, - organization_id, - metric_aggregates=metric_aggregates, - insight_aggregates=insight_aggregates if is_internal else [], - period_delta_by_metric=period_delta_by_metric, - evidence_samples=evidence_samples if is_internal else {}, - report_config=report_config, - ) - - generated_at = datetime.now(timezone.utc) - branding_images, custom_heading = _report_branding_for_import_workspace( - db, - organization_id, - call_import.workspace_id, - internal_brand_image_id=payload.internal_brand_image_id, - external_brand_image_id=payload.external_brand_image_id, - ) - eval_row_list = [eval_row for eval_row, _ in rows] - pdf_aggregates = _compute_metric_aggregates(db, evaluation, eval_row_list) - pdf_parent_ids = [ - m.id - for m in metrics - if getattr(m, "selection_mode", None) - and not getattr(m, "parent_metric_id", None) - ] - pdf_child_map = _child_names_by_parent( - db, evaluation.organization_id, pdf_parent_ids - ) - failure_policies_for_pdf, _fp_source = effective_policies( - evaluation, - metrics, - pdf_aggregates, - child_names_by_parent=pdf_child_map, - ) - try: - pdf_started = datetime.now(timezone.utc) - pdf_bytes = await asyncio.to_thread( - call_import_evaluation_pdf_report_service.render_pdf, - vendor_name=payload.vendor_name, - call_import=call_import, - evaluation=evaluation, - metrics=metrics, - rows=rows, - failure_policies=failure_policies_for_pdf, - generated_at=generated_at, - internal=is_internal, - logo_data_uris=branding_images, - custom_heading=custom_heading, - include_weekly_delta=include_period_delta, - period_delta_by_metric=period_delta_by_metric, - use_case=payload.use_case, - period_display=period_display, - total_metric_count=db.query(Metric) - .filter(Metric.organization_id == organization_id, Metric.enabled.is_(True)) - .count(), - report_config=report_config, - narrative=narrative, - audit_summary=_audit_summary_text_from_tldr(cached_tldr_summary), - metric_insights=_metric_insights_from_tldr(cached_tldr_summary), - benchmark_context=benchmark_context, - generated_user_insights=generated_insights_for_pdf, - user_insights_overview=( - cached_user_insights.overview if cached_user_insights else None - ), - metric_clusters=metric_clusters_for_pdf, - metric_clusters_overview=( - cached_metric_clusters.overview if cached_metric_clusters else None - ), - prompt_improvements=prompt_improvements_for_pdf, - platform_base_url=payload.platform_base_url, - ) - logger.info( - "PDF report render finished in {:.1f}s for evaluation {}", - (datetime.now(timezone.utc) - pdf_started).total_seconds(), - eval_id, - ) - except Exception as exc: # noqa: BLE001 - logger.exception( - "Failed to generate PDF report for call import {} evaluation {}", - call_import_id, - eval_id, - ) - raise HTTPException( - status_code=500, - detail=f"Failed to generate PDF report: {exc}", - ) from exc - - snapshot = CallImportEvaluationReportSnapshot( - evaluation_id=evaluation.id, - call_import_id=call_import.id, - organization_id=organization_id, - workspace_id=call_import.workspace_id, - period_label=period_label, - period_start=period_start, - period_end=period_end, - report_config=report_config, - selected_metric_ids=[str(metric.id) for metric in metrics], - metric_aggregates=metric_aggregates, - insight_aggregates=insight_aggregates, - narrative=narrative, - total_calls=evaluation.total_rows, - selected_metric_count=len(metrics), - total_metric_count=db.query(Metric) - .filter(Metric.organization_id == organization_id, Metric.enabled.is_(True)) - .count(), - ) - db.add(snapshot) - db.commit() - - filename = ( - f"{_report_filename_slug(payload.vendor_name)}-" - f"{payload.report_type}-quality-metric-audit-{eval_id}.pdf" - ) - return StreamingResponse( - iter([pdf_bytes]), - media_type="application/pdf", - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, - ) - - -@router.patch( - "/{eval_id}", - response_model=CallImportEvaluationResponse, - operation_id="updateCallImportEvaluation", -) -async def update_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - payload: CallImportEvaluationUpdate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationResponse: - """Edit metadata on an existing evaluation run (currently just ``name``).""" - - del api_key - _require_import(db, call_import_id, organization_id) - - row = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not row: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - # Treat unset vs explicit ``None`` differently: unset = leave alone, - # explicit ``None`` or empty string = clear the name. - payload_data = payload.model_dump(exclude_unset=True) - if "name" in payload_data: - row.name = _normalize_name(payload_data["name"]) - - db.commit() - db.refresh(row) - return _serialize_eval(db, row) - - -def _revoke_pending_tasks(evaluation: CallImportEvaluation) -> None: - """Best-effort cancel of any in-flight Celery tasks for an evaluation.""" - - if not evaluation.celery_group_id and not any( - r.celery_task_id for r in evaluation.row_results - ): - return - try: - from app.workers.celery_app import celery_app - - pending_task_ids = [ - eval_row.celery_task_id - for eval_row in evaluation.row_results - if eval_row.celery_task_id - and eval_row.status in {"pending", "running"} - ] - if pending_task_ids: - celery_app.control.revoke(pending_task_ids, terminate=False) - except Exception: - # Best effort — DB delete remains the source of truth. - pass - - -# --------------------------------------------------------------------------- -# User-initiated cancel for in-flight evaluation rows -# --------------------------------------------------------------------------- -# -# Evaluation rows can sit in ``running`` for many minutes when the underlying -# LLM / audio metric call is slow or wedged (the worker carries an 8 min -# soft / 10 min hard time limit). Without a cancel affordance the operator's -# only recourse is to wait for Celery's time limit to fire — or to manually -# mutate the DB. These helpers + the two endpoints below give the UI a -# first-class "Abort" button mirroring the diarisation cancel pattern at -# ``app.api.v1.routes.call_imports`` (``_apply_diarisation_cancel`` etc.). -# -# Why ``terminate=True``: the legacy ``_revoke_pending_tasks`` above uses -# ``terminate=False`` because it's called from delete-flow paths where the -# task may simply not get to run (a worker pulls it off the queue and drops -# it). For a user-initiated cancel we want SIGTERM to interrupt the worker -# mid-LLM/audio call so the in-flight HTTP request actually aborts. -# ``terminate=True`` routes the signal to the executing process; we spell -# ``signal="SIGTERM"`` out for clarity even though it's the default. - -# Sentinel error message stamped on cancelled rows. Read by the eval worker's -# ``_was_cancelled_externally`` guard (see -# :mod:`app.workers.tasks.evaluate_call_import_row`) so a worker that's already -# past its slowest operation can't overwrite the cancelled state with its own -# terminal status. Touching either copy means touching both. -EVAL_CANCELLED_BY_USER_ERROR: str = "Evaluation cancelled by user" - - -def _cancellable_eval_states() -> Tuple[str, ...]: - """States that an evaluation row can be cancelled from. - - Kept as a tiny helper so adding a future ``"queued"`` / ``"retrying"`` - state only needs one edit. - """ - return ("pending", "running") - - -def _revoke_eval_task(eval_row: CallImportEvaluationRow) -> None: - """Best-effort revoke of a single eval row's Celery task. - - Always swallows control-plane exceptions — Celery's control bus is - inherently best-effort and a missed revoke is not catastrophic - because the DB row is already flipped to ``failed`` by the caller - before this runs (so the UI immediately reflects the cancel; if - the task happens to finish anyway, the worker's finaliser skips - over the row via :data:`EVAL_CANCELLED_BY_USER_ERROR`). - """ - task_id = (eval_row.celery_task_id or "").strip() - if not task_id: - return - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke( - task_id, terminate=True, signal="SIGTERM" - ) - logger.info( - "Revoked evaluation task {} for eval row {}", - task_id, - eval_row.id, - ) - except Exception as exc: # noqa: BLE001 — revoke is best-effort - logger.warning( - "Failed to revoke evaluation task {} for eval row {}: {}", - task_id, - eval_row.id, - exc, - ) - - -def _apply_evaluation_cancel( - eval_rows: List[CallImportEvaluationRow], -) -> Tuple[int, int]: - """Cancel every cancellable row in ``eval_rows``. - - Returns ``(cancelled, skipped)`` so the caller can build a typed - response without re-querying the DB. The caller is responsible for - ``db.commit()`` after this returns — we deliberately don't commit - here so a batch endpoint can flush all rows in one transaction. - """ - cancellable_states = _cancellable_eval_states() - cancelled = 0 - skipped = 0 - now = datetime.now(timezone.utc) - for eval_row in eval_rows: - if (eval_row.status or "").lower() not in cancellable_states: - skipped += 1 - continue - # Flip the row state BEFORE we revoke so the UI's next poll - # already shows the cancel, even if Celery's control plane is - # slow to ack. - eval_row.status = "failed" - eval_row.error_message = EVAL_CANCELLED_BY_USER_ERROR - eval_row.finished_at = now - _revoke_eval_task(eval_row) - # Drop the task id so a follow-up retry (or a stale poll) can't - # accidentally re-revoke or get confused. - eval_row.celery_task_id = None - cancelled += 1 - return cancelled, skipped - - -def _claim_evaluation_bulk_operation( - evaluation_id: UUID, - operation: str, -) -> None: - """Reserve the run for a single bulk worker pass; 409 if one is active.""" - from app.services.call_imports.evaluation_bulk_op import ( - get_evaluation_bulk_operation, - try_set_evaluation_bulk_operation, - ) - - if try_set_evaluation_bulk_operation(evaluation_id, operation): # type: ignore[arg-type] - return - existing = get_evaluation_bulk_operation(evaluation_id) or operation - raise HTTPException( - status_code=409, - detail=( - f"A bulk {existing.replace('_', ' ')} operation is already in " - "progress for this evaluation. Wait for it to finish before " - "starting another action." - ), - ) - - -def _require_no_evaluation_bulk_operation(evaluation_id: UUID) -> None: - from app.services.call_imports.evaluation_bulk_op import ( - get_evaluation_bulk_operation, - ) - - existing = get_evaluation_bulk_operation(evaluation_id) - if existing: - raise HTTPException( - status_code=409, - detail=( - f"A bulk {existing.replace('_', ' ')} operation is already in " - "progress for this evaluation. Wait for it to finish before " - "starting another action." - ), - ) - - -@router.post( - "/{eval_id}/cancel", - response_model=CallImportEvaluationBulkActionResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="cancelCallImportEvaluation", -) -async def cancel_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationBulkActionResponse: - """Abort all in-flight (or queued) rows in a single evaluation run. - - Idempotent: calling on a run whose rows are already terminal returns - ``target_count=0`` with 202 so the UI can fire this from an - "Abort" button without having to pre-check the state. - - Heavy row resets and Celery revokes run in a background worker so - large batches do not block the API thread. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - from app.services.call_imports.bulk_ops import count_evaluation_cancel_targets - - target_count = count_evaluation_cancel_targets(db, eval_id, mode="abort") - if target_count == 0: - return CallImportEvaluationBulkActionResponse( - accepted=True, - target_count=0, - evaluation_id=eval_id, - ) - - _claim_evaluation_bulk_operation(eval_id, "abort") - evaluation.status = "cancelled" - db.commit() - - from app.workers.tasks.call_import_bulk_ops import ( - cancel_call_import_evaluation_task, - ) - - cancel_call_import_evaluation_task.delay(str(eval_id), mode="abort") - return CallImportEvaluationBulkActionResponse( - accepted=True, - target_count=target_count, - evaluation_id=eval_id, - ) - - -@router.post( - "/{eval_id}/force-fail-pending", - response_model=CallImportEvaluationBulkActionResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="forceFailCallImportEvaluationPending", -) -async def force_fail_pending_call_import_evaluation_rows( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationBulkActionResponse: - """Force-fail only rows currently in ``pending`` for a single run. - - This is narrower than :func:`cancel_call_import_evaluation`: it leaves - ``running`` rows untouched so operators can clear permanently queued rows - without interrupting in-flight evaluations. - - Row updates run in a background worker so large batches do not block - the API thread. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - from app.services.call_imports.bulk_ops import count_evaluation_cancel_targets - - target_count = count_evaluation_cancel_targets( - db, eval_id, mode="force_fail_pending" - ) - if target_count == 0: - return CallImportEvaluationBulkActionResponse( - accepted=True, - target_count=0, - evaluation_id=eval_id, - ) - - _claim_evaluation_bulk_operation(eval_id, "force_fail_pending") - - from app.workers.tasks.call_import_bulk_ops import ( - cancel_call_import_evaluation_task, - ) - - cancel_call_import_evaluation_task.delay( - str(eval_id), mode="force_fail_pending" - ) - return CallImportEvaluationBulkActionResponse( - accepted=True, - target_count=target_count, - evaluation_id=eval_id, - ) - - -@router.post( - "/{eval_id}/rows/{eval_row_id}/cancel", - response_model=CallImportEvaluationRowResponse, - status_code=status.HTTP_200_OK, - operation_id="cancelCallImportEvaluationRow", -) -async def cancel_call_import_evaluation_row( - call_import_id: UUID, - eval_id: UUID, - eval_row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationRowResponse: - """Abort an in-flight (or queued) evaluation for a single row. - - Idempotent: calling on a row that's already terminal (``completed`` - / ``failed``) returns the row unchanged with a 200 so the UI can - wire this to a "Stop" button without having to pre-check the - state. Updates the parent run's rollup so its counters reflect - the cancel immediately. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - _require_no_evaluation_bulk_operation(eval_id) - - from app.db_sharding.eval_rows import evaluation_row_session - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - try: - with evaluation_row_session(eval_row_id) as ( - row_db, - _catalog_db, - eval_row, - source_row, - _shard_id, - ): - if eval_row.evaluation_id != eval_id: - raise HTTPException( - status_code=404, - detail="Evaluation row not found in this run", - ) - _apply_evaluation_cancel([eval_row]) - row_db.commit() - _rollup_evaluation_status(evaluation, db) - db.commit() - row_db.refresh(eval_row) - return _to_evaluation_row_response(eval_row, source_row, evaluation) - except LookupError as exc: - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) from exc - - eval_row = ( - db.query(CallImportEvaluationRow) - .filter( - CallImportEvaluationRow.id == eval_row_id, - CallImportEvaluationRow.evaluation_id == eval_id, - ) - .first() - ) - if not eval_row: - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) - - _apply_evaluation_cancel([eval_row]) - db.flush() - _rollup_evaluation_status(evaluation, db) - db.commit() - db.refresh(eval_row) - - source_row = ( - db.query(CallImportRow) - .filter(CallImportRow.id == eval_row.call_import_row_id) - .first() - ) - - return _to_evaluation_row_response(eval_row, source_row, evaluation) - - -@router.delete( - "/{eval_id}", - status_code=status.HTTP_204_NO_CONTENT, - operation_id="deleteCallImportEvaluation", -) -async def delete_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Response: - del api_key - _require_import(db, call_import_id, organization_id) - - row = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not row: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - _revoke_pending_tasks(row) - - db.delete(row) - db.commit() - return Response(status_code=status.HTTP_204_NO_CONTENT) - - -@router.post( - "/bulk-delete", - status_code=status.HTTP_200_OK, - operation_id="bulkDeleteCallImportEvaluations", -) -async def bulk_delete_call_import_evaluations( - call_import_id: UUID, - payload: CallImportEvaluationBulkDelete, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Dict[str, int]: - """Delete multiple evaluation runs scoped to one call import. - - Mirrors :func:`delete_call_import_evaluation` but in bulk so the UI - can clear out a multi-select. Unknown ids (already deleted, or - belonging to a different org/import) are silently skipped — the - response just reports how many actually went away. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - if not payload.evaluation_ids: - return {"deleted": 0} - - rows = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id.in_(payload.evaluation_ids), - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .all() - ) - deleted = 0 - for row in rows: - _revoke_pending_tasks(row) - db.delete(row) - deleted += 1 - db.commit() - return {"deleted": deleted} - - -# --------------------------------------------------------------------------- -# Aggregation: turns per-row metric scores into histograms / value counts. -# -# Designed to be cheap enough to call on every page load: we read each -# evaluation row once, bucket numeric values into a fixed 10-bin -# histogram, and tally the top categorical values. Scaling concerns -# (millions of rows) are deferred — at that point we'd push this into a -# Postgres aggregate query, but for typical CSV imports (<10k rows) the -# Python pass is fast enough and dramatically simpler. -# --------------------------------------------------------------------------- - - -_HISTOGRAM_BUCKETS = 10 -_TOP_VALUE_COUNTS = 10 - - -def _coerce_numeric(value: Any) -> Optional[float]: - """Return ``value`` as ``float`` when it's numeric; ``None`` otherwise.""" - if isinstance(value, bool): - # Booleans are ints in Python; treat them as categorical so - # pass/fail metrics show up in value_counts instead of becoming - # a degenerate {0,1} histogram. - return None - if isinstance(value, (int, float)) and math.isfinite(value): - return float(value) - if isinstance(value, str): - try: - f = float(value) - if math.isfinite(f): - return f - except ValueError: - return None - return None - - -def _coerce_category(value: Any) -> Optional[str]: - """Render ``value`` as a label suitable for a value_counts bucket.""" - if value is None: - return None - if isinstance(value, bool): - return "true" if value else "false" - if isinstance(value, (int, float)): - return str(value) - if isinstance(value, str): - text = value.strip() - return text or None - # Lists / dicts: stringify so they still group sensibly without - # exploding the cardinality (worst case: everything is "[…]" once). - return str(value) - - -def _build_histogram( - values: List[float], -) -> List[CallImportMetricHistogramBucket]: - """Fixed-bin histogram over ``values``; returns [] for <2 values.""" - if len(values) < 2: - return [] - lo = min(values) - hi = max(values) - if lo == hi: - # All values identical — render a single bucket so the UI shows a - # spike rather than empty space. - return [ - CallImportMetricHistogramBucket(x0=lo, x1=hi, count=len(values)) - ] - width = (hi - lo) / _HISTOGRAM_BUCKETS - buckets: List[List[float]] = [[] for _ in range(_HISTOGRAM_BUCKETS)] - for v in values: - # Right-edge inclusive on the last bucket so ``hi`` doesn't fall - # off into a non-existent bucket index. - idx = int((v - lo) / width) - if idx >= _HISTOGRAM_BUCKETS: - idx = _HISTOGRAM_BUCKETS - 1 - buckets[idx].append(v) - return [ - CallImportMetricHistogramBucket( - x0=lo + i * width, - x1=lo + (i + 1) * width, - count=len(bucket), - ) - for i, bucket in enumerate(buckets) - ] - - -def _percentile(values: List[float], pct: float) -> Optional[float]: - """Linear-interpolated percentile compatible with NumPy default.""" - if not values: - return None - sorted_vals = sorted(values) - if len(sorted_vals) == 1: - return sorted_vals[0] - rank = (pct / 100.0) * (len(sorted_vals) - 1) - lo = int(math.floor(rank)) - hi = int(math.ceil(rank)) - if lo == hi: - return sorted_vals[lo] - frac = rank - lo - return sorted_vals[lo] + (sorted_vals[hi] - sorted_vals[lo]) * frac - - -def _compute_metric_aggregates( - db: Session, - evaluation: CallImportEvaluation, - eval_rows: List[CallImportEvaluationRow], -) -> List[CallImportMetricAggregate]: - """Collapse per-row ``metric_scores`` into one aggregate per metric. - - Selected metrics are read fresh from the DB so the response always - surfaces the current ``metric.name`` / ``metric_type`` even when a - metric was renamed after the run finished. - """ - - selected_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) - # Include parent metrics from selected_metric_groups so they appear - # alongside their children in the aggregate response. Use ``getattr`` - # with a default so the helper still works for callers that pass - # lightweight objects (tests, in-memory shims) that don't carry the - # attribute at all. - groups_raw_candidate = getattr(evaluation, "selected_metric_groups", None) - groups_raw = ( - groups_raw_candidate if isinstance(groups_raw_candidate, dict) else {} - ) - for parent_str in groups_raw.keys(): - try: - pid = UUID(parent_str) - if pid not in selected_ids: - selected_ids.append(pid) - except (TypeError, ValueError): - continue - - metrics = _metrics_for_ids(db, evaluation.organization_id, selected_ids) - metric_meta: Dict[str, Metric] = {str(m.id): m for m in metrics} - - # Default to selected metrics, but also include any metric ids that - # surface in row scores even if missing from the metric registry — - # otherwise renaming/deleting a metric mid-run would silently drop - # results from the chart. - discovered_ids: List[str] = list(metric_meta.keys()) - for row in eval_rows: - scores = row.metric_scores if isinstance(row.metric_scores, dict) else {} - for metric_id_str in scores.keys(): - if metric_id_str not in metric_meta and metric_id_str not in discovered_ids: - discovered_ids.append(metric_id_str) - - results: List[CallImportMetricAggregate] = [] - - for metric_id_str in discovered_ids: - meta = metric_meta.get(metric_id_str) - numeric_values: List[float] = [] - category_counts: Dict[str, int] = {} - # For multi-label parents we still need to know how many rows - # were scored (each row votes for >=1 label) so the n-badge in - # the UI shows "n=50" instead of the misleading "n=208" sum. - multi_label_rows_scored = 0 - # Unordered pair tally for the co-occurrence heatmap. Keys are - # ``(label_a, label_b)`` with ``a < b`` so we never double-count - # the same unordered pair. Only populated for multi-label - # parents — every other metric leaves this empty. - multi_label_pair_counts: Dict[Tuple[str, str], int] = {} - skipped = 0 - errored = 0 - observed_metric_type: Optional[str] = None - observed_name: Optional[str] = None - - # ``meta`` is a real ``Metric`` row in production, but tests - # frequently pass a lightweight stub. Pull the two attributes - # we need via ``getattr`` so a stub that only sets ``id`` / - # ``name`` / ``metric_type`` doesn't blow up here. - is_multi_label_parent = bool( - meta - and getattr(meta, "selection_mode", None) == "multi_label" - and not getattr(meta, "parent_metric_id", None) - ) - - for row in eval_rows: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else {} - ) - entry = scores.get(metric_id_str) - if not isinstance(entry, dict): - continue - if entry.get("metric_name"): - observed_name = entry.get("metric_name") - if entry.get("type"): - observed_metric_type = entry.get("type") - if entry.get("skipped"): - skipped += 1 - continue - if entry.get("error"): - errored += 1 - continue - - # Multi-label parents store a comma-joined value that - # isn't useful as a single category; instead tally each - # selected child individually so the chart shows per-label - # counts that mirror the children's own boolean histograms. - if is_multi_label_parent: - selected = entry.get("selected_child_names") - if isinstance(selected, list) and selected: - multi_label_rows_scored += 1 - cleaned: List[str] = [] - for label in selected: - text_label = str(label).strip() or None - if text_label: - cleaned.append(text_label) - category_counts[text_label] = ( - category_counts.get(text_label, 0) + 1 - ) - # Emit one increment per unordered pair of distinct - # labels that fired together on this row. ``cleaned`` - # is deduplicated first because the LLM occasionally - # repeats a label inside ``selected_child_names``. - distinct = sorted(set(cleaned)) - for i in range(len(distinct)): - for j in range(i + 1, len(distinct)): - pair = (distinct[i], distinct[j]) - multi_label_pair_counts[pair] = ( - multi_label_pair_counts.get(pair, 0) + 1 - ) - continue - - value = entry.get("value") - numeric = _coerce_numeric(value) - if numeric is not None: - numeric_values.append(numeric) - continue - category = _coerce_category(value) - if category is not None: - category_counts[category] = category_counts.get(category, 0) + 1 - - # ``count`` is "rows scored". For numeric / single-choice - # metrics that's the same as ``len(numeric) + sum(categories)`` - # because each scored row contributes exactly one observation. - # Multi-label parents however contribute one observation per - # selected child, so summing ``category_counts`` over-counts — - # we tracked rows-scored separately above and use it here. - rows_scored = ( - multi_label_rows_scored - if is_multi_label_parent - else len(numeric_values) + sum(category_counts.values()) - ) - - # Build numeric stats first, then categorical (both can coexist). - agg = CallImportMetricAggregate( - metric_id=metric_id_str, - metric_name=( - (meta.name if meta else observed_name) or "Unknown metric" - ), - metric_type=( - meta.metric_type if meta else observed_metric_type - ), - metric_category=( - "user_insight" - if meta is not None and _metric_is_user_insight(meta) - else "quality" - ) - or "quality", - is_multi_label_parent=is_multi_label_parent, - count=rows_scored, - skipped_count=skipped, - error_count=errored, - ) - if numeric_values: - agg.mean = float(statistics.fmean(numeric_values)) - agg.median = float(statistics.median(numeric_values)) - agg.min = min(numeric_values) - agg.max = max(numeric_values) - agg.stddev = ( - float(statistics.pstdev(numeric_values)) - if len(numeric_values) > 1 - else 0.0 - ) - agg.p25 = _percentile(numeric_values, 25) - agg.p75 = _percentile(numeric_values, 75) - agg.p95 = _percentile(numeric_values, 95) - agg.histogram_buckets = _build_histogram(numeric_values) - if category_counts: - sorted_counts = sorted( - category_counts.items(), key=lambda kv: kv[1], reverse=True - ) - agg.value_counts = [ - CallImportMetricValueCount(label=label, count=count) - for label, count in sorted_counts[:_TOP_VALUE_COUNTS] - ] - # Restrict the heatmap to pairs of labels we actually - # rendered above so the frontend never has to match - # against truncated/missing rows. Sorted desc by pair - # count to keep the most informative cells in the - # response when ``_TOP_VALUE_COUNTS`` clipped the matrix. - if is_multi_label_parent and multi_label_pair_counts: - kept_labels = { - label for label, _ in sorted_counts[:_TOP_VALUE_COUNTS] - } - pair_items = [ - (a, b, count) - for (a, b), count in multi_label_pair_counts.items() - if a in kept_labels and b in kept_labels - ] - pair_items.sort(key=lambda t: t[2], reverse=True) - agg.co_occurrence = [ - CallImportMetricLabelPair(a=a, b=b, count=count) - for a, b, count in pair_items - ] - - results.append(agg) - - # Sort so each parent metric immediately precedes its children. - # The Visualizations grid renders metrics top-to-bottom in this - # order, so multi-label parents (the "summary" chart) sit above - # the per-child boolean histograms that drill into them. Metrics - # whose ``meta`` row was deleted mid-run (``meta is None``) sink - # to the bottom but keep their relative order. - enumerated = list(enumerate(results)) - - def _sort_key(item: Tuple[int, CallImportMetricAggregate]): - original_idx, agg = item - meta = metric_meta.get(agg.metric_id) - if meta is None: - return (1, "", 1, "", original_idx) - parent_id = getattr(meta, "parent_metric_id", None) - # Group key: a child shares its parent's UUID; a parent - # uses its own UUID. Within a group, depth=0 (parent) sorts - # before depth=1 (child); ties break alphabetically by name - # so children render in a stable order regardless of which - # row scored which label first. - if parent_id is None: - group_key = str(meta.id) - depth = 0 - else: - group_key = str(parent_id) - depth = 1 - return ( - 0, - group_key, - depth, - (getattr(meta, "name", "") or "").lower(), - original_idx, - ) - - enumerated.sort(key=_sort_key) - return [agg for _idx, agg in enumerated] - - -@router.get( - "/{eval_id}/aggregate", - response_model=CallImportEvaluationAggregateResponse, - operation_id="getCallImportEvaluationAggregate", -) -async def get_call_import_evaluation_aggregate( - call_import_id: UUID, - eval_id: UUID, - baseline_evaluation_id: Optional[UUID] = Query(None), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationAggregateResponse: - """Return per-metric distributions for the Visualizations tab. - - The shape is intentionally chart-friendly: histograms for numeric - metrics, top-N value counts for categorical/text metrics, plus - summary stats (mean/p50/p95) so the UI can render summary cards - without recomputing on the client. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - eval_rows = _load_eval_rows(db, eval_id) - - metrics = _compute_metric_aggregates(db, evaluation, eval_rows) - - period_deltas: dict[str, MetricPeriodDelta] = {} - resolved_baseline_id: Optional[UUID] = None - if baseline_evaluation_id is not None: - call_import = _require_import(db, call_import_id, organization_id) - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - - rows = load_evaluation_row_pairs(db, eval_id) - period_start, _, _, _ = _report_period_from_rows(rows) - baseline_evaluation = _resolve_baseline_evaluation( - db, - organization_id, - call_import.workspace_id, - evaluation, - period_start, - str(baseline_evaluation_id), - ) - if baseline_evaluation: - resolved_baseline_id = baseline_evaluation.id - metric_aggregates_dicts = [ - _aggregate_to_dict(agg) for agg in metrics - ] - raw_deltas = _period_deltas_from_evaluation( - db, - baseline_evaluation, - metric_aggregates_dicts, - evaluation, - eval_rows, - ) - raw_deltas = _period_deltas_with_explanations( - db, - organization_id, - evaluation, - baseline_evaluation, - raw_deltas, - ) - period_deltas = { - metric_id: MetricPeriodDelta( - label=delta.get("label") or "", - detail=delta.get("detail") or "", - why=(delta.get("why") or "").strip() or None, - ) - for metric_id, delta in raw_deltas.items() - } - - _fp_stored, failure_policies_source = policies_from_evaluation_raw( - evaluation.metric_clusters - ) - return CallImportEvaluationAggregateResponse( - evaluation_id=eval_id, - total_rows=evaluation.total_rows, - completed_rows=evaluation.completed_rows, - failed_rows=evaluation.failed_rows, - metrics=metrics, - period_deltas=period_deltas, - baseline_evaluation_id=resolved_baseline_id, - failure_policies_source=failure_policies_source, - ) - - -# --------------------------------------------------------------------------- -# TLDR insights: LLM-generated narrative + bullet patterns rendered above -# the Visualizations charts. Cached on ``CallImportEvaluation.tldr_summary`` -# so the page never auto-burns LLM tokens; the user explicitly clicks -# "Generate summary" or "Regenerate" from the empty-state CTA. -# --------------------------------------------------------------------------- - - -_INSIGHTS_SYSTEM_PROMPT = ( - "You are a senior conversation-analytics reviewer. You will be " - "given aggregated metric statistics + a sample of rationales for " - "the rows of a single call-import evaluation. Identify the most " - "useful PATTERNS that hold ACROSS the calls -- not just per-metric " - "numbers. Look for combinations (e.g. `when X happens, Y also " - "tends to happen`), notable outliers, frequent failure modes, and " - "any signal that would change how a reviewer triages the run.\n\n" - "Return STRICT JSON only, with this shape and no extra keys:\n" - "{\n" - ' "narrative": "",\n' - ' "patterns": ["", "", ...],\n' - ' "metric_insights": {"": "<2-3 line business meaning>"}\n' - "}\n\n" - "Constraints:\n" - "- narrative is the ONLY text shown in the external audit summary and " - "Visualizations TLDR; keep it to at most 3 short sentences (~300 chars).\n" - "- patterns are optional supporting notes and are NOT rendered in the " - "audit summary; keep 0 to 3 bullets if supplied, each <= 120 characters.\n" - "- metric_insights must include one entry for each top-level metric id supplied.\n" - "- Each metric insight should explain what the metric means for the business and what the current distribution suggests, not restate the metric rubric.\n" - "- Avoid restating raw counts unless they reveal a pattern.\n" - "- Use neutral, factual language ('frustration appeared in...') " - "rather than judgemental ('the agents failed to...')." -) - - -def _tldr_summary_payload( - evaluation: CallImportEvaluation, -) -> Optional[EvaluationTldrSummary]: - """Return the cached TLDR (with ``is_stale`` set) or ``None``. - - ``CallImportEvaluation.tldr_summary`` is a ``JSON`` column so we - have to validate shape defensively -- a half-written or hand-edited - blob should not break the aggregate response. Returns ``None`` when - no cached summary exists. - """ - raw = evaluation.tldr_summary - if not isinstance(raw, dict): - return None - narrative = raw.get("narrative") - if not isinstance(narrative, str) or not narrative.strip(): - return None - patterns_raw = raw.get("patterns") - patterns = ( - [str(p) for p in patterns_raw if isinstance(p, str) and p.strip()] - if isinstance(patterns_raw, list) - else [] - ) - metric_insights_raw = raw.get("metric_insights") - metric_insights = ( - { - str(metric_id): str(insight).strip() - for metric_id, insight in metric_insights_raw.items() - if str(metric_id).strip() - and isinstance(insight, str) - and insight.strip() - } - if isinstance(metric_insights_raw, dict) - else {} - ) - generated_at_raw = raw.get("generated_at") - try: - generated_at = ( - datetime.fromisoformat(generated_at_raw) - if isinstance(generated_at_raw, str) - else evaluation.updated_at or datetime.now(timezone.utc) - ) - except ValueError: - generated_at = evaluation.updated_at or datetime.now(timezone.utc) - snapshot = raw.get("generated_at_completed_rows") - snapshot_int = int(snapshot) if isinstance(snapshot, (int, float)) else 0 - return EvaluationTldrSummary( - narrative=_clamp_prose_to_sentences(narrative.strip()), - patterns=patterns, - metric_insights=metric_insights, - generated_at=generated_at, - generated_at_completed_rows=snapshot_int, - provider=raw.get("provider") if isinstance(raw.get("provider"), str) else None, - model=raw.get("model") if isinstance(raw.get("model"), str) else None, - is_stale=evaluation.completed_rows > snapshot_int, - ) - - -def _sample_rationales_per_metric( - eval_rows: List[CallImportEvaluationRow], - *, - per_metric_cap: int = 3, - rationale_char_cap: int = 600, -) -> Dict[str, List[str]]: - """Collect up to ``per_metric_cap`` distinct rationales per metric. - - Distinctness is case- and whitespace-insensitive. We truncate each - rationale to ``rationale_char_cap`` so a few unusually verbose rows - can't dominate the prompt budget. Empty / non-string rationales are - skipped. - """ - out: Dict[str, List[str]] = {} - seen: Dict[str, set[str]] = {} - for row in eval_rows: - scores = row.metric_scores if isinstance(row.metric_scores, dict) else {} - for metric_id, entry in scores.items(): - if not isinstance(entry, dict): - continue - rationale = entry.get("rationale") - if not isinstance(rationale, str): - continue - text = rationale.strip() - if not text: - continue - bucket = out.setdefault(metric_id, []) - if len(bucket) >= per_metric_cap: - continue - key = " ".join(text.lower().split()) - seen_set = seen.setdefault(metric_id, set()) - if key in seen_set: - continue - seen_set.add(key) - bucket.append(text[:rationale_char_cap]) - return out - - -def _build_insights_messages( - evaluation: CallImportEvaluation, - aggregate: List[CallImportMetricAggregate], - rationale_samples: Dict[str, List[str]], - metric_meta: Dict[str, Metric], -) -> List[Dict[str, str]]: - """Render the user prompt fed to the LLM. - - The shape is plain markdown-ish text instead of JSON so the LLM can - skim it without us spending tokens on verbose schema delimiters. - Parent metrics surface their child metrics nested underneath so the - model sees the hierarchy and can talk about "X often co-occurred - with Y" rather than treating sub-labels as standalone metrics. - """ - name = evaluation.name or f"Run {str(evaluation.id)[:8]}" - lines: List[str] = [ - f"Evaluation: {name}", - ( - f"Rows: total={evaluation.total_rows} " - f"completed={evaluation.completed_rows} " - f"failed={evaluation.failed_rows}" - ), - "", - "## Per-metric aggregate", - ] - - # Group metrics by parent so the prompt mirrors the hierarchy. Any - # aggregate row whose ``metric_id`` is missing from ``metric_meta`` - # is rendered as a leaf at the top-level list (handles renamed / - # deleted parents). - children_by_parent: Dict[str, List[CallImportMetricAggregate]] = {} - top_level: List[CallImportMetricAggregate] = [] - for agg in aggregate: - meta = metric_meta.get(agg.metric_id) - parent_id = ( - str(meta.parent_metric_id) - if meta is not None and getattr(meta, "parent_metric_id", None) - else None - ) - if parent_id: - children_by_parent.setdefault(parent_id, []).append(agg) - else: - top_level.append(agg) - - def _format_metric_block(agg: CallImportMetricAggregate, indent: int) -> List[str]: - prefix = " " * indent + "- " - bits: List[str] = [f"{prefix}{agg.metric_name} [id={agg.metric_id}] (n={agg.count}"] - if agg.skipped_count: - bits.append(f", skipped={agg.skipped_count}") - if agg.error_count: - bits.append(f", errors={agg.error_count}") - bits.append(")") - meta = metric_meta.get(agg.metric_id) - description = (meta.description or "").strip() if meta else "" - if description: - bits.append(f" | definition={description[:500]}") - if agg.mean is not None: - mean_s = f"{agg.mean:.2f}" - stddev_s = f"{agg.stddev:.2f}" if agg.stddev is not None else "-" - bits.append(f" | mean={mean_s} stddev={stddev_s}") - if agg.min is not None and agg.max is not None: - bits.append(f" range=[{agg.min:.2f}, {agg.max:.2f}]") - if agg.value_counts: - total = sum(v.count for v in agg.value_counts) or 1 - top = agg.value_counts[:3] - shares = ", ".join( - f'"{v.label}"={v.count}/{total}' for v in top - ) - bits.append(f" | top={shares}") - result = ["".join(bits)] - rationales = rationale_samples.get(agg.metric_id, []) - for r in rationales: - result.append(" " * (indent + 1) + f"- rationale: {r}") - return result - - for agg in top_level: - lines.extend(_format_metric_block(agg, indent=0)) - meta = metric_meta.get(agg.metric_id) - children = children_by_parent.get(str(meta.id), []) if meta else [] - for child in children: - lines.extend(_format_metric_block(child, indent=1)) - - lines.append("") - top_level_ids = [agg.metric_id for agg in top_level] - if top_level_ids: - lines.append( - "metric_insights keys must exactly use these top-level metric ids: " - + ", ".join(top_level_ids) - ) - lines.append("") - lines.append( - "Write the JSON object as instructed. Do not include " - "preamble, code fences, or trailing commentary." - ) - - return [ - {"role": "system", "content": _INSIGHTS_SYSTEM_PROMPT}, - {"role": "user", "content": "\n".join(lines)}, - ] - - -def _parse_insights_response(text: str) -> EvaluationTldrSummary: - """Coerce the LLM response into ``narrative`` + ``patterns``. - - Matches the JSON-with-fallback pattern used by - ``app.api.v1.routes.metrics._parse_metric_generation_response``: try - ``json.loads`` first, then fall back to regex extraction of the - first ``{...}`` block. Raises ``HTTPException`` with a 502 when the - response can't be parsed at all. - """ - cleaned = (text or "").strip() - if not cleaned: - raise HTTPException( - status_code=502, detail="LLM returned an empty insights response" - ) - try: - parsed = json.loads(cleaned) - except json.JSONDecodeError: - import re - - match = re.search(r"\{.*\}", cleaned, re.DOTALL) - if not match: - raise HTTPException( - status_code=502, - detail="Could not parse LLM insights response as JSON", - ) - try: - parsed = json.loads(match.group(0)) - except json.JSONDecodeError as e: - raise HTTPException( - status_code=502, - detail=f"Could not parse LLM insights response: {e}", - ) - - if not isinstance(parsed, dict): - raise HTTPException( - status_code=502, detail="LLM insights JSON was not an object" - ) - - narrative = parsed.get("narrative") - if not isinstance(narrative, str) or not narrative.strip(): - raise HTTPException( - status_code=502, - detail="LLM insights JSON missing 'narrative' string", - ) - - patterns_raw = parsed.get("patterns") - if patterns_raw is None: - patterns: List[str] = [] - elif isinstance(patterns_raw, list): - patterns = [ - str(p).strip() - for p in patterns_raw - if isinstance(p, str) and p.strip() - ] - else: - raise HTTPException( - status_code=502, - detail="LLM insights JSON 'patterns' must be a list of strings", - ) - metric_insights_raw = parsed.get("metric_insights") - if metric_insights_raw is None: - metric_insights: Dict[str, str] = {} - elif isinstance(metric_insights_raw, dict): - metric_insights = { - str(metric_id): str(insight).strip() - for metric_id, insight in metric_insights_raw.items() - if str(metric_id).strip() - and isinstance(insight, str) - and insight.strip() - } - else: - raise HTTPException( - status_code=502, - detail="LLM insights JSON 'metric_insights' must be an object", - ) - - return EvaluationTldrSummary( - narrative=_clamp_prose_to_sentences(narrative.strip()), - patterns=patterns, - metric_insights=metric_insights, - generated_at=datetime.now(timezone.utc), - generated_at_completed_rows=0, # filled in by caller - is_stale=False, - ) - - -def _generate_and_persist_tldr_summary( - db: Session, - evaluation: CallImportEvaluation, - *, - organization_id: UUID, - provider: Optional[str] = None, - model: Optional[str] = None, -) -> EvaluationTldrSummary: - """LLM TLDR generation used by the imports-queue Celery worker.""" - eval_id = evaluation.id - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - - pairs = load_evaluation_row_pairs(db, eval_id) - eval_rows = [eval_row for eval_row, _ in pairs] - aggregate = _compute_metric_aggregates(db, evaluation, eval_rows) - if not aggregate: - raise HTTPException( - status_code=400, - detail=( - "No metric data yet. Wait for at least one row to " - "finish scoring before generating a summary." - ), - ) - - metric_ids: List[UUID] = [] - for agg in aggregate: - try: - metric_ids.append(UUID(agg.metric_id)) - except (TypeError, ValueError): - continue - metrics = _metrics_for_ids(db, organization_id, metric_ids) - metric_meta: Dict[str, Metric] = {str(m.id): m for m in metrics} - - rationale_samples = _sample_rationales_per_metric(eval_rows) - messages = _build_insights_messages( - evaluation, aggregate, rationale_samples, metric_meta - ) - - from app.services.ai.llm_resolver import get_llm_provider_and_model - from app.services.ai.llm_service import llm_service - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, provider, model - ) - - try: - llm_result = llm_service.generate_response( - messages=messages, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - temperature=0.4, - max_tokens=1400, - ) - except Exception as e: - logger.error(f"[CallImportInsights] LLM call failed: {e}") - raise HTTPException( - status_code=502, detail=f"LLM call failed: {e}" - ) from e - - summary = _parse_insights_response(llm_result.get("text", "")) - total = int(evaluation.total_rows or 0) - ui_completed = min(int(evaluation.completed_rows or 0), total) if total else int( - evaluation.completed_rows or 0 - ) - summary.generated_at_completed_rows = ui_completed - summary.provider = provider_enum.value - summary.model = model_str - summary.is_stale = False - - evaluation.tldr_summary = { - "narrative": summary.narrative, - "patterns": summary.patterns, - "metric_insights": summary.metric_insights, - "generated_at": summary.generated_at.isoformat(), - "generated_at_completed_rows": summary.generated_at_completed_rows, - "provider": summary.provider, - "model": summary.model, - } - flag_modified(evaluation, "tldr_summary") - db.commit() - db.refresh(evaluation) - return summary - - -@router.get( - "/{eval_id}/insights", - response_model=Optional[EvaluationTldrSummary], - operation_id="getCallImportEvaluationInsights", -) -async def get_call_import_evaluation_insights( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Optional[EvaluationTldrSummary]: - """Return the cached TLDR (or ``null``) without contacting the LLM. - - Used by the Visualizations tab on first paint so the empty-state - CTA can show up before the user opts into generation. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - return _tldr_summary_payload(evaluation) - - -@router.post( - "/{eval_id}/insights", - response_model=EvaluationTldrSummary, - operation_id="generateCallImportEvaluationInsights", -) -async def generate_call_import_evaluation_insights( - call_import_id: UUID, - eval_id: UUID, - body: EvaluationInsightsRequest = Body(default_factory=EvaluationInsightsRequest), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> EvaluationTldrSummary: - """Generate (or return-cached) the LLM TLDR for an evaluation run. - - Behavior: - - * ``body.regenerate=False`` and a cached summary at the current - ``completed_rows`` watermark exists -> return it as-is. - * ``body.regenerate=False`` and a stale cached summary exists - (``generated_at_completed_rows < completed_rows``) -> return it - with ``is_stale=True``; the UI prompts the user to regenerate. - * Otherwise -> resolve provider+model (auto-detect when omitted), - call the LLM, persist the new summary, return it. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - if not body.regenerate: - cached = _tldr_summary_payload(evaluation) - if cached is not None: - return cached - - # Run the TLDR LLM on the imports worker (not the default worker or API). - from app.workers.tasks.generate_evaluation_tldr_insights import ( - generate_evaluation_tldr_insights_task, - ) - - try: - task_result = generate_evaluation_tldr_insights_task.apply_async( - kwargs={ - "evaluation_id": str(eval_id), - "call_import_id": str(call_import_id), - "organization_id": str(organization_id), - "provider": body.provider, - "model": body.model, - }, - ).get(timeout=25 * 60) - except Exception as exc: - logger.error( - "[CallImportInsights] TLDR task failed for evaluation {}: {}", - eval_id, - exc, - ) - raise HTTPException( - status_code=502, - detail=f"Summary generation failed: {exc}", - ) from exc - - if isinstance(task_result, dict) and task_result.get("error"): - status_code = int(task_result.get("status_code") or 502) - raise HTTPException( - status_code=status_code, - detail=str(task_result["error"]), - ) - - summary = EvaluationTldrSummary.model_validate(task_result) - db.refresh(evaluation) - - from app.services.ai.llm_resolver import get_llm_provider_and_model - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, body.provider, body.model, body.credential_id - ) - - _enqueue_user_insights_job( - evaluation, - provider=summary.provider or provider_enum.value, - model=summary.model or model_str, - force=body.regenerate, - max_llm_calls=body.max_llm_calls, - db=db, - ) - - return summary - - -def _user_insights_payload( - evaluation: CallImportEvaluation, -) -> Optional[EvaluationUserInsightsState]: - raw = getattr(evaluation, "user_insights", None) - if raw is None: - return None - return user_insights_state_from_raw( - raw, - completed_rows=evaluation.completed_rows, - ) - - -def _selected_generated_user_insights( - state: Optional[EvaluationUserInsightsState], - report_config: dict[str, Any], -) -> list[dict[str, Any]]: - """Filter and order generated insights for PDF section 03.""" - if state is None or state.status != "completed" or not state.insights: - return [] - - selected_ids = report_config.get("user_insight_ids") - if isinstance(selected_ids, list) and selected_ids: - allowed = {str(item) for item in selected_ids if item} - items = [item for item in state.insights if item.id in allowed] - else: - items = list(state.insights) - - order_raw = report_config.get("order") - order_ids: list[str] = [] - if isinstance(order_raw, dict): - user_order = order_raw.get("user_insights") - if isinstance(user_order, list): - order_ids = [str(item) for item in user_order if item] - - if order_ids: - by_id = {item.id: item for item in items} - ordered = [by_id[iid] for iid in order_ids if iid in by_id] - seen = set(order_ids) - ordered.extend(item for item in items if item.id not in seen) - items = ordered - - return [item.model_dump(mode="json") for item in items] - - -def _enqueue_user_insights_job( - evaluation: CallImportEvaluation, - *, - provider: Optional[str] = None, - model: Optional[str] = None, - force: bool = False, - max_llm_calls: Optional[int] = None, - db: Optional[Session] = None, -) -> None: - """Enqueue background user-insights generation unless already running.""" - current = _user_insights_payload(evaluation) - if current is not None and current.status == "running" and not force: - return - - llm_budget = normalize_max_llm_calls(max_llm_calls) - - completed_count = ( - _count_completed_eval_rows(db, evaluation.id) - if db is not None - else evaluation.completed_rows - ) - total_calls = total_llm_calls_for_rows(completed_count, max_llm_calls=llm_budget) - evaluation.user_insights = { - "status": "running", - "insights": ( - (evaluation.user_insights or {}).get("insights", []) - if isinstance(evaluation.user_insights, dict) - else [] - ), - "generated_at": datetime.now(timezone.utc).isoformat(), - "generated_at_completed_rows": evaluation.completed_rows, - "progress": {"completed_llm_calls": 0, "total_llm_calls": total_calls}, - "provider": provider, - "model": model, - "max_llm_calls": llm_budget, - "llm_calls_used": 0, - "error_message": None, - } - if db is not None: - flag_modified(evaluation, "user_insights") - db.commit() - - from app.workers.tasks.generate_evaluation_user_insights import ( - generate_evaluation_user_insights_task, - ) - - generate_evaluation_user_insights_task.delay( - str(evaluation.id), - provider=provider, - model=model, - max_llm_calls=llm_budget, - ) - - -@router.get( - "/{eval_id}/user-insights", - response_model=Optional[EvaluationUserInsightsState], - operation_id="getCallImportEvaluationUserInsights", -) -async def get_call_import_evaluation_user_insights( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Optional[EvaluationUserInsightsState]: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - return _user_insights_payload(evaluation) - - -@router.post( - "/{eval_id}/user-insights", - response_model=EvaluationUserInsightsState, - operation_id="generateCallImportEvaluationUserInsights", -) -async def generate_call_import_evaluation_user_insights( - call_import_id: UUID, - eval_id: UUID, - body: EvaluationUserInsightsRequest = Body( - default_factory=EvaluationUserInsightsRequest - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> EvaluationUserInsightsState: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - if not body.regenerate and not body.force: - cached = _user_insights_payload(evaluation) - if cached is not None and cached.status in {"running", "completed"}: - return cached - - eval_rows = _load_eval_rows(db, eval_id) - if not any(row.status == "completed" for row in eval_rows): - raise HTTPException( - status_code=400, - detail=( - "No completed rows yet. Wait for at least one row to " - "finish scoring before generating user insights." - ), - ) - - from app.services.ai.llm_resolver import get_llm_provider_and_model - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, body.provider, body.model, body.credential_id - ) - - _enqueue_user_insights_job( - evaluation, - provider=provider_enum.value, - model=model_str, - force=body.force or body.regenerate, - max_llm_calls=body.max_llm_calls, - db=db, - ) - - db.refresh(evaluation) - return _user_insights_payload(evaluation) or EvaluationUserInsightsState( - status="running" - ) - - -def _metric_clusters_payload( - evaluation: CallImportEvaluation, -) -> Optional[EvaluationMetricClustersState]: - raw = getattr(evaluation, "metric_clusters", None) - if raw is None: - return None - return metric_clusters_state_from_raw( - raw, - completed_rows=evaluation.completed_rows, - ) - - -def _selected_metric_clusters_for_pdf( - state: Optional[EvaluationMetricClustersState], - report_config: dict[str, Any], -) -> dict[str, Any]: - if state is None or state.status != "completed": - return {} - sections = report_config.get("sections") - if isinstance(sections, dict) and sections.get("failure_diagnostics") is False: - return {} - payload: dict[str, Any] = { - "groups": [g.model_dump(mode="json") for g in state.groups], - "discovered_problems": [ - d.model_dump(mode="json") for d in state.discovered_problems - ], - } - if state.rca_summary is not None: - payload["rca_summary"] = state.rca_summary.model_dump(mode="json") - return payload - - -def _prompt_improvements_payload( - evaluation: CallImportEvaluation, -) -> Optional[EvaluationPromptImprovementsState]: - from app.services.call_import_prompt_improvements import ( - prompt_improvements_state_from_raw, - ) - - raw = getattr(evaluation, "prompt_improvements", None) - if raw is None: - return None - return prompt_improvements_state_from_raw( - raw, - completed_rows=evaluation.completed_rows, - ) - - -def _selected_prompt_improvements_for_pdf( - state: Optional[EvaluationPromptImprovementsState], - report_config: dict[str, Any], -) -> dict[str, Any]: - if state is None or state.status != "completed": - return {} - sections = report_config.get("sections") - if isinstance(sections, dict) and sections.get("prompt_improvements") is False: - return {} - return { - "imported_agent_id": state.imported_agent_id, - "imported_agent_name": state.imported_agent_name, - "overview": state.overview, - "suggestions": [s.model_dump(mode="json") for s in state.suggestions], - } - - -def _enqueue_prompt_improvements_job( - evaluation: CallImportEvaluation, - *, - imported_agent_id: UUID, - imported_agent_name: str, - provider: Optional[str] = None, - model: Optional[str] = None, - credential_id: Optional[UUID] = None, - force: bool = False, - db: Optional[Session] = None, -) -> None: - current = _prompt_improvements_payload(evaluation) - if current is not None and current.status == "running" and not force: - return - - evaluation.prompt_improvements = { - "status": "running", - "imported_agent_id": str(imported_agent_id), - "imported_agent_name": imported_agent_name, - "suggestions": [], - "generated_at": datetime.now(timezone.utc).isoformat(), - "generated_at_completed_rows": evaluation.completed_rows, - "provider": provider, - "model": model, - "error_message": None, - } - if db is not None: - flag_modified(evaluation, "prompt_improvements") - db.commit() - - from app.workers.tasks.generate_evaluation_prompt_improvements import ( - generate_evaluation_prompt_improvements_task, - ) - - async_result = generate_evaluation_prompt_improvements_task.apply_async( - kwargs={ - "evaluation_id": str(evaluation.id), - "imported_agent_id": str(imported_agent_id), - "provider": provider, - "model": model, - "credential_id": str(credential_id) if credential_id else None, - }, - queue="imports", - ) - if db is not None and isinstance(evaluation.prompt_improvements, dict): - evaluation.prompt_improvements["celery_task_id"] = async_result.id - flag_modified(evaluation, "prompt_improvements") - db.commit() - - -def _load_eval_rows(db: Session, evaluation_id: UUID) -> List[CallImportEvaluationRow]: - from app.db_sharding.eval_rows import load_evaluation_rows_for_run - - return load_evaluation_rows_for_run(db, evaluation_id) - - -def _count_completed_eval_rows(db: Session, evaluation_id: UUID) -> int: - from app.db_sharding.eval_rows import count_evaluation_rows_for_run - - return count_evaluation_rows_for_run( - db, evaluation_id, statuses=["completed"] - ) - - -def _completed_row_pairs_for_evaluation( - db: Session, - evaluation_id: UUID, -) -> List[Tuple[CallImportEvaluationRow, CallImportRow]]: - from app.db_sharding.scatter_gather import load_evaluation_row_pairs - - row_pairs = load_evaluation_row_pairs(db, evaluation_id) - return [ - (eval_row, source_row) - for eval_row, source_row in row_pairs - if eval_row.status == "completed" - ] - - -def _resolve_metric_cluster_row_selection( - db: Session, - evaluation: CallImportEvaluation, - eval_rows: List[CallImportEvaluationRow], - evaluation_row_ids: Optional[List[UUID]], - *, - row_limit: Optional[int] = None, - policies: Optional[Dict[str, MetricFailurePolicy]] = None, -) -> Tuple[List[Tuple[CallImportEvaluationRow, CallImportRow]], List[str]]: - """Return filtered completed row pairs and the selected row id strings.""" - completed_pairs = _completed_row_pairs_for_evaluation(db, evaluation.id) - metrics = _metrics_for_clustering(db, evaluation, eval_rows) - if policies is None: - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - parent_ids = [ - m.id - for m in metrics - if getattr(m, "selection_mode", None) - and not getattr(m, "parent_metric_id", None) - ] - child_names_by_parent = _child_names_by_parent( - db, evaluation.organization_id, parent_ids - ) - policies, _ = effective_policies( - evaluation, - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - ) - eligible = list_eligible_cluster_rows( - evaluation, completed_pairs, metrics, policies - ) - eligible_ordered_ids = [str(item["evaluation_row_id"]) for item in eligible] - eligible_id_set = set(eligible_ordered_ids) - - if evaluation_row_ids is None and row_limit is not None: - selected_ids = eligible_ordered_ids[:row_limit] - filtered = filter_completed_row_pairs( - completed_pairs, - [UUID(rid) for rid in selected_ids], - ) - return filtered, selected_ids - - if evaluation_row_ids is None: - selected_ids = eligible_ordered_ids - filtered = filter_completed_row_pairs( - completed_pairs, - [UUID(rid) for rid in selected_ids], - ) - return filtered, selected_ids - - requested = {str(rid) for rid in evaluation_row_ids} - completed_id_set = {str(eval_row.id) for eval_row, _ in completed_pairs} - unknown = sorted(requested - completed_id_set) - if unknown: - raise HTTPException( - status_code=400, - detail=( - "One or more evaluation_row_ids are missing or not completed: " - + ", ".join(unknown[:5]) - + ("…" if len(unknown) > 5 else "") - ), - ) - not_eligible = sorted(requested - eligible_id_set) - if not_eligible: - raise HTTPException( - status_code=400, - detail=( - "Each selected row must have at least one flagged quality metric. " - "Ineligible row(s): " - + ", ".join(not_eligible[:5]) - + ("…" if len(not_eligible) > 5 else "") - ), - ) - selected_ids = sorted(requested) - filtered = filter_completed_row_pairs(completed_pairs, evaluation_row_ids) - return filtered, selected_ids - - -def _enqueue_metric_clusters_job( - evaluation: CallImportEvaluation, - *, - provider: Optional[str] = None, - model: Optional[str] = None, - credential_id: Optional[UUID] = None, - force: bool = False, - max_llm_calls: Optional[int] = None, - evaluation_row_ids: Optional[List[UUID]] = None, - selected_evaluation_row_ids: Optional[List[str]] = None, - failure_policies: Optional[Dict[str, MetricFailurePolicy]] = None, - db: Optional[Session] = None, -) -> None: - current = _metric_clusters_payload(evaluation) - if current is not None and current.status == "running" and not force: - return - - llm_budget = normalize_max_llm_calls(max_llm_calls) - total_calls = 1 - row_ids_for_task: Optional[List[str]] = None - if db is not None: - eval_rows = _load_eval_rows(db, evaluation.id) - if selected_evaluation_row_ids is None: - _, selected_evaluation_row_ids = _resolve_metric_cluster_row_selection( - db, - evaluation, - eval_rows, - evaluation_row_ids, - ) - completed_pairs = filter_completed_row_pairs( - _completed_row_pairs_for_evaluation(db, evaluation.id), - [UUID(rid) for rid in selected_evaluation_row_ids], - ) - metrics = _metrics_for_clustering(db, evaluation, eval_rows) - policies_for_estimate = failure_policies - if policies_for_estimate is None: - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - parent_ids = [ - m.id - for m in metrics - if getattr(m, "selection_mode", None) - and not getattr(m, "parent_metric_id", None) - ] - child_names_by_parent = _child_names_by_parent( - db, evaluation.organization_id, parent_ids - ) - policies_for_estimate, _ = effective_policies( - evaluation, - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - ) - _, total_calls = estimate_metric_clusters_llm_calls( - evaluation, - metrics, - completed_pairs, - policies_for_estimate, - max_llm_calls=llm_budget, - ) - row_ids_for_task = list(selected_evaluation_row_ids) - - prior_raw = ( - evaluation.metric_clusters - if isinstance(evaluation.metric_clusters, dict) - else {} - ) - policy_blob: Dict[str, Any] = {} - if failure_policies: - policy_blob = failure_policies_to_db(failure_policies, source="user") - - evaluation.metric_clusters = { - "status": "running", - "groups": prior_raw.get("groups", []) if isinstance(prior_raw, dict) else [], - "discovered_problems": ( - prior_raw.get("discovered_problems", []) - if isinstance(prior_raw, dict) - else [] - ), - "generated_at": datetime.now(timezone.utc).isoformat(), - "generated_at_completed_rows": evaluation.completed_rows, - "progress": {"completed_llm_calls": 0, "total_llm_calls": total_calls}, - "provider": provider, - "model": model, - "max_llm_calls": llm_budget, - "llm_calls_used": 0, - "error_message": None, - "selected_evaluation_row_ids": selected_evaluation_row_ids or [], - **policy_blob, - } - if db is not None: - flag_modified(evaluation, "metric_clusters") - db.commit() - - from app.workers.tasks.generate_evaluation_metric_clusters import ( - generate_evaluation_metric_clusters_task, - ) - - async_result = generate_evaluation_metric_clusters_task.apply_async( - kwargs={ - "evaluation_id": str(evaluation.id), - "provider": provider, - "model": model, - "credential_id": str(credential_id) if credential_id else None, - "max_llm_calls": llm_budget, - "evaluation_row_ids": row_ids_for_task, - }, - queue="imports", - ) - if db is not None and isinstance(evaluation.metric_clusters, dict): - evaluation.metric_clusters["celery_task_id"] = async_result.id - flag_modified(evaluation, "metric_clusters") - db.commit() - - -def _revoke_metric_clusters_task(evaluation: CallImportEvaluation) -> None: - """Best-effort SIGTERM revoke of the in-flight clustering Celery task.""" - raw = evaluation.metric_clusters - if not isinstance(raw, dict): - return - task_id = str(raw.get("celery_task_id") or "").strip() - if not task_id: - return - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke(task_id, terminate=True, signal="SIGTERM") - logger.info( - "Revoked metric-clusters task {} for evaluation {}", - task_id, - evaluation.id, - ) - except Exception as exc: # noqa: BLE001 - logger.warning( - "Failed to revoke metric-clusters task {} for evaluation {}: {}", - task_id, - evaluation.id, - exc, - ) - - -def _apply_metric_clusters_cancel(evaluation: CallImportEvaluation) -> bool: - """Mark clustering as cancelled and revoke the worker task. - - Returns True if a running job was cancelled, False if already terminal. - """ - raw = evaluation.metric_clusters - if not isinstance(raw, dict): - return False - if (raw.get("status") or "").lower() != "running": - return False - - _revoke_metric_clusters_task(evaluation) - progress = raw.get("progress") if isinstance(raw.get("progress"), dict) else {} - evaluation.metric_clusters = { - **raw, - "status": "cancelled", - "error_message": METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, - "progress": progress, - "celery_task_id": None, - } - return True - - -@router.get( - "/{eval_id}/metric-clusters/failure-policies", - response_model=MetricFailurePoliciesResponse, - operation_id="getCallImportEvaluationMetricClusterFailurePolicies", -) -async def get_call_import_evaluation_metric_cluster_failure_policies( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> MetricFailurePoliciesResponse: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - eval_rows = _load_eval_rows(db, eval_id) - metrics, aggregates, policies, source, child_names_by_parent = _clustering_context( - db, evaluation, eval_rows - ) - previews = build_failure_policy_previews( - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - effective=policies, - ) - updated_at = None - raw_mc = evaluation.metric_clusters - if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): - try: - updated_at = datetime.fromisoformat( - str(raw_mc["failure_policies_updated_at"]) - ) - except ValueError: - updated_at = None - return MetricFailurePoliciesResponse( - previews=previews, - policies=policies, - source=source, - updated_at=updated_at, - ) - - -@router.put( - "/{eval_id}/metric-clusters/failure-policies", - response_model=MetricFailurePoliciesResponse, - operation_id="saveCallImportEvaluationMetricClusterFailurePolicies", -) -async def save_call_import_evaluation_metric_cluster_failure_policies( - call_import_id: UUID, - eval_id: UUID, - body: MetricFailurePoliciesSaveRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> MetricFailurePoliciesResponse: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - eval_rows = _load_eval_rows(db, eval_id) - metrics, aggregates, _existing, _source, child_names_by_parent = _clustering_context( - db, evaluation, eval_rows - ) - try: - validate_failure_policies_for_metrics(body.policies, metrics) - except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc - - prior = ( - evaluation.metric_clusters - if isinstance(evaluation.metric_clusters, dict) - else {} - ) - evaluation.metric_clusters = merge_failure_policies_into_raw( - prior, - body.policies, - source="user", - ) - flag_modified(evaluation, "metric_clusters") - db.commit() - db.refresh(evaluation) - - policies, source = policies_from_evaluation_raw(evaluation.metric_clusters) - if source != "user": - source = "user" - previews = build_failure_policy_previews( - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - effective=policies, - ) - updated_at = None - raw_mc = evaluation.metric_clusters - if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): - try: - updated_at = datetime.fromisoformat( - str(raw_mc["failure_policies_updated_at"]) - ) - except ValueError: - updated_at = None - return MetricFailurePoliciesResponse( - previews=previews, - policies=policies, - source="user", - updated_at=updated_at, - ) - - -@router.get( - "/{eval_id}/metric-clusters/eligible-rows", - response_model=MetricClusterEligibleRowsResponse, - operation_id="listCallImportEvaluationMetricClusterEligibleRows", -) -async def list_call_import_evaluation_metric_cluster_eligible_rows( - call_import_id: UUID, - eval_id: UUID, - limit: Optional[int] = Query(default=None, ge=1), - count_only: bool = Query(default=False), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> MetricClusterEligibleRowsResponse: - """Completed rows that have at least one flagged quality metric.""" - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - eval_rows = _load_eval_rows(db, eval_id) - completed_pairs = _completed_row_pairs_for_evaluation(db, eval_id) - metrics, _aggregates, policies, _source, _child_map = _clustering_context( - db, evaluation, eval_rows - ) - all_eligible = list_eligible_cluster_rows( - evaluation, completed_pairs, metrics, policies - ) - total = len(all_eligible) - if count_only: - return MetricClusterEligibleRowsResponse(items=[], total=total) - raw_items = all_eligible if limit is None else all_eligible[:limit] - items = [MetricClusterEligibleRow.model_validate(item) for item in raw_items] - return MetricClusterEligibleRowsResponse(items=items, total=total) - - -@router.get( - "/{eval_id}/metric-clusters", - response_model=Optional[EvaluationMetricClustersState], - operation_id="getCallImportEvaluationMetricClusters", -) -async def get_call_import_evaluation_metric_clusters( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Optional[EvaluationMetricClustersState]: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - return _metric_clusters_payload(evaluation) - - -@router.post( - "/{eval_id}/metric-clusters", - response_model=EvaluationMetricClustersState, - operation_id="generateCallImportEvaluationMetricClusters", -) -async def generate_call_import_evaluation_metric_clusters( - call_import_id: UUID, - eval_id: UUID, - body: EvaluationMetricClustersRequest = Body( - default_factory=EvaluationMetricClustersRequest - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> EvaluationMetricClustersState: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - if not body.regenerate and not body.force: - cached = _metric_clusters_payload(evaluation) - if cached is not None and cached.status in {"running", "completed"}: - return cached - - eval_rows = _load_eval_rows(db, eval_id) - if not any(row.status == "completed" for row in eval_rows): - raise HTTPException( - status_code=400, - detail=( - "No completed rows yet. Wait for at least one row to " - "finish scoring before generating metric clusters." - ), - ) - - if body.evaluation_row_ids and body.row_limit is not None: - raise HTTPException( - status_code=400, - detail="Specify either evaluation_row_ids or row_limit, not both.", - ) - - if body.evaluation_row_ids: - completed_pairs = _completed_row_pairs_for_evaluation(db, evaluation.id) - completed_id_set = {str(eval_row.id) for eval_row, _ in completed_pairs} - requested = {str(rid) for rid in body.evaluation_row_ids} - unknown = sorted(requested - completed_id_set) - if unknown: - raise HTTPException( - status_code=400, - detail=( - "One or more evaluation_row_ids are missing or not completed: " - + ", ".join(unknown[:5]) - + ("…" if len(unknown) > 5 else "") - ), - ) - - from app.services.ai.llm_resolver import get_llm_provider_and_model - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, body.provider, body.model, body.credential_id - ) - - metrics, aggregates, _inferred, _source, child_names_by_parent = _clustering_context( - db, evaluation, eval_rows - ) - merged_policies = merge_clustering_policies( - body.failure_policies, - evaluation, - metrics, - aggregates, - child_names_by_parent=child_names_by_parent, - ) - try: - validate_failure_policies_for_metrics( - body.failure_policies or merged_policies, metrics - ) - except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc - - if not has_clusterable_metrics(metrics, merged_policies, eval_rows): - raise HTTPException( - status_code=400, - detail=( - "No calls match any failure policy. Select failure values on " - "metrics that have matching rows, or leave metrics with no " - "failures unchecked — they are skipped automatically." - ), - ) - - filtered_pairs, selected_row_ids = _resolve_metric_cluster_row_selection( - db, - evaluation, - eval_rows, - body.evaluation_row_ids, - row_limit=body.row_limit, - policies=merged_policies, - ) - if not selected_row_ids: - raise HTTPException( - status_code=400, - detail=( - "No eligible rows to cluster. Select completed calls that match " - "at least one configured failure policy." - ), - ) - if not filtered_pairs: - raise HTTPException( - status_code=400, - detail="No completed rows match the selected evaluation_row_ids.", - ) - - _enqueue_metric_clusters_job( - evaluation, - provider=provider_enum.value, - model=model_str, - credential_id=body.credential_id, - force=body.force or body.regenerate, - max_llm_calls=body.max_llm_calls, - evaluation_row_ids=body.evaluation_row_ids, - selected_evaluation_row_ids=selected_row_ids, - failure_policies=merged_policies, - db=db, - ) - - db.refresh(evaluation) - return _metric_clusters_payload(evaluation) or EvaluationMetricClustersState( - status="running" - ) - - -@router.post( - "/{eval_id}/metric-clusters/cancel", - response_model=EvaluationMetricClustersState, - operation_id="cancelCallImportEvaluationMetricClusters", -) -async def cancel_call_import_evaluation_metric_clusters( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> EvaluationMetricClustersState: - """Abort in-flight failure-diagnostics clustering. - - Idempotent: if clustering is not ``running``, returns the current state - unchanged. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - _apply_metric_clusters_cancel(evaluation) - flag_modified(evaluation, "metric_clusters") - db.commit() - db.refresh(evaluation) - - return _metric_clusters_payload(evaluation) or EvaluationMetricClustersState( - status="idle" - ) - - -@router.get( - "/{eval_id}/prompt-improvements", - response_model=Optional[EvaluationPromptImprovementsState], - operation_id="getCallImportEvaluationPromptImprovements", -) -async def get_call_import_evaluation_prompt_improvements( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Optional[EvaluationPromptImprovementsState]: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - return _prompt_improvements_payload(evaluation) - - -@router.post( - "/{eval_id}/prompt-improvements", - response_model=EvaluationPromptImprovementsState, - operation_id="generateCallImportEvaluationPromptImprovements", -) -async def generate_call_import_evaluation_prompt_improvements( - call_import_id: UUID, - eval_id: UUID, - body: EvaluationPromptImprovementsRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> EvaluationPromptImprovementsState: - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - clusters = _metric_clusters_payload(evaluation) - if clusters is None or clusters.status != "completed": - raise HTTPException( - status_code=400, - detail=( - "Metric clusters must be completed before generating prompt " - "improvements. Run failure diagnostics first." - ), - ) - - from app.services.call_import_prompt_improvements import is_imported_agent - from app.services.ai.llm_resolver import get_llm_provider_and_model - - imported_agent = ( - db.query(PromptPartial) - .filter( - PromptPartial.id == body.imported_agent_id, - PromptPartial.organization_id == organization_id, - PromptPartial.workspace_id == workspace_id, - ) - .first() - ) - if imported_agent is None or not is_imported_agent(imported_agent): - raise HTTPException( - status_code=404, - detail="Imported agent not found in the active workspace", - ) - - if not body.regenerate and not body.force: - cached = _prompt_improvements_payload(evaluation) - if ( - cached is not None - and cached.status in {"running", "completed"} - and cached.imported_agent_id == str(body.imported_agent_id) - ): - return cached - - provider_enum, model_str = get_llm_provider_and_model( - organization_id, db, body.provider, body.model, body.credential_id - ) - - _enqueue_prompt_improvements_job( - evaluation, - imported_agent_id=body.imported_agent_id, - imported_agent_name=imported_agent.name, - provider=provider_enum.value, - model=model_str, - credential_id=body.credential_id, - force=body.force or body.regenerate, - db=db, - ) - - db.refresh(evaluation) - return _prompt_improvements_payload(evaluation) or EvaluationPromptImprovementsState( - status="running", - imported_agent_id=str(body.imported_agent_id), - imported_agent_name=imported_agent.name, - ) - - -# --------------------------------------------------------------------------- -# Flow chart: turns per-row LLM-inferred ``sequence`` arrays into a -# directed graph of (label -> label) transitions across the whole run. -# Powers the aggregate Sankey-style React Flow chart on the evaluation -# overview; per-call flow charts are built client-side from the same -# ``sequence`` field on a single row's metric_scores entry. -# --------------------------------------------------------------------------- - - -_FLOW_TERMINAL_THRESHOLD = 0.2 # Mark as terminal when >=20% of sequences end here. -_FLOW_START_NODE_ID = "__START__" -_DISCOVERED_NODE_PREFIX = "disc:" - - -def _slug_label(value: Any) -> str: - """Lowercase + whitespace-collapse + underscore-join. - - Used everywhere we need a stable key for a metric/label name — - matching the same convention the worker uses when emitting - ``sequence`` entries and discovered keys. - """ - if value is None: - return "" - return "_".join(str(value).strip().lower().split()) - - -def _resolve_alias(alias_map: Dict[str, str], key: str) -> str: - """Walk the alias map until we hit a slug that doesn't redirect. - - The merge endpoint stores ``from_slug -> to_slug`` pairs. The delete - endpoint stores ``from_slug -> ""`` (empty string sentinel) to mark - a slug as tombstoned. Chains can accumulate when the user merges - A→B and later merges B→C; this helper collapses them so callers - always land on the final canonical slug. - - Returns: - * the canonical slug if it still resolves to a real label, - * an empty string if the slug has been tombstoned (callers MUST - treat an empty result as "drop this entry entirely"), - * the input ``key`` if it isn't aliased. - - Cycles are guarded by a hard step limit since the alias map is - user-driven. - """ - if not key: - return "" - if not alias_map: - return key - current = key - seen: set[str] = set() - for _ in range(16): - if current in seen: - return current - seen.add(current) - if current not in alias_map: - return current - nxt = alias_map[current] - if nxt == current: - return current - if nxt == "": - # Deletion sentinel — the user has explicitly retired this - # slug. Propagate the empty string up so callers drop it. - return "" - current = nxt - return current - - -# Reserved JSON key under which the worker stores top-level metric -# discoveries on each row's ``metric_scores`` dict. Mirrors the constant -# in ``app/workers/tasks/helpers/llm_evaluation.py`` — kept local here to -# avoid a worker import cycle from the routes module. -DISCOVERED_METRICS_KEY = "__discovered_metrics__" - -# Allowed values for an LLM-suggested top-level metric type. Kept in -# sync with ``DiscoveredMetricSuggestedType`` in -# ``app/models/schemas.py``. -_DISCOVERED_METRIC_TYPES = ("boolean", "rating", "category") - - -def normalize_scores_with_aliases( - metric_scores: Dict[str, Any], - evaluation: CallImportEvaluation, - db: Session, - organization_id: UUID, -) -> Dict[str, Any]: - """Rewrite per-row ``metric_scores`` to honor merges + promotions. - - Called by the worker right after ``evaluate_with_llm`` returns so - every row that finishes AFTER a user has merged or promoted a - discovered label persists data already reflecting that decision. - Without this hook, a worker holding a stale prompt could re-emit a - ``from_key`` slug long after the user merged it away. - - For every parent entry (``selection_mode != null`` and a - ``discovered_labels`` / ``sequence`` field) we: - - * resolve discovered slugs through the evaluation's - ``discovered_label_aliases`` map (transitively), - * drop any discovered_labels entry whose canonical slug now - matches a real promoted child of the parent (merging them out - of the panel for free), and - * collapse adjacent duplicate sequence entries that result. - - Returns ``metric_scores`` (mutated in place) for chaining. - """ - if not isinstance(metric_scores, dict): - return metric_scores - - aliases_top = ( - evaluation.discovered_label_aliases - if isinstance(evaluation.discovered_label_aliases, dict) - else {} - ) - - # Identify the parent entries inside metric_scores. They're the - # dicts that carry a ``selection_mode`` key (set by the LLM - # hierarchy parser) and either a ``sequence`` or a - # ``discovered_labels`` list. - for key, entry in list(metric_scores.items()): - if not isinstance(entry, dict): - continue - if entry.get("type") != "category" and not entry.get("selection_mode"): - continue - try: - parent_uuid = UUID(str(key)) - except (TypeError, ValueError): - continue - - alias_map = {} - sub = aliases_top.get(str(parent_uuid)) - if isinstance(sub, dict): - alias_map = { - str(k): str(v) - for k, v in sub.items() - if isinstance(k, str) and isinstance(v, str) - } - promoted = _promoted_child_slugs(db, parent_uuid, organization_id) - - # Rewrite discovered_labels: alias-resolve keys, drop duplicates - # post-resolution, and drop entries that have been promoted. - discovered = entry.get("discovered_labels") - if isinstance(discovered, list): - kept_disc: List[Dict[str, Any]] = [] - seen: set[str] = set() - for d in discovered: - if not isinstance(d, dict): - continue - slug = _slug_label(d.get("key") or d.get("name")) - slug = _resolve_alias(alias_map, slug) - if not slug or slug in promoted or slug in seen: - continue - seen.add(slug) - new_entry = dict(d) - new_entry["key"] = slug - kept_disc.append(new_entry) - entry["discovered_labels"] = kept_disc - - # Rewrite sequence: alias-resolve every entry; collapse adjacent - # duplicates that result. We DON'T drop slugs that match - # promoted children — the promoted child slug is still a valid - # sequence entry; the flow chart will resolve it to the real - # child node. - seq = entry.get("sequence") - if isinstance(seq, list): - new_seq: List[str] = [] - last: Optional[str] = None - for item in seq: - if not isinstance(item, str): - continue - slug = _resolve_alias(alias_map, _slug_label(item)) - if not slug or slug == last: - continue - new_seq.append(slug) - last = slug - entry["sequence"] = new_seq - - # Top-level metric discoveries live alongside the parent entries - # under the reserved ``DISCOVERED_METRICS_KEY`` slot. Apply the - # flat evaluation-level alias/tombstone map + suppress slugs that - # already correspond to a real top-level Metric so workers that - # finish AFTER the user has merged / deleted / promoted can't - # resurrect a retired candidate. - discovered_metrics_payload = metric_scores.get(DISCOVERED_METRICS_KEY) - if isinstance(discovered_metrics_payload, list): - flat_alias_map = ( - evaluation.discovered_metric_aliases - if isinstance(evaluation.discovered_metric_aliases, dict) - else {} - ) - promoted_metric_slugs = _promoted_top_level_metric_slugs( - db, organization_id - ) - kept_metrics: List[Dict[str, Any]] = [] - seen_metrics: set[str] = set() - for d in discovered_metrics_payload: - if not isinstance(d, dict): - continue - slug = _slug_label(d.get("key") or d.get("name")) - slug = _resolve_alias(flat_alias_map, slug) - if ( - not slug - or slug in promoted_metric_slugs - or slug in seen_metrics - ): - continue - seen_metrics.add(slug) - new_entry = dict(d) - new_entry["key"] = slug - kept_metrics.append(new_entry) - if kept_metrics: - metric_scores[DISCOVERED_METRICS_KEY] = kept_metrics - else: - # No survivors — drop the empty array so empty-discovery rows - # keep their pre-feature payload shape. - metric_scores.pop(DISCOVERED_METRICS_KEY, None) - - return metric_scores - - -def _alias_map_for_parent( - evaluation: CallImportEvaluation, parent_metric_id: UUID -) -> Dict[str, str]: - """Pull ``{from_slug: to_slug}`` for one parent out of the eval's blob. - - Stored shape on the evaluation row is - ``{parent_id_str: {from_slug: to_slug, ...}}``. Returns an empty - dict for parents that have never had a merge applied. - """ - raw = getattr(evaluation, "discovered_label_aliases", None) - if not isinstance(raw, dict): - return {} - submap = raw.get(str(parent_metric_id)) - if not isinstance(submap, dict): - return {} - return { - str(k): str(v) - for k, v in submap.items() - if isinstance(k, str) and isinstance(v, str) - } - - -def _promoted_child_slugs( - db: Session, parent_metric_id: UUID, organization_id: UUID -) -> set[str]: - """Slugs of every real child currently sitting under the parent. - - The Discovered Labels panel hides any candidate whose slug already - matches a real child — that covers both freshly-promoted candidates - and legacy children the LLM happened to re-discover. We pull from - the live ``metrics`` table rather than the eval's - ``selected_metric_groups`` snapshot so newly-promoted children take - effect immediately, even on evaluations that ran before the - promotion. - """ - children = ( - db.query(Metric.name) - .filter( - Metric.parent_metric_id == parent_metric_id, - Metric.organization_id == organization_id, - ) - .all() - ) - out: set[str] = set() - for (name,) in children: - slug = _slug_label(name) - if slug: - out.add(slug) - return out - - -def _promoted_top_level_metric_slugs( - db: Session, organization_id: UUID -) -> set[str]: - """Slugs of every top-level (non-child) Metric in the organization. - - Used to suppress discovered-metric candidates whose slug already - matches a real standalone metric. We intentionally include both - standalone metrics AND parent category metrics — a top-level - discovery that collides with either name is a duplicate by - definition. - """ - rows = ( - db.query(Metric.name) - .filter( - Metric.organization_id == organization_id, - Metric.parent_metric_id.is_(None), - ) - .all() - ) - out: set[str] = set() - for (name,) in rows: - slug = _slug_label(name) - if slug: - out.add(slug) - return out - - -def _get_running_discovered_labels( - db: Session, - eval_id: UUID, - parent_metric_id: UUID, - organization_id: Optional[UUID] = None, - alias_map: Optional[Dict[str, str]] = None, -) -> List[Dict[str, Any]]: - """Slug-deduped view of every discovered label seen in this eval so far. - - Walks each ``call_import_evaluation_rows`` row's - ``metric_scores[parent_id]["discovered_labels"]`` and folds entries - that share the same slug. Returns a list ordered by descending - count and stable on label key, shaped like:: - - [{"key": "customer_on_hold", "name": "Customer put on hold", - "description": "...", "sample_rationale": "...", "count": 12}] - - Powers two callers: - * The worker prompt builder ("REUSE the existing key if it fits") - — invoked just before each row's LLM call to feed the model the - running list of previously-discovered labels in this evaluation. - * The ``/discovered-labels`` API surface used by the frontend - Discovered Labels panel to render candidates with counts + - sample rationales. - - Non-completed rows are skipped: an in-flight row's discoveries are - not yet reliable (the row could fail and never produce final - metric_scores). We accept the tradeoff that rows running - concurrently won't see each other's labels — slug-collision dedup - catches identical re-inventions, and near-paraphrases surface in - the UI panel where the user can manually merge. - """ - - parent_id_str = str(parent_metric_id) - from app.db_sharding.eval_rows import load_evaluation_rows_for_run - - eval_rows = load_evaluation_rows_for_run(db, eval_id) - rows = [ - (row.metric_scores,) - for row in eval_rows - if row.status == CallImportRowStatus.COMPLETED.value - ] - - # Suppress slugs that have either: - # * been promoted to a real child of the parent (so the panel doesn't - # keep nagging the user about a candidate they've already - # accepted), or - # * been merged INTO another slug (the "from" side of a merge) — - # those occurrences fold into the canonical target instead. - promoted_slugs: set[str] = set() - if organization_id is not None: - promoted_slugs = _promoted_child_slugs( - db, parent_metric_id, organization_id - ) - aliases = alias_map or {} - - by_key: Dict[str, Dict[str, Any]] = {} - for (scores,) in rows: - if not isinstance(scores, dict): - continue - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - continue - discovered = parent_entry.get("discovered_labels") - if not isinstance(discovered, list): - continue - for entry in discovered: - if not isinstance(entry, dict): - continue - raw_key = entry.get("key") or entry.get("name") - key = _slug_label(raw_key) - if not key: - continue - # Apply user merges + deletions first, THEN drop anything - # that ended up on a real child slug. Order matters: a - # candidate that was merged into a slug which has since - # been promoted should disappear, not show up at the - # canonical slug. An empty resolved key means the slug was - # tombstoned via the delete endpoint. - key = _resolve_alias(aliases, key) - if not key or key in promoted_slugs: - continue - name = (entry.get("name") or "").strip() or key.replace("_", " ") - description = (entry.get("description") or "").strip() or None - sample = (entry.get("rationale") or "").strip() or None - - existing = by_key.get(key) - if existing is None: - # Track up to N=3 distinct rationales per candidate so - # the Promote-to-child flow can pre-fill the new - # sub-metric's rubric with concrete LLM examples - # without the user copy-pasting from the row table. - # ``sample_rationale`` is preserved for back-compat - # with older clients; ``examples`` is the new field. - examples = [sample] if sample else [] - by_key[key] = { - "key": key, - "name": name, - "description": description, - "sample_rationale": sample, - "examples": examples, - "count": 1, - } - continue - - existing["count"] += 1 - if not existing["description"] and description: - existing["description"] = description - if not existing["sample_rationale"] and sample: - existing["sample_rationale"] = sample - # Append distinct rationales (case-insensitive trim) up - # to a small cap. Headroom is intentionally one above - # what the UI surfaces (2) so we have a backup when the - # first rationale is unhelpful. - if sample: - ex_list: List[str] = existing.setdefault("examples", []) - if len(ex_list) < 3 and not any( - s.strip().lower() == sample.strip().lower() for s in ex_list - ): - ex_list.append(sample) - - return sorted( - by_key.values(), - key=lambda item: (-item["count"], item["key"]), - ) - - -def _get_running_discovered_metrics( - db: Session, - eval_id: UUID, - organization_id: Optional[UUID] = None, - alias_map: Optional[Dict[str, str]] = None, -) -> List[Dict[str, Any]]: - """Slug-deduped view of every discovered top-level metric in this eval. - - Mirrors :func:`_get_running_discovered_labels` but is keyed at the - evaluation level (no ``parent_metric_id``). Walks each completed - row's ``metric_scores[DISCOVERED_METRICS_KEY]`` list, folds entries - that share the same slug (post-alias resolution), and suppresses - slugs that already correspond to a real top-level :class:`Metric` - in the organization. - - Each returned entry is shaped:: - - {"key": "customer_satisfaction", - "name": "Customer Satisfaction", - "description": "...", - "suggested_type": "boolean" | "rating" | "category", - "sample_rationale": "...", - "examples": ["..."], - "count": 12} - """ - - from app.db_sharding.eval_rows import load_evaluation_rows_for_run - - eval_rows = load_evaluation_rows_for_run(db, eval_id) - rows = [ - (row.metric_scores,) - for row in eval_rows - if row.status == CallImportRowStatus.COMPLETED.value - ] - - promoted_slugs: set[str] = set() - if organization_id is not None: - promoted_slugs = _promoted_top_level_metric_slugs( - db, organization_id - ) - aliases = alias_map or {} - - by_key: Dict[str, Dict[str, Any]] = {} - for (scores,) in rows: - if not isinstance(scores, dict): - continue - discovered = scores.get(DISCOVERED_METRICS_KEY) - if not isinstance(discovered, list): - continue - for entry in discovered: - if not isinstance(entry, dict): - continue - raw_key = entry.get("key") or entry.get("name") - key = _slug_label(raw_key) - if not key: - continue - # Apply user merges + deletions first, THEN drop anything - # that ended up on an already-existing top-level metric - # slug. Empty resolved key = tombstoned. - key = _resolve_alias(aliases, key) - if not key or key in promoted_slugs: - continue - name = (entry.get("name") or "").strip() or key.replace( - "_", " " - ) - description = (entry.get("description") or "").strip() or None - sample = (entry.get("rationale") or "").strip() or None - raw_type = str(entry.get("suggested_type") or "").strip().lower() - if raw_type not in _DISCOVERED_METRIC_TYPES: - raw_type = "boolean" - - existing = by_key.get(key) - if existing is None: - examples = [sample] if sample else [] - by_key[key] = { - "key": key, - "name": name, - "description": description, - "suggested_type": raw_type, - "sample_rationale": sample, - "examples": examples, - "count": 1, - } - continue - - existing["count"] += 1 - if not existing["description"] and description: - existing["description"] = description - if not existing["sample_rationale"] and sample: - existing["sample_rationale"] = sample - # Keep the most-frequently-suggested type. We don't track - # per-type frequency yet; defer to the first non-default - # type encountered when the existing entry has the default. - if existing.get("suggested_type") == "boolean" and raw_type != "boolean": - existing["suggested_type"] = raw_type - if sample: - ex_list: List[str] = existing.setdefault("examples", []) - if len(ex_list) < 3 and not any( - s.strip().lower() == sample.strip().lower() for s in ex_list - ): - ex_list.append(sample) - - return sorted( - by_key.values(), - key=lambda item: (-item["count"], item["key"]), - ) - - -def _build_flow_graph( - eval_rows: List[CallImportEvaluationRow], - parent_metric: Metric, - children: List[Metric], - alias_map: Optional[Dict[str, str]] = None, - extra_children: Optional[List[Metric]] = None, -) -> MetricFlowResponse: - """Walk per-row ``sequence`` arrays and produce aggregate nodes/edges. - - A synthetic ``START`` node is prepended to every sequence so the - diagram has a single origin. Children that never appear in any - sequence are still emitted as nodes (count=0) so the UI can render - them in the legend. - - ``alias_map`` lets callers fold merged-out discovered slugs into - their canonical target before building the graph; ``extra_children`` - are children of the parent that aren't in the legend list (e.g. - children promoted *after* the evaluation was created and therefore - missing from ``selected_metric_groups``) but should still resolve in - sequences so the slug doesn't get redrawn as a discovered candidate. - """ - parent_id_str = str(parent_metric.id) - aliases = alias_map or {} - # Build a fast lookup keyed by both the lower_snake child key (what the - # LLM emits in ``sequence``) and the child UUID (what some clients may - # store) so legacy / drifted payloads still resolve. - child_lookup: Dict[str, Metric] = {} - for child in children: - slug = _slug_label(child.name) - child_lookup[slug] = child - child_lookup[str(child.id)] = child - # ``extra_children`` are resolved-only — they shouldn't add legend - # nodes (those come from the explicit ``children`` argument), but - # they need to be in ``child_lookup`` so a sequence step that - # matches a freshly-promoted child resolves to the real child UUID - # instead of falling through to ``discovered_lookup`` and rendering - # as a "discovered" node. - if extra_children: - for child in extra_children: - slug = _slug_label(child.name) - if slug and slug not in child_lookup: - child_lookup[slug] = child - cid = str(child.id) - child_lookup.setdefault(cid, child) - - # Discovered labels: walk every row's discovered_labels first so we - # know which discovered slugs are valid before resolving sequences. - # Discovered nodes get a ``disc:`` prefixed id so they can't collide - # with real child UUIDs in the node/edge graph. We apply - # ``alias_map`` first so merged-out source slugs fold into their - # canonical target — preserving the user's "merge" intent on still- - # in-flight rows whose JSON wasn't rewritten by the merge endpoint. - discovered_lookup: Dict[str, Dict[str, Any]] = {} - for row in eval_rows: - scores = ( - row.metric_scores if isinstance(row.metric_scores, dict) else {} - ) - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - continue - raw_discovered = parent_entry.get("discovered_labels") - if not isinstance(raw_discovered, list): - continue - for entry in raw_discovered: - if not isinstance(entry, dict): - continue - slug = _slug_label(entry.get("key") or entry.get("name")) - slug = _resolve_alias(aliases, slug) - if not slug or slug in child_lookup: - continue - name = (entry.get("name") or "").strip() or slug.replace("_", " ") - existing = discovered_lookup.get(slug) - if existing is None: - discovered_lookup[slug] = { - "id": f"{_DISCOVERED_NODE_PREFIX}{slug}", - "name": name, - } - - node_counts: Dict[str, int] = {} - edge_counts: Dict[tuple[str, str], int] = {} - terminal_counts: Dict[str, int] = {} - - total_rows = len(eval_rows) - rows_with_sequence = 0 - - for row in eval_rows: - scores = ( - row.metric_scores if isinstance(row.metric_scores, dict) else {} - ) - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - continue - raw_sequence = parent_entry.get("sequence") - if not isinstance(raw_sequence, list): - continue - - resolved_ids: List[str] = [] - last_resolved: Optional[str] = None - for item in raw_sequence: - if not isinstance(item, str): - continue - normalized = _resolve_alias(aliases, _slug_label(item)) - child = child_lookup.get(normalized) or child_lookup.get(item) - if child is not None: - cid = str(child.id) - # Adjacent dedupe AFTER alias resolution so two - # different raw slugs that fold to the same target - # don't draw a self-edge through the chart. - if cid == last_resolved: - continue - resolved_ids.append(cid) - last_resolved = cid - continue - disc = discovered_lookup.get(normalized) - if disc is not None: - if disc["id"] == last_resolved: - continue - resolved_ids.append(disc["id"]) - last_resolved = disc["id"] - - if not resolved_ids: - continue - - rows_with_sequence += 1 - for nid in resolved_ids: - node_counts[nid] = node_counts.get(nid, 0) + 1 - - edge_counts[(_FLOW_START_NODE_ID, resolved_ids[0])] = ( - edge_counts.get((_FLOW_START_NODE_ID, resolved_ids[0]), 0) + 1 - ) - for src, tgt in zip(resolved_ids, resolved_ids[1:]): - if src == tgt: - continue - edge_counts[(src, tgt)] = edge_counts.get((src, tgt), 0) + 1 - - terminal_id = resolved_ids[-1] - terminal_counts[terminal_id] = terminal_counts.get(terminal_id, 0) + 1 - - nodes: List[MetricFlowNode] = [] - # Always include a START node so the UI has a stable entry point. - nodes.append( - MetricFlowNode( - id=_FLOW_START_NODE_ID, - label="Start", - count=rows_with_sequence, - is_terminal=False, - ) - ) - - def _emit_child_node(child: Metric) -> None: - cid = str(child.id) - count = node_counts.get(cid, 0) - terminal_count = terminal_counts.get(cid, 0) - is_terminal = False - if rows_with_sequence > 0: - is_terminal = ( - terminal_count / rows_with_sequence - ) >= _FLOW_TERMINAL_THRESHOLD - nodes.append( - MetricFlowNode( - id=cid, - label=child.name, - count=count, - is_terminal=is_terminal, - ) - ) - - emitted_child_ids: set[str] = set() - for child in children: - cid = str(child.id) - if cid in emitted_child_ids: - continue - emitted_child_ids.add(cid) - _emit_child_node(child) - # Extra children (promoted after the eval was created) only get - # legend nodes if they actually appear in the data — otherwise we'd - # pollute the diagram with every standalone promotion the user has - # ever made under this parent. - if extra_children: - for child in extra_children: - cid = str(child.id) - if cid in emitted_child_ids: - continue - if node_counts.get(cid, 0) == 0: - continue - emitted_child_ids.add(cid) - _emit_child_node(child) - # Append discovered nodes after the real children so legend ordering - # keeps user-defined labels first. - for slug, info in discovered_lookup.items(): - nid = info["id"] - count = node_counts.get(nid, 0) - terminal_count = terminal_counts.get(nid, 0) - is_terminal = False - if rows_with_sequence > 0: - is_terminal = ( - terminal_count / rows_with_sequence - ) >= _FLOW_TERMINAL_THRESHOLD - nodes.append( - MetricFlowNode( - id=nid, - label=info["name"], - count=count, - is_terminal=is_terminal, - is_discovered=True, - ) - ) - - edges: List[MetricFlowEdge] = [ - MetricFlowEdge(source=src, target=tgt, count=count) - for (src, tgt), count in sorted( - edge_counts.items(), key=lambda kv: kv[1], reverse=True - ) - ] - - return MetricFlowResponse( - parent_metric_id=parent_id_str, - parent_metric_name=parent_metric.name, - selection_mode=parent_metric.selection_mode, - nodes=nodes, - edges=edges, - total_rows=total_rows, - rows_with_sequence=rows_with_sequence, - ) - - -@router.get( - "/{eval_id}/flow", - response_model=MetricFlowResponse, - operation_id="getCallImportEvaluationFlow", -) -async def get_call_import_evaluation_flow( - call_import_id: UUID, - eval_id: UUID, - parent_metric_id: UUID = Query( - ..., - description=( - "Parent (category) metric whose children's sequences should be " - "aggregated into a flow graph." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> MetricFlowResponse: - """Aggregate the LLM-inferred per-row sequences into one flow graph. - - Returns ``nodes`` (one per child of the parent metric, plus a - synthetic ``START`` node) and ``edges`` (counts of consecutive - label transitions across every row that produced a sequence). The - frontend feeds this directly into a React Flow / xyflow canvas; - edge thickness should scale with ``count / total_rows`` and - ``is_terminal`` nodes should be styled as outcomes. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - parent = ( - db.query(Metric) - .filter( - Metric.id == parent_metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if not parent: - raise HTTPException( - status_code=404, - detail="Parent metric not found in this organization.", - ) - if not parent.selection_mode: - raise HTTPException( - status_code=400, - detail=( - "Flow charts are only meaningful for parent metrics " - "(selection_mode set). This metric is standalone." - ), - ) - - # Children are taken from selected_metric_groups when present so the - # flow chart reflects exactly the subset that ran in this - # evaluation; otherwise fall back to every enabled child of the - # parent. - groups_raw = ( - evaluation.selected_metric_groups - if isinstance(evaluation.selected_metric_groups, dict) - else {} - ) - parent_id_str = str(parent.id) - children: List[Metric] = [] - if parent_id_str in groups_raw and isinstance( - groups_raw[parent_id_str], list - ): - child_ids: List[UUID] = [] - for c in groups_raw[parent_id_str]: - try: - child_ids.append(UUID(str(c))) - except (TypeError, ValueError): - continue - if child_ids: - children = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.id.in_(child_ids), - ) - .order_by(Metric.created_at.asc()) - .all() - ) - if not children: - children = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.parent_metric_id == parent.id, - ) - .order_by(Metric.created_at.asc()) - .all() - ) - - # Children promoted AFTER this evaluation was created aren't in - # ``selected_metric_groups`` but their slugs still appear in already- - # scored rows' sequences. Pass them as ``extra_children`` so those - # sequence entries resolve against the real (now promoted) child - # instead of being redrawn as discovered candidates. - extra_children: List[Metric] = [] - if children: - existing_ids = {child.id for child in children} - all_children = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.parent_metric_id == parent.id, - ) - .all() - ) - extra_children = [c for c in all_children if c.id not in existing_ids] - - eval_rows = _load_eval_rows(db, eval_id) - - alias_map = _alias_map_for_parent(evaluation, parent.id) - return _build_flow_graph( - eval_rows, - parent, - children, - alias_map=alias_map, - extra_children=extra_children, - ) - - -@router.get( - "/{eval_id}/discovered-labels", - response_model=DiscoveredLabelsResponse, - operation_id="getCallImportEvaluationDiscoveredLabels", -) -async def get_call_import_evaluation_discovered_labels( - call_import_id: UUID, - eval_id: UUID, - parent_metric_id: UUID = Query( - ..., - description=( - "Parent (category) metric whose LLM-discovered candidate " - "sub-labels should be aggregated across rows." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> DiscoveredLabelsResponse: - """Aggregate candidate sub-labels the LLM discovered during this eval. - - Only meaningful for parents with ``allow_discovery=true``; for other - parents we just return an empty ``items`` list rather than 400-ing - so the frontend can call the endpoint unconditionally for every - parent on the Flow tab without branching. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - parent = ( - db.query(Metric) - .filter( - Metric.id == parent_metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if not parent: - raise HTTPException( - status_code=404, - detail="Parent metric not found in this organization.", - ) - - alias_map = _alias_map_for_parent(evaluation, parent_metric_id) - items_raw = _get_running_discovered_labels( - db, - eval_id, - parent_metric_id, - organization_id=organization_id, - alias_map=alias_map, - ) - items = [DiscoveredLabelItem(**item) for item in items_raw] - return DiscoveredLabelsResponse( - parent_metric_id=str(parent.id), items=items - ) - - -@router.post( - "/{eval_id}/discovered-labels/merge", - response_model=DiscoveredLabelsResponse, - operation_id="mergeCallImportEvaluationDiscoveredLabels", -) -async def merge_call_import_evaluation_discovered_labels( - call_import_id: UUID, - eval_id: UUID, - body: DiscoveredLabelMergeRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> DiscoveredLabelsResponse: - """Rewrite every row's ``discovered_labels`` entry from from_key -> to_key. - - Idempotent — re-merging the same pair is a no-op. Discovered slugs - inside per-row ``sequence`` arrays are also rewritten so the flow - chart stays consistent with the panel. When a row already has - ``to_key`` and we're merging ``from_key`` into it, we drop the - ``from_key`` entry instead of producing two entries with the same - slug. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - parent = ( - db.query(Metric) - .filter( - Metric.id == body.parent_metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if not parent: - raise HTTPException( - status_code=404, - detail="Parent metric not found in this organization.", - ) - - from_key = _slug_label(body.from_key) - to_key = _slug_label(body.to_key) - if not from_key or not to_key: - raise HTTPException( - status_code=400, - detail="from_key and to_key must be non-empty slugs.", - ) - if from_key == to_key: - # No-op; just return the current aggregate so the client can - # refresh its view. - alias_map_existing = _alias_map_for_parent(evaluation, parent.id) - items_raw = _get_running_discovered_labels( - db, - eval_id, - body.parent_metric_id, - organization_id=organization_id, - alias_map=alias_map_existing, - ) - return DiscoveredLabelsResponse( - parent_metric_id=str(parent.id), - items=[DiscoveredLabelItem(**item) for item in items_raw], - ) - - parent_id_str = str(parent.id) - from app.db_sharding.eval_rows import foreach_evaluation_row_mutating - - def _merge_discovered_label_row(row: CallImportEvaluationRow) -> bool: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else None - ) - if not scores: - return False - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - return False - - mutated = False - discovered = parent_entry.get("discovered_labels") - if isinstance(discovered, list): - kept: List[Dict[str, Any]] = [] - existing_to = next( - ( - e - for e in discovered - if isinstance(e, dict) - and _slug_label(e.get("key") or e.get("name")) == to_key - ), - None, - ) - for entry in discovered: - if not isinstance(entry, dict): - kept.append(entry) - continue - key = _slug_label(entry.get("key") or entry.get("name")) - if key == from_key: - if existing_to is not None: - mutated = True - continue - new_entry = dict(entry) - new_entry["key"] = to_key - kept.append(new_entry) - mutated = True - else: - kept.append(entry) - if mutated: - parent_entry["discovered_labels"] = kept - - seq = parent_entry.get("sequence") - if isinstance(seq, list): - new_seq: List[str] = [] - seq_changed = False - last_added: Optional[str] = None - for item in seq: - if isinstance(item, str) and _slug_label(item) == from_key: - seq_changed = True - if last_added == to_key: - continue - new_seq.append(to_key) - last_added = to_key - else: - new_seq.append(item) - last_added = ( - _slug_label(item) if isinstance(item, str) else None - ) - if seq_changed: - parent_entry["sequence"] = new_seq - mutated = True - - if mutated: - row.metric_scores = dict(scores) - return mutated - - foreach_evaluation_row_mutating(db, eval_id, _merge_discovered_label_row) - - # Persist the merge at the evaluation level too. This is what makes - # the merge survive future scoring: rows that finish AFTER this - # call (e.g. retries, in-flight workers) will go through the - # alias map in the API surface even if the per-row JSON they - # write still mentions ``from_key``. We chain through any existing - # alias so merging A→B and then B→C resolves A→C in the panel. - raw_aliases = ( - evaluation.discovered_label_aliases - if isinstance(evaluation.discovered_label_aliases, dict) - else {} - ) - aliases_top = dict(raw_aliases) - parent_aliases = dict(aliases_top.get(parent_id_str) or {}) - # Resolve transitively: if to_key itself was previously merged into - # something else, point from_key at the canonical end-of-chain. - canonical_to = _resolve_alias(parent_aliases, to_key) - parent_aliases[from_key] = canonical_to - # Re-target any earlier aliases that pointed AT from_key — without - # this, A→B and then B→C would leave A still pointing to B (now a - # broken pointer because B is gone). Rewriting them keeps the - # alias map self-consistent. - for k, v in list(parent_aliases.items()): - if v == from_key: - parent_aliases[k] = canonical_to - aliases_top[parent_id_str] = parent_aliases - evaluation.discovered_label_aliases = aliases_top - - db.commit() - - alias_map_after = _alias_map_for_parent(evaluation, parent.id) - items_raw = _get_running_discovered_labels( - db, - eval_id, - body.parent_metric_id, - organization_id=organization_id, - alias_map=alias_map_after, - ) - return DiscoveredLabelsResponse( - parent_metric_id=str(parent.id), - items=[DiscoveredLabelItem(**item) for item in items_raw], - ) - - -@router.post( - "/{eval_id}/discovered-labels/delete", - response_model=DiscoveredLabelsResponse, - operation_id="deleteCallImportEvaluationDiscoveredLabel", -) -async def delete_call_import_evaluation_discovered_label( - call_import_id: UUID, - eval_id: UUID, - body: DiscoveredLabelDeleteRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> DiscoveredLabelsResponse: - """Tombstone a single LLM-discovered candidate for this evaluation. - - Symmetric with the merge endpoint, but instead of redirecting the - slug at another candidate we mark it as deleted. After this call: - - * the slug is stripped from every row's - ``metric_scores[parent].discovered_labels`` list, and from - every row's ``sequence`` array (so the flow chart no longer - draws a node for it); - * the slug is recorded in - ``evaluation.discovered_label_aliases[parent][slug] = ""`` - so any worker that finishes a row AFTER this call (e.g. a row - still in flight when the user clicked Delete) silently drops - the slug instead of resurrecting it. - - Idempotent: deleting an already-deleted slug is a no-op. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - parent = ( - db.query(Metric) - .filter( - Metric.id == body.parent_metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if not parent: - raise HTTPException( - status_code=404, - detail="Parent metric not found in this organization.", - ) - - target_key = _slug_label(body.key) - if not target_key: - raise HTTPException( - status_code=400, - detail="key must be a non-empty slug.", - ) - - parent_id_str = str(parent.id) - from app.db_sharding.eval_rows import foreach_evaluation_row_mutating - - def _delete_discovered_label_row(row: CallImportEvaluationRow) -> bool: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else None - ) - if not scores: - return False - parent_entry = scores.get(parent_id_str) - if not isinstance(parent_entry, dict): - return False - - mutated = False - discovered = parent_entry.get("discovered_labels") - if isinstance(discovered, list): - kept = [ - e - for e in discovered - if not ( - isinstance(e, dict) - and _slug_label(e.get("key") or e.get("name")) - == target_key - ) - ] - if len(kept) != len(discovered): - parent_entry["discovered_labels"] = kept - mutated = True - - seq = parent_entry.get("sequence") - if isinstance(seq, list): - new_seq: List[str] = [] - seq_changed = False - last_added: Optional[str] = None - for item in seq: - if isinstance(item, str) and _slug_label(item) == target_key: - seq_changed = True - continue - if isinstance(item, str): - norm = _slug_label(item) - if norm == last_added: - seq_changed = True - continue - last_added = norm - new_seq.append(item) - if seq_changed: - parent_entry["sequence"] = new_seq - mutated = True - - if mutated: - row.metric_scores = dict(scores) - return mutated - - foreach_evaluation_row_mutating(db, eval_id, _delete_discovered_label_row) - - # 3. Persist the tombstone on the evaluation so workers that finish - # later don't re-surface the deleted slug. We also retarget any - # existing aliases whose ``to_key`` was the deleted slug — without - # this, a previous merge that pointed at this slug would leave a - # dangling pointer. - raw_aliases = ( - evaluation.discovered_label_aliases - if isinstance(evaluation.discovered_label_aliases, dict) - else {} - ) - aliases_top = dict(raw_aliases) - parent_aliases = dict(aliases_top.get(parent_id_str) or {}) - parent_aliases[target_key] = "" # deletion sentinel - for k, v in list(parent_aliases.items()): - if v == target_key: - parent_aliases[k] = "" - aliases_top[parent_id_str] = parent_aliases - evaluation.discovered_label_aliases = aliases_top - - db.commit() - - alias_map_after = _alias_map_for_parent(evaluation, parent.id) - items_raw = _get_running_discovered_labels( - db, - eval_id, - body.parent_metric_id, - organization_id=organization_id, - alias_map=alias_map_after, - ) - return DiscoveredLabelsResponse( - parent_metric_id=str(parent.id), - items=[DiscoveredLabelItem(**item) for item in items_raw], - ) - - -# --------------------------------------------------------------------------- -# Discovered TOP-LEVEL METRICS (per-evaluation discovery) -# -# These endpoints are the parallel of the discovered-labels trio above but -# scoped to the evaluation as a whole instead of to a parent metric. They -# all live under ``/{eval_id}/discovered-metrics`` and operate on the -# reserved ``DISCOVERED_METRICS_KEY`` slot of each per-row -# ``metric_scores`` plus the flat ``CallImportEvaluation.discovered_metric_aliases`` -# map (no parent-id nesting). -# --------------------------------------------------------------------------- - - -def _flat_metric_aliases( - evaluation: CallImportEvaluation, -) -> Dict[str, str]: - """Pull the flat ``{from_slug: to_slug}`` map for an evaluation.""" - raw = getattr(evaluation, "discovered_metric_aliases", None) - if not isinstance(raw, dict): - return {} - return { - str(k): str(v) - for k, v in raw.items() - if isinstance(k, str) and isinstance(v, str) - } - - -@router.get( - "/{eval_id}/discovered-metrics", - response_model=DiscoveredMetricsResponse, - operation_id="getCallImportEvaluationDiscoveredMetrics", -) -async def get_call_import_evaluation_discovered_metrics( - call_import_id: UUID, - eval_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> DiscoveredMetricsResponse: - """Aggregate top-level metric candidates the LLM discovered during this eval. - - Returns an empty ``items`` list when the evaluation did not opt - into top-level metric discovery; this keeps the frontend able to - call the endpoint unconditionally without branching on the - evaluation's ``discover_new_metrics`` flag. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - if not bool(getattr(evaluation, "discover_new_metrics", False)): - return DiscoveredMetricsResponse(evaluation_id=evaluation.id, items=[]) - - items_raw = _get_running_discovered_metrics( - db, - eval_id, - organization_id=organization_id, - alias_map=_flat_metric_aliases(evaluation), - ) - return DiscoveredMetricsResponse( - evaluation_id=evaluation.id, - items=[DiscoveredMetricItem(**item) for item in items_raw], - ) - - -@router.post( - "/{eval_id}/discovered-metrics/merge", - response_model=DiscoveredMetricsResponse, - operation_id="mergeCallImportEvaluationDiscoveredMetrics", -) -async def merge_call_import_evaluation_discovered_metrics( - call_import_id: UUID, - eval_id: UUID, - body: DiscoveredMetricMergeRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> DiscoveredMetricsResponse: - """Rewrite every row's ``__discovered_metrics__`` entry from→to. - - Mirrors the discovered-labels merge endpoint but operates on the - flat top-level metric list. Idempotent — re-merging is a no-op. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - from_key = _slug_label(body.from_key) - to_key = _slug_label(body.to_key) - if not from_key or not to_key: - raise HTTPException( - status_code=400, - detail="from_key and to_key must be non-empty slugs.", - ) - if from_key == to_key: - items_raw = _get_running_discovered_metrics( - db, - eval_id, - organization_id=organization_id, - alias_map=_flat_metric_aliases(evaluation), - ) - return DiscoveredMetricsResponse( - evaluation_id=evaluation.id, - items=[DiscoveredMetricItem(**item) for item in items_raw], - ) - - from app.db_sharding.eval_rows import foreach_evaluation_row_mutating - - def _merge_discovered_metric_row(row: CallImportEvaluationRow) -> bool: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else None - ) - if not scores: - return False - discovered = scores.get(DISCOVERED_METRICS_KEY) - if not isinstance(discovered, list): - return False - - kept: List[Dict[str, Any]] = [] - mutated = False - existing_to = next( - ( - e - for e in discovered - if isinstance(e, dict) - and _slug_label(e.get("key") or e.get("name")) == to_key - ), - None, - ) - for entry in discovered: - if not isinstance(entry, dict): - kept.append(entry) - continue - key = _slug_label(entry.get("key") or entry.get("name")) - if key == from_key: - if existing_to is not None: - mutated = True - continue - new_entry = dict(entry) - new_entry["key"] = to_key - kept.append(new_entry) - mutated = True - else: - kept.append(entry) - if mutated: - scores[DISCOVERED_METRICS_KEY] = kept - row.metric_scores = dict(scores) - return mutated - - foreach_evaluation_row_mutating(db, eval_id, _merge_discovered_metric_row) - - raw_aliases = ( - evaluation.discovered_metric_aliases - if isinstance(evaluation.discovered_metric_aliases, dict) - else {} - ) - aliases = dict(raw_aliases) - canonical_to = _resolve_alias(aliases, to_key) - aliases[from_key] = canonical_to - for k, v in list(aliases.items()): - if v == from_key: - aliases[k] = canonical_to - evaluation.discovered_metric_aliases = aliases - - db.commit() - - items_raw = _get_running_discovered_metrics( - db, - eval_id, - organization_id=organization_id, - alias_map=_flat_metric_aliases(evaluation), - ) - return DiscoveredMetricsResponse( - evaluation_id=evaluation.id, - items=[DiscoveredMetricItem(**item) for item in items_raw], - ) - - -@router.post( - "/{eval_id}/discovered-metrics/delete", - response_model=DiscoveredMetricsResponse, - operation_id="deleteCallImportEvaluationDiscoveredMetric", -) -async def delete_call_import_evaluation_discovered_metric( - call_import_id: UUID, - eval_id: UUID, - body: DiscoveredMetricDeleteRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> DiscoveredMetricsResponse: - """Tombstone a single LLM-discovered top-level metric candidate.""" - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - target_key = _slug_label(body.key) - if not target_key: - raise HTTPException( - status_code=400, - detail="key must be a non-empty slug.", - ) - - from app.db_sharding.eval_rows import foreach_evaluation_row_mutating - - def _delete_discovered_metric_row(row: CallImportEvaluationRow) -> bool: - scores = ( - row.metric_scores - if isinstance(row.metric_scores, dict) - else None - ) - if not scores: - return False - discovered = scores.get(DISCOVERED_METRICS_KEY) - if not isinstance(discovered, list): - return False - kept = [ - e - for e in discovered - if not ( - isinstance(e, dict) - and _slug_label(e.get("key") or e.get("name")) - == target_key - ) - ] - if len(kept) == len(discovered): - return False - if kept: - scores[DISCOVERED_METRICS_KEY] = kept - else: - scores.pop(DISCOVERED_METRICS_KEY, None) - row.metric_scores = dict(scores) - return True - - foreach_evaluation_row_mutating(db, eval_id, _delete_discovered_metric_row) - - raw_aliases = ( - evaluation.discovered_metric_aliases - if isinstance(evaluation.discovered_metric_aliases, dict) - else {} - ) - aliases = dict(raw_aliases) - aliases[target_key] = "" # tombstone - for k, v in list(aliases.items()): - if v == target_key: - aliases[k] = "" - evaluation.discovered_metric_aliases = aliases - - db.commit() - - items_raw = _get_running_discovered_metrics( - db, - eval_id, - organization_id=organization_id, - alias_map=_flat_metric_aliases(evaluation), - ) - return DiscoveredMetricsResponse( - evaluation_id=evaluation.id, - items=[DiscoveredMetricItem(**item) for item in items_raw], - ) - - -@router.delete( - "/{eval_id}/rows/{eval_row_id}", - status_code=status.HTTP_204_NO_CONTENT, - operation_id="deleteCallImportEvaluationRow", -) -async def delete_call_import_evaluation_row( - call_import_id: UUID, - eval_id: UUID, - eval_row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Response: - """Delete a single per-row scoring entry within an evaluation run. - - Useful when the user wants to drop a noisy row before re-exporting - the CSV — e.g. a row whose audio was corrupt and skewed the - aggregate. Counters on the parent are recomputed so the rolled-up - status stays accurate. - """ - - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException(status_code=404, detail="Call import evaluation not found") - - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - from app.db_sharding.eval_rows import delete_evaluation_row_on_shards - - if not delete_evaluation_row_on_shards(eval_row_id, eval_id): - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) - _rollup_evaluation_status(evaluation, db) - db.commit() - return Response(status_code=status.HTTP_204_NO_CONTENT) - - eval_row = ( - db.query(CallImportEvaluationRow) - .filter( - CallImportEvaluationRow.id == eval_row_id, - CallImportEvaluationRow.evaluation_id == eval_id, - ) - .first() - ) - if not eval_row: - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) - - # If the row was still in flight, best-effort revoke the worker task - # so it doesn't try to write into a deleted DB row mid-execution. - if eval_row.celery_task_id and eval_row.status in {"pending", "running"}: - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke(eval_row.celery_task_id, terminate=False) - except Exception: - pass - - db.delete(eval_row) - db.flush() - _rollup_evaluation_status(evaluation, db) - db.commit() - return Response(status_code=status.HTTP_204_NO_CONTENT) - - -# --------------------------------------------------------------------------- -# Retry endpoints -# --------------------------------------------------------------------------- -# -# The create endpoint enqueues every row of a fresh run; these endpoints -# let the user re-enqueue a *subset* of rows in an existing run — most -# commonly the ones that failed. We keep the worker contract identical -# (``evaluate_call_import_row_task(eval_row_id)``), so the retry path -# only has to reset row state and re-fan-out. When a row is missing its -# diarised transcript and the run was configured for diarised -# transcripts, we chain through ``transcribe_call_import_row_task`` the -# same way the create endpoint does — that's what makes "retry" feel -# like "just fix it" instead of "fail again immediately". - - -def _prepare_source_row_for_retry( - source_row: CallImportRow, - *, - transcribe_overwrite: bool, -) -> None: - """Clear stale diarisation markers so retry dispatch can re-run the pipeline.""" - source_row.celery_task_id = None - - # Re-fetch recordings when a prior import failed or stalled without S3 audio. - # Mirrors retry_failed_call_import_rows so eval retry can re-enqueue imports. - if ( - source_row.status - in (CallImportRowStatus.FAILED, CallImportRowStatus.PROCESSING) - and not (source_row.recording_s3_key or "").strip() - ): - source_row.status = CallImportRowStatus.PENDING - source_row.error_message = None - - if transcribe_overwrite and (source_row.diarised_transcript or "").strip(): - source_row.diarised_transcript = None - - has_dia = bool((source_row.diarised_transcript or "").strip()) - dia_status = (source_row.diarised_transcript_status or "").strip().lower() - - if has_dia and not transcribe_overwrite: - source_row.diarised_transcript_status = "completed" - source_row.diarised_transcript_error = None - return - - if dia_status in {"failed", "pending", "running", "idle"}: - source_row.diarised_transcript_status = "idle" - source_row.diarised_transcript_error = None - - -def _reset_eval_row_for_retry( - eval_row: CallImportEvaluationRow, - *, - metric_ids: Optional[List[UUID]] = None, - skip_revoke: bool = False, -) -> None: - """Wipe per-row state so the worker can re-run it cleanly. - - Mirrors the initial state used by ``create_call_import_evaluation`` - when it first inserts a row, with the addition of revoking any - lingering Celery task id. - - When ``metric_ids`` is provided, this is a **metric-subset retry**: - only the scores for those metrics are removed from - ``metric_scores`` (other metrics' previously-computed values are - preserved so the worker's partial-merge write keeps them intact). - Otherwise the entire ``metric_scores`` dict is reset, matching the - legacy behaviour. - """ - if ( - not skip_revoke - and eval_row.celery_task_id - and eval_row.status in {"pending", "running"} - ): - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke(eval_row.celery_task_id, terminate=False) - except Exception: # noqa: BLE001 — revoke is best-effort - pass - eval_row.status = "pending" - eval_row.error_message = None - if metric_ids: - # Strip ONLY the targeted metric keys. Both string and UUID - # forms can appear in ``metric_scores`` depending on which - # code path wrote the dict, so we normalise to lower-case - # strings for the comparison. - existing = ( - eval_row.metric_scores if isinstance(eval_row.metric_scores, dict) else {} - ) - target_keys = {str(mid).lower() for mid in metric_ids} - eval_row.metric_scores = { - key: value - for key, value in existing.items() - if str(key).lower() not in target_keys - } - else: - eval_row.metric_scores = {} - eval_row.started_at = None - eval_row.finished_at = None - eval_row.celery_task_id = None - - -def _enqueue_eval_rows_with_optional_transcribe( - db: Session, - evaluation: CallImportEvaluation, - eval_rows_with_source: List[ - Tuple[CallImportEvaluationRow, CallImportRow] - ], - *, - transcribe_overwrite: bool = False, - restricted_metric_ids: Optional[List[UUID]] = None, -) -> Tuple[int, int]: - """Schedule throttled evaluation dispatch for pending eval rows. - - Returns ``(evaluate_only_count, transcribe_then_evaluate_count)`` for - logging/UI compatibility. Actual Celery fan-out is handled by - :func:`dispatch_evaluation_rows_task` under Redis fair-share limits. - """ - from app.workers.concurrency.eval_dispatch import _needs_transcribe_for_eval - from app.workers.concurrency.fair_dispatch import ( - schedule_fair_dispatch, - store_evaluation_transcribe_overwrite, - store_row_restricted_metrics, - ) - - eval_only_count = 0 - transcribe_count = 0 - if eval_rows_with_source: - for eval_row, source_row in eval_rows_with_source: - if _needs_transcribe_for_eval( - evaluation, - source_row, - transcribe_overwrite=transcribe_overwrite, - ): - transcribe_count += 1 - else: - eval_only_count += 1 - - restricted_metric_ids_str: Optional[List[str]] = ( - [str(mid) for mid in restricted_metric_ids] - if restricted_metric_ids - else None - ) - if restricted_metric_ids_str: - for eval_row, _ in eval_rows_with_source: - store_row_restricted_metrics(eval_row.id, restricted_metric_ids_str) - else: - restricted_metric_ids_str = ( - [str(mid) for mid in restricted_metric_ids] if restricted_metric_ids else None - ) - store_evaluation_transcribe_overwrite( - evaluation.id, - overwrite=transcribe_overwrite, - ) - schedule_fair_dispatch(max_workspace_turns=999) - return eval_only_count, transcribe_count - - -def _apply_telephony_retry_overrides( - db: Session, - *, - call_import: CallImport, - organization_id: UUID, - payload: CallImportEvaluationRetryRequest, -) -> None: - """Pin or clear telephony credentials on the batch for this retry pass.""" - fields_set = payload.model_fields_set - if ( - "provider" not in fields_set - and "telephony_integration_id" not in fields_set - ): - return - - from app.api.v1.routes.call_imports import _resolve_telephony_integration - - if payload.telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, - organization_id, - payload.telephony_integration_id, - payload.provider or "", - ) - call_import.provider = integration.provider - call_import.telephony_integration_id = integration.id - else: - call_import.provider = None - call_import.telephony_integration_id = None - db.flush() - - -def _apply_retry_overrides( - db: Session, - evaluation: CallImportEvaluation, - organization_id: UUID, - payload: CallImportEvaluationRetryRequest, -) -> None: - """Validate + persist the LLM/STT override fields on the run. - - Mirrors the validation in ``create_call_import_evaluation`` but - only touches the fields the caller actually sent — leaving any - field ``None`` preserves the run's existing value. Raises - ``HTTPException(400)`` on bad input so the route handler can let - FastAPI turn it into a clean 400 response. - """ - # --- LLM provider + model (must be sent together) --- - if payload.llm_provider is not None or payload.llm_model is not None: - if not (payload.llm_provider and payload.llm_model): - raise HTTPException( - status_code=400, - detail=( - "Both llm_provider and llm_model are required when " - "overriding the run LLM on retry." - ), - ) - try: - evaluation.llm_provider = ModelProvider( - payload.llm_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Unknown LLM provider '{payload.llm_provider}'. " - "Valid keys are documented in ModelProvider." - ), - ) - new_model = payload.llm_model.strip() or None - if not new_model: - raise HTTPException( - status_code=400, detail="llm_model cannot be empty." - ) - evaluation.llm_model = new_model - - # --- LLM credential pin --- - if payload.llm_credential_id is not None: - cred = ( - db.query(AIProvider) - .filter( - AIProvider.id == payload.llm_credential_id, - AIProvider.organization_id == organization_id, - ) - .first() - ) - if not cred: - raise HTTPException( - status_code=400, - detail=( - "The provided llm_credential_id does not exist in " - "this organization." - ), - ) - evaluation.llm_credential_id = payload.llm_credential_id - - if payload.llm_config is not None: - evaluation.llm_config = payload.llm_config - - # --- Per-metric LLM overrides --- - # We accept the same dict shape as the create endpoint but - # constrain keys to leaf metrics that are actually in this run. - # Passing an empty dict explicitly clears existing overrides. - if payload.metric_llm_overrides is not None: - valid_leaf_ids = { - str(mid) for mid in (evaluation.selected_metric_ids or []) - } - overrides_payload: Dict[str, Dict[str, Any]] = {} - for metric_id, override in payload.metric_llm_overrides.items(): - if metric_id not in valid_leaf_ids: - raise HTTPException( - status_code=400, - detail=( - "metric_llm_overrides references metric " - f"{metric_id} which is not a leaf metric in " - "this run." - ), - ) - override_dict: Dict[str, Any] = {} - if override.provider is not None: - if not override.model: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} has a " - "provider but no model." - ), - ) - try: - override_dict["provider"] = ModelProvider( - override.provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} uses " - f"unknown provider '{override.provider}'." - ), - ) - override_dict["model"] = override.model.strip() - elif override.model: - raise HTTPException( - status_code=400, - detail=( - f"Override for metric {metric_id} has a model " - "but no provider." - ), - ) - if override.credential_id is not None: - override_dict["credential_id"] = str(override.credential_id) - if override.llm_config is not None: - override_dict["llm_config"] = override.llm_config - if override_dict: - overrides_payload[metric_id] = override_dict - evaluation.metric_llm_overrides = overrides_payload or None - - # --- STT provider + model (must be sent together) --- - if payload.stt_provider is not None or payload.stt_model is not None: - if not (payload.stt_provider and payload.stt_model): - raise HTTPException( - status_code=400, - detail=( - "Both stt_provider and stt_model are required " - "when overriding the run STT on retry." - ), - ) - try: - evaluation.stt_provider = ModelProvider( - payload.stt_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=f"Unknown STT provider '{payload.stt_provider}'.", - ) - new_stt_model = payload.stt_model.strip() or None - if not new_stt_model: - raise HTTPException( - status_code=400, detail="stt_model cannot be empty." - ) - evaluation.stt_model = new_stt_model - - # --- STT credential pin --- - if payload.stt_credential_id is not None: - evaluation.stt_credential_id = payload.stt_credential_id - - # --- LLM diariser provider + model (must be sent together) --- - if ( - payload.diarization_llm_provider is not None - or payload.diarization_llm_model is not None - ): - if not ( - payload.diarization_llm_provider - and payload.diarization_llm_model - ): - raise HTTPException( - status_code=400, - detail=( - "Both diarization_llm_provider and " - "diarization_llm_model are required when overriding " - "the run diariser on retry." - ), - ) - try: - evaluation.diarisation_llm_provider = ModelProvider( - payload.diarization_llm_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - "Unknown diarisation LLM provider " - f"'{payload.diarization_llm_provider}'." - ), - ) - new_diariser_model = ( - payload.diarization_llm_model.strip() or None - ) - if not new_diariser_model: - raise HTTPException( - status_code=400, - detail="diarization_llm_model cannot be empty.", - ) - evaluation.diarisation_llm_model = new_diariser_model - - if payload.diarization_llm_credential_id is not None: - evaluation.diarisation_llm_credential_id = ( - payload.diarization_llm_credential_id - ) - - # ``diarization_prompt`` semantics: None = leave untouched; - # empty string = clear (fall back to the canonical default at - # worker time); anything else = persist verbatim. - if payload.diarization_prompt is not None: - cleaned = payload.diarization_prompt.strip() - evaluation.diarisation_prompt = cleaned or None - - if payload.transcribe_mode is not None: - mode = payload.transcribe_mode.strip().lower() - if mode not in {"stt_llm", "llm_only"}: - raise HTTPException( - status_code=400, - detail=( - f"Unknown transcribe_mode '{payload.transcribe_mode}'. " - "Valid values are 'stt_llm' and 'llm_only'." - ), - ) - evaluation.transcribe_mode = mode - - -def _gather_retry_targets( - db: Session, - evaluation: CallImportEvaluation, - requested_ids: Optional[List[UUID]], - *, - include_completed: bool = False, -) -> Tuple[ - List[Tuple[CallImportEvaluationRow, CallImportRow]], - List[CallImportEvaluationRetrySkippedItem], -]: - """Resolve which rows to retry + reasons for any we refuse. - - When ``requested_ids`` is None we retry every row whose status is - ``failed`` (or every row when ``include_completed`` is also set — - used by the metric-subset retry path which legitimately wants to - recompute a metric on already-successful rows). When the caller - passes ids explicitly we still filter out rows that are currently - in flight; ``include_completed`` controls whether previously- - successful rows are eligible. - """ - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - from app.db_sharding.eval_rows import gather_retry_targets_sharded - - return gather_retry_targets_sharded( - db, - evaluation, - requested_ids, - include_completed=include_completed, - ) - - eval_rows_query = db.query(CallImportEvaluationRow).filter( - CallImportEvaluationRow.evaluation_id == evaluation.id - ) - - targets: List[Tuple[CallImportEvaluationRow, CallImportRow]] = [] - skipped: List[CallImportEvaluationRetrySkippedItem] = [] - - if requested_ids is None: - if include_completed: - # "Retry everything" path used by the metric-subset re-run - # UI. Still skip in-flight rows below so we don't trample - # work the worker is actively doing. - candidate_rows = eval_rows_query.filter( - CallImportEvaluationRow.status.in_(["failed", "completed"]) - ).all() - else: - candidate_rows = eval_rows_query.filter( - CallImportEvaluationRow.status == "failed" - ).all() - else: - requested_set = set(requested_ids) - candidate_rows = eval_rows_query.filter( - CallImportEvaluationRow.id.in_(requested_set) - ).all() - found_ids = {row.id for row in candidate_rows} - for missing in requested_set - found_ids: - skipped.append( - CallImportEvaluationRetrySkippedItem( - eval_row_id=missing, - reason="unknown", - ) - ) - - if not candidate_rows: - return targets, skipped - - source_row_ids = [row.call_import_row_id for row in candidate_rows] - source_rows = ( - db.query(CallImportRow) - .filter(CallImportRow.id.in_(source_row_ids)) - .all() - ) - source_by_id = {row.id: row for row in source_rows} - - for eval_row in candidate_rows: - if eval_row.status in {"pending", "running"}: - skipped.append( - CallImportEvaluationRetrySkippedItem( - eval_row_id=eval_row.id, - reason="in_progress", - ) - ) - continue - if eval_row.status == "completed" and not include_completed: - skipped.append( - CallImportEvaluationRetrySkippedItem( - eval_row_id=eval_row.id, - reason="completed", - ) - ) - continue - source_row = source_by_id.get(eval_row.call_import_row_id) - if source_row is None: - skipped.append( - CallImportEvaluationRetrySkippedItem( - eval_row_id=eval_row.id, - reason="source_row_missing", - ) - ) - continue - targets.append((eval_row, source_row)) - - return targets, skipped - - -@router.post( - "/{eval_id}/retry", - response_model=CallImportEvaluationRetryResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="retryCallImportEvaluation", -) -async def retry_call_import_evaluation( - call_import_id: UUID, - eval_id: UUID, - payload: Optional[CallImportEvaluationRetryRequest] = Body(default=None), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationRetryResponse: - """Re-enqueue failed rows in an evaluation run. - - Default behavior (no body) is "retry every row that failed". Pass - ``eval_row_ids`` to scope the retry to a specific subset (e.g. the - single row a user clicked in the UI). Rows that are still - in-flight or already completed are returned in ``skipped`` rather - than re-enqueued, so this endpoint is always safe to call. - - When ``metric_ids`` is set in the payload, this is a **metric- - subset retry**: only the listed metrics are recomputed (and merged - into the row's existing ``metric_scores`` — other metrics' values - are preserved). The route auto-flips ``include_completed=True`` in - that case so previously-successful rows are eligible for re- - scoring; without it the call would no-op because every row would - be skipped as ``completed``. - - The worker contract is the same as the create endpoint: - ``evaluate_call_import_row_task(eval_row_id, [restricted_metric_ids])``. - When the run is configured for diarised transcripts and the row's - diarised transcript is missing, we chain through - ``transcribe_call_import_row_task`` first — matching the - auto-transcribe behavior of POST ``/evaluations``. - """ - del api_key - call_import = _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - requested_ids = payload.eval_row_ids if payload else None - # Metric-subset retry: validate that every metric is something this - # run actually scored. Empty list is rejected too — callers that - # want a full re-run should omit the field entirely. - # - # ``selected_metric_ids`` holds the LEAVES only (children for - # hierarchical / category metrics, standalone metrics otherwise) — - # see ``leaf_metric_ids`` in :func:`create_call_import_evaluation`. - # Parent IDs for hierarchical metrics live separately in - # ``selected_metric_groups`` (``{parent_id: [child_ids]}``) so the - # UI can reconstruct the tree without round-tripping through the - # metric table. - # - # The Re-run-metrics modal surfaces PARENTS for hierarchical - # metrics (it suppresses individual children via - # ``childrenInGroups`` in ``CallImportEvaluationDetail.tsx``), so a - # naive ``metric_ids ⊆ selected_metric_ids`` check rejects every - # parent-ID request with a misleading "unknown ids" 400. We accept - # both shapes here and then EXPAND any parent IDs into - # ``{parent_id, *child_ids}`` so the downstream helpers see the - # full set of keys that need clearing + the full set of leaves - # that need re-scoring. - metric_ids: Optional[List[UUID]] = ( - payload.metric_ids if payload else None - ) - if metric_ids is not None: - if not metric_ids: - raise HTTPException( - status_code=400, - detail=( - "metric_ids must be a non-empty list. Omit the " - "field to re-run all metrics." - ), - ) - - leaf_set: Set[str] = { - str(item).lower() - for item in (evaluation.selected_metric_ids or []) - } - # ``selected_metric_groups`` is a dict ``{parent_id_str: - # [child_id_str, ...]}`` (see line ~487 in - # ``create_call_import_evaluation``). We tolerate stale data - # (string / UUID / non-dict) without crashing the retry path — - # if it's malformed we just treat it as "no parents" and fall - # back to the leaf-only check. - groups_raw = ( - evaluation.selected_metric_groups - if isinstance(evaluation.selected_metric_groups, dict) - else {} - ) - parent_to_children_str: Dict[str, List[str]] = {} - for parent_key, children_raw in groups_raw.items(): - if not isinstance(children_raw, (list, tuple)): - continue - children_norm = [ - str(c).lower() for c in children_raw if c is not None - ] - parent_to_children_str[str(parent_key).lower()] = children_norm - parent_set = set(parent_to_children_str.keys()) - - unknown = [ - mid for mid in metric_ids - if str(mid).lower() not in leaf_set - and str(mid).lower() not in parent_set - ] - if unknown: - raise HTTPException( - status_code=400, - detail=( - "metric_ids must be a subset of this evaluation's " - f"selected metrics; unknown ids: {[str(u) for u in unknown]}." - ), - ) - - # Expand parent IDs into ``{parent, *children}`` so: - # * ``_reset_eval_row_for_retry`` strips BOTH the parent - # entry (with ``chosen_child_id`` / rationale) AND every - # per-child boolean entry that the LLM evaluator wrote - # under each child's ID (see - # ``app/workers/tasks/helpers/llm_evaluation.py`` lines - # 1584 and 1649). - # * ``_enqueue_eval_rows_with_optional_transcribe`` → - # ``evaluate_call_import_row_task`` filters the work-list - # off ``selected_metric_ids`` (leaves), so we MUST hand it - # the child IDs for the parent to actually get re-scored. - # Leaves pass through unchanged. - expanded: List[UUID] = [] - seen: Set[str] = set() - for mid in metric_ids: - mid_norm = str(mid).lower() - children_str = parent_to_children_str.get(mid_norm) - if children_str is not None: - # Parent: include the parent ID itself (so the parent - # entry in ``metric_scores`` is also cleared) and all - # of its children. - candidates = [mid_norm, *children_str] - else: - candidates = [mid_norm] - for candidate in candidates: - if candidate in seen: - continue - try: - expanded.append(UUID(candidate)) - except (TypeError, ValueError): - # Defensive: skip junk values rather than 500. - continue - seen.add(candidate) - metric_ids = expanded - - # ``include_completed`` is auto-enabled when the caller asked for a - # metric subset (otherwise the metric-subset retry would always - # no-op on a green run, which is the whole reason this feature - # exists). The explicit payload flag wins for full-row retries. - include_completed = bool( - (payload.include_completed if payload else False) - or (metric_ids is not None) - ) - - transcribe_overwrite = bool( - payload.transcribe_overwrite if payload else False - ) - - skipped: List[CallImportEvaluationRetrySkippedItem] = [] - if requested_ids is None: - from app.db_sharding.eval_rows import count_evaluation_rows_for_run - from app.db_sharding.sessions import is_sharding_enabled - - if is_sharding_enabled(): - statuses = ( - ["failed", "completed"] if include_completed else ["failed"] - ) - target_count = count_evaluation_rows_for_run( - db, eval_id, statuses=statuses - ) - else: - from sqlalchemy import func - - count_query = db.query(func.count(CallImportEvaluationRow.id)).filter( - CallImportEvaluationRow.evaluation_id == eval_id - ) - if include_completed: - count_query = count_query.filter( - CallImportEvaluationRow.status.in_(["failed", "completed"]) - ) - else: - count_query = count_query.filter( - CallImportEvaluationRow.status == "failed" - ) - target_count = int(count_query.scalar() or 0) - if target_count == 0: - return CallImportEvaluationRetryResponse( - requeued=0, - transcribe_requeued=0, - skipped=skipped, - ) - else: - targets, skipped = _gather_retry_targets( - db, - evaluation, - requested_ids, - include_completed=include_completed, - ) - if not targets: - return CallImportEvaluationRetryResponse( - requeued=0, - transcribe_requeued=0, - skipped=skipped, - ) - target_count = len(targets) - - # Apply LLM / STT overrides BEFORE enqueueing so the persisted run - # config is correct by the time the worker reads it. - if payload is not None: - _apply_retry_overrides(db, evaluation, organization_id, payload) - _apply_telephony_retry_overrides( - db, - call_import=call_import, - organization_id=organization_id, - payload=payload, - ) - - evaluation.error_message = None - evaluation.finished_at = None - evaluation.status = "running" - if not evaluation.started_at: - from datetime import datetime, timezone - - evaluation.started_at = datetime.now(timezone.utc) - - _claim_evaluation_bulk_operation(eval_id, "retry") - db.commit() - - from app.workers.tasks.call_import_bulk_ops import ( - retry_call_import_evaluation_task, - ) - - retry_call_import_evaluation_task.delay( - str(eval_id), - { - "eval_row_ids": [str(rid) for rid in requested_ids] - if requested_ids - else None, - "metric_ids": [str(mid) for mid in metric_ids] if metric_ids else None, - "include_completed": include_completed, - "transcribe_overwrite": transcribe_overwrite, - }, - ) - - return CallImportEvaluationRetryResponse( - requeued=target_count, - transcribe_requeued=0, - skipped=skipped, - ) - - -@router.post( - "/{eval_id}/rows/{eval_row_id}/retry", - response_model=CallImportEvaluationRowResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="retryCallImportEvaluationRow", -) -async def retry_call_import_evaluation_row( - call_import_id: UUID, - eval_id: UUID, - eval_row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportEvaluationRowResponse: - """Re-enqueue a single failed evaluation row. - - Convenience wrapper around ``retry_call_import_evaluation`` for the - "Retry this row" affordance in the row table. Returns the - refreshed row so the UI can update its badge immediately, without - waiting for the next polling tick. - """ - del api_key - _require_import(db, call_import_id, organization_id) - - evaluation = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.id == eval_id, - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .first() - ) - if not evaluation: - raise HTTPException( - status_code=404, detail="Call import evaluation not found" - ) - - _require_no_evaluation_bulk_operation(eval_id) - - from app.db_sharding.eval_rows import ( - evaluation_row_session, - find_evaluation_row_in_run, - ) - from app.db_sharding.sessions import is_sharding_enabled - - eval_row, _source_stub = find_evaluation_row_in_run(db, eval_id, eval_row_id) - if eval_row is None: - raise HTTPException( - status_code=404, detail="Evaluation row not found in this run" - ) - - if eval_row.status in {"pending", "running"}: - raise HTTPException( - status_code=409, - detail=( - "This row is still in progress — wait for it to finish " - "before retrying." - ), - ) - - targets, _ = _gather_retry_targets(db, evaluation, [eval_row.id]) - if not targets: - raise HTTPException( - status_code=409, - detail=( - "This row cannot be retried in its current state " - f"(status={eval_row.status})." - ), - ) - - if is_sharding_enabled(): - with evaluation_row_session(eval_row_id) as ( - row_db, - _catalog_db, - eval_row, - source_row, - _shard_id, - ): - _prepare_source_row_for_retry(source_row, transcribe_overwrite=False) - _reset_eval_row_for_retry(eval_row) - row_db.commit() - targets = [(eval_row, source_row)] - else: - for er, source_row in targets: - _prepare_source_row_for_retry(source_row, transcribe_overwrite=False) - _reset_eval_row_for_retry(er) - - evaluation.error_message = None - evaluation.finished_at = None - evaluation.status = "running" - if not evaluation.started_at: - from datetime import datetime, timezone - - evaluation.started_at = datetime.now(timezone.utc) - db.flush() - _rollup_evaluation_status(evaluation, db) - db.commit() - - try: - _enqueue_eval_rows_with_optional_transcribe(db, evaluation, targets) - except Exception as exc: # noqa: BLE001 - logger.exception( - "Failed to re-enqueue retry for evaluation row {}", eval_row_id - ) - if is_sharding_enabled(): - with evaluation_row_session(eval_row_id) as ( - row_db, - _catalog_db, - eval_row, - _source_row, - _shard_id, - ): - eval_row.status = "failed" - eval_row.error_message = f"Failed to re-enqueue retry: {exc}" - row_db.commit() - else: - eval_row.status = "failed" - eval_row.error_message = f"Failed to re-enqueue retry: {exc}" - _rollup_evaluation_status(evaluation, db) - db.commit() - raise HTTPException( - status_code=500, - detail=f"Failed to re-enqueue retry: {exc}", - ) - - if is_sharding_enabled(): - with evaluation_row_session(eval_row_id) as ( - _row_db, - _catalog_db, - eval_row, - source_row, - _shard_id, - ): - return _to_evaluation_row_response(eval_row, source_row, evaluation) - - db.refresh(eval_row) - source_row = targets[0][1] - return _to_evaluation_row_response(eval_row, source_row, evaluation) - - -from app.core.auth.capabilities import EVALS_RUN, EVALS_VIEW, REPORTS_GENERATE -from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities - -apply_workspace_route_capabilities( - router, - view_capability=EVALS_VIEW, - manage_capability=EVALS_RUN, - run_capability=EVALS_RUN, - report_capability=REPORTS_GENERATE, -) +"""Evaluation routes scoped to a Call Import batch.""" + +from __future__ import annotations + +import asyncio +import csv +import base64 +import io +import json +import math +import re +import statistics +from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple +from uuid import UUID + +from datetime import date, datetime, timedelta, timezone + +from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, Query, Response, status +from fastapi.responses import StreamingResponse +from loguru import logger +from pydantic import BaseModel, Field, field_validator +from sqlalchemy import desc, func, or_, text +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.core.auth import Principal, get_principal +from app.core.auth.capabilities import REPORTS_GENERATE, capability_denied_message +from app.database import get_db +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.call_imports.audit import ( + actor_emails_for_evaluation, + emails_for_user_ids, + stamp_evaluation_actor, + user_ids_from_evaluations, +) +from app.services.workspace_rbac import resolve_workspace_capabilities +from app.models.database import ( + AIProvider, + CallImport, + CallImportEvaluation, + CallImportEvaluationReportSnapshot, + CallImportEvaluationRow, + CallImportRow, + Metric, + PromptPartial, + Workspace, +) +from app.models.enums import CallImportRowStatus, ModelProvider +from app.models.schemas import ( + CallImportEvaluationAggregateResponse, + CallImportEvaluationBulkDelete, + CallImportEvaluationBulkActionResponse, + CallImportEvaluationCreate, + CallImportEvaluationListResponse, + CallImportEvaluationResponse, + CallImportEvaluationRetryRequest, + CallImportEvaluationRetryResponse, + CallImportEvaluationRetrySkippedItem, + CallImportEvaluationRowListResponse, + CallImportEvaluationRowResponse, + CallImportEvaluationUpdate, + CallImportMetricAggregate, + CallImportMetricHistogramBucket, + CallImportMetricLabelPair, + CallImportMetricSummary, + CallImportMetricValueCount, + DiscoveredLabelDeleteRequest, + DiscoveredLabelItem, + DiscoveredLabelMergeRequest, + DiscoveredLabelsResponse, + DiscoveredMetricDeleteRequest, + DiscoveredMetricItem, + DiscoveredMetricMergeRequest, + DiscoveredMetricsResponse, + EvaluationInsightsRequest, + EvaluationTldrSummary, + EvaluationMetricClustersRequest, + EvaluationMetricClustersState, + EvaluationPromptImprovementsRequest, + EvaluationPromptImprovementsState, + MetricFailurePoliciesResponse, + MetricFailurePoliciesSaveRequest, + MetricFailurePolicy, + MetricClusterEligibleRow, + MetricClusterEligibleRowsResponse, + EvaluationUserInsightsRequest, + EvaluationUserInsightsState, + MetricFlowEdge, + MetricPeriodDelta, + MetricFlowNode, + MetricFlowResponse, +) +from app.services.reporting.call_import_evaluation_pdf_report import ( + call_import_evaluation_pdf_report_service, +) +from app.services.call_import_metric_clusters import ( + METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + estimate_metric_clusters_llm_calls, + filter_completed_row_pairs, + list_eligible_cluster_rows, + metric_clusters_raw_is_cancelled, + metric_clusters_state_from_raw, + metric_clusters_state_to_db, +) +from app.services.metric_failure_policy import ( + aggregate_primary_percent, + build_failure_policy_previews, + effective_policies, + failure_rate_percent_from_rows, + failure_policies_to_db, + has_clusterable_metrics, + merge_clustering_policies, + merge_failure_policies_into_raw, + policies_from_evaluation_raw, + validate_failure_policies_for_metrics, +) +from app.services.call_import_user_insights import ( + normalize_max_llm_calls, + total_llm_calls_for_rows, + user_insights_state_from_raw, +) + +router = APIRouter( + prefix="/call-imports/{call_import_id}/evaluations", + tags=["Call Import Evaluations"], + dependencies=[Depends(require_enterprise_feature("call_imports"))], +) + + +class CallImportEvaluationPdfReportRequest(BaseModel): + vendor_name: str = Field(..., min_length=1, max_length=120) + report_type: Literal["external", "internal"] = "external" + include_weekly_delta: bool = False + include_period_delta: bool = False + baseline_evaluation_id: Optional[str] = None + period_label: Optional[str] = Field(default=None, max_length=64) + use_case: Optional[str] = Field(default=None, max_length=120) + internal_brand_image_id: Optional[str] = None + external_brand_image_id: Optional[str] = None + report_config: Dict[str, Any] = Field(default_factory=dict) + platform_base_url: Optional[str] = Field( + default=None, + max_length=512, + description="Frontend origin for deep links to example calls in internal PDFs.", + ) + + @field_validator("vendor_name") + @classmethod + def _clean_vendor_name(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("Vendor name is required.") + return cleaned + + +class CallImportEvaluationBaselineCandidate(BaseModel): + evaluation_id: str + name: str + dataset: str + period_label: Optional[str] = None + period_start: Optional[date] = None + period_end: Optional[date] = None + period_display: str + completed_rows: int + created_at: datetime + is_default: bool = False + + +class CallImportEvaluationBaselineCandidatesResponse(BaseModel): + items: List[CallImportEvaluationBaselineCandidate] + default_evaluation_id: Optional[str] = None + + +def _require_import( + db: Session, + call_import_id: UUID, + organization_id: UUID, +) -> CallImport: + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException(status_code=404, detail="Call import not found") + return call_import + + +def require_call_import_capability(capability: str): + """Ensure the caller has *capability* in the call import's workspace (not just the header).""" + + def _dep( + call_import_id: UUID, + principal: Principal = Depends(get_principal), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), + ) -> CallImport: + call_import = _require_import(db, call_import_id, organization_id) + caps, _, role = resolve_workspace_capabilities( + db, + principal=principal, + workspace_id=call_import.workspace_id, + organization_id=organization_id, + ) + if capability not in caps: + raise HTTPException( + status_code=403, + detail=capability_denied_message( + capability, + role_name=role.name if role else None, + workspace_label="the active workspace", + ), + ) + return call_import + + return _dep + + +def _flatten_transcript(text: Optional[str]) -> str: + """Collapse a multi-line transcript onto a single line for spreadsheet export. + + The diarised transcript is stored as ``: `` lines joined + by ``\\n`` because the in-app ``TranscriptView`` parses those line + breaks to render chat bubbles. In Excel / Google Sheets that same + newline-per-turn formatting causes each cell to balloon vertically, + which the user reads as "lots of empty space on top of the cell". + Flattening at export time keeps the DB shape intact while giving the + spreadsheet a single-line cell per row. + """ + if not text: + return "" + parts = [ + segment.strip() + for segment in text.replace("\r\n", "\n").replace("\r", "\n").split("\n") + ] + return " ".join(p for p in parts if p) + + +def _evaluated_transcript_source_label( + evaluation: CallImportEvaluation, + source_row: CallImportRow, +) -> str: + """Label which transcript source this row was scored against.""" + source = (evaluation.transcript_source or "diarised").strip().lower() + if source == "production": + if not (source_row.transcript or "").strip(): + return "" + return "Production" + if not (source_row.diarised_transcript or "").strip(): + return "" + return "Diarised" + + +def _pick_evaluation_row_transcript( + source_row: Optional[CallImportRow], + evaluation: Optional[CallImportEvaluation] = None, +) -> Optional[str]: + """Transcript shown in evaluation row detail for the run's source.""" + if source_row is None: + return None + source = ( + (evaluation.transcript_source or "diarised").strip().lower() + if evaluation is not None + else "diarised" + ) + if source == "production": + raw = (source_row.transcript or "").strip() + return raw or None + diarised = (source_row.diarised_transcript or "").strip() + if diarised: + return diarised + raw = (source_row.transcript or "").strip() + return raw or None + + +def _to_evaluation_row_response( + eval_row_obj: CallImportEvaluationRow, + source_row: Optional[CallImportRow], + evaluation: Optional[CallImportEvaluation] = None, +) -> CallImportEvaluationRowResponse: + """Serialize one evaluation row plus joined source-row metadata.""" + return CallImportEvaluationRowResponse( + id=eval_row_obj.id, + evaluation_id=eval_row_obj.evaluation_id, + call_import_row_id=eval_row_obj.call_import_row_id, + row_index=source_row.row_index if source_row else None, + conversation_id=source_row.conversation_id if source_row else None, + transcript=_pick_evaluation_row_transcript(source_row, evaluation), + raw_columns=source_row.raw_columns if source_row else None, + recording_url=source_row.recording_url if source_row else None, + recording_date=source_row.recording_date if source_row else None, + recording_s3_key=source_row.recording_s3_key if source_row else None, + diarised_transcript_status=( + source_row.diarised_transcript_status if source_row else None + ), + diarised_transcript_error=( + source_row.diarised_transcript_error if source_row else None + ), + status=eval_row_obj.status, + metric_scores=eval_row_obj.metric_scores or {}, + error_message=eval_row_obj.error_message, + started_at=eval_row_obj.started_at, + finished_at=eval_row_obj.finished_at, + created_at=eval_row_obj.created_at, + updated_at=eval_row_obj.updated_at, + ) + + +def _serialize_selected_metric_ids(value) -> List[UUID]: + result: List[UUID] = [] + if not isinstance(value, list): + return result + for item in value: + try: + result.append(UUID(str(item))) + except (TypeError, ValueError): + continue + return result + + +def _metrics_for_ids(db: Session, org_id: UUID, ids: List[UUID]) -> List[Metric]: + if not ids: + return [] + rows = ( + db.query(Metric) + .filter( + Metric.organization_id == org_id, + Metric.id.in_(ids), + ) + .all() + ) + by_id = {row.id: row for row in rows} + return [by_id[mid] for mid in ids if mid in by_id] + + +def _expand_metric_selection( + db: Session, + org_id: UUID, + selected_ids: List[UUID], +) -> Tuple[List[Metric], Dict[UUID, List[Metric]]]: + """Resolve user-supplied metric ids into actual leaves + parent grouping. + + Rules: + * If a parent id is in ``selected_ids`` and no specific children of + that parent are also listed, include EVERY enabled child of that + parent. + * If a parent id AND some of its children are listed, include only + the listed children (treat the parent selection as the + "container" so users can deselect labels). + * Standalone metrics (no parent, no children) pass through + unchanged. + * Disabled metrics are filtered out at this layer so the caller + doesn't have to repeat the check. + + Returns: + (effective_metrics, parent_to_children) + + ``effective_metrics`` is the deduplicated list of metrics the + worker will actually score (children + standalone). Order is + preserved from ``selected_ids`` for display stability. + + ``parent_to_children`` maps each parent metric id (UUID) to the + list of its selected children. Useful for grouping in the LLM + prompt builder. + """ + if not selected_ids: + return [], {} + + requested = list(selected_ids) + initial_rows = ( + db.query(Metric) + .filter( + Metric.organization_id == org_id, + Metric.id.in_(requested), + ) + .all() + ) + initial_by_id = {row.id: row for row in initial_rows} + + parent_ids_requested = { + m.id for m in initial_rows if m.selection_mode and not m.parent_metric_id + } + # Map parent id -> children explicitly requested by the user. + explicit_children_by_parent: Dict[UUID, List[Metric]] = {} + for m in initial_rows: + if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: + explicit_children_by_parent.setdefault( + m.parent_metric_id, [] + ).append(m) + + # For parents without explicit children, hydrate every enabled child. + parents_needing_full_expansion = [ + pid + for pid in parent_ids_requested + if pid not in explicit_children_by_parent + ] + auto_expanded_children: Dict[UUID, List[Metric]] = {} + if parents_needing_full_expansion: + for pid in parents_needing_full_expansion: + child_rows = ( + db.query(Metric) + .filter( + Metric.organization_id == org_id, + Metric.parent_metric_id == pid, + Metric.enabled.is_(True), + ) + .order_by(Metric.created_at.asc()) + .all() + ) + auto_expanded_children[pid] = child_rows + + parent_to_children: Dict[UUID, List[Metric]] = {} + for pid in parent_ids_requested: + children = explicit_children_by_parent.get( + pid + ) or auto_expanded_children.get(pid, []) + # Drop disabled children so the worker doesn't waste a slot on + # them. Empty parents (no enabled children) are still tracked + # because the UI may want to show "0 of 0" rather than swallow + # them silently. + parent_to_children[pid] = [c for c in children if c.enabled] + + effective: List[Metric] = [] + seen: set[UUID] = set() + for mid in requested: + m = initial_by_id.get(mid) + if m is None: + continue + if m.selection_mode and not m.parent_metric_id: + # Parent row itself is not scored — only its children. + for child in parent_to_children.get(m.id, []): + if child.id in seen or not child.enabled: + continue + seen.add(child.id) + effective.append(child) + continue + if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: + # Already accounted for via the parent expansion above. + continue + if not m.enabled: + continue + if m.id in seen: + continue + seen.add(m.id) + effective.append(m) + + return effective, parent_to_children + + +def _evaluation_bulk_operation_for_response( + evaluation_id: UUID, +) -> Optional[str]: + from app.services.call_imports.evaluation_bulk_op import ( + get_evaluation_bulk_operation, + ) + + return get_evaluation_bulk_operation(evaluation_id) + + +def _serialize_eval( + db: Session, + row: CallImportEvaluation, + *, + sibling_evaluation_ids: Optional[List[UUID]] = None, + user_emails: Optional[Dict[UUID, str]] = None, +) -> CallImportEvaluationResponse: + selected_ids = _serialize_selected_metric_ids(row.selected_metric_ids) + + # Pull every metric referenced anywhere in the run's grouping (leaves, + # standalone, AND parents from selected_metric_groups) so the UI can + # render parent labels even when only children were materialized into + # selected_metric_ids. + groups_raw: Dict[str, List[str]] = {} + if isinstance(row.selected_metric_groups, dict): + for parent_str, children in row.selected_metric_groups.items(): + if not isinstance(children, list): + continue + cleaned: List[str] = [] + for c in children: + try: + UUID(str(c)) + cleaned.append(str(c)) + except (TypeError, ValueError): + continue + try: + UUID(parent_str) + groups_raw[parent_str] = cleaned + except (TypeError, ValueError): + continue + + metric_ids_for_lookup: List[UUID] = list(selected_ids) + for parent_str in groups_raw.keys(): + try: + pid = UUID(parent_str) + if pid not in metric_ids_for_lookup: + metric_ids_for_lookup.append(pid) + except (TypeError, ValueError): + continue + + metrics = _metrics_for_ids( + db, row.organization_id, metric_ids_for_lookup + ) + + from app.services.call_imports.progress_counters import merge_eval_counters_for_ui + + ui_completed_raw, ui_failed_raw = merge_eval_counters_for_ui(row) + total = int(row.total_rows or 0) + ui_completed = ( + min(ui_completed_raw, total) if total else ui_completed_raw + ) + ui_failed = min(ui_failed_raw, total) if total else ui_failed_raw + + if user_emails is None: + user_emails = emails_for_user_ids(db, user_ids_from_evaluations([row])) + created_email, updated_email = actor_emails_for_evaluation(row, user_emails) + + return CallImportEvaluationResponse( + id=row.id, + call_import_id=row.call_import_id, + organization_id=row.organization_id, + name=row.name, + selected_metric_ids=selected_ids, + selected_metric_groups=groups_raw or None, + metrics=[ + CallImportMetricSummary( + id=metric.id, + name=metric.name, + metric_type=metric.metric_type, + description=metric.description, + parent_metric_id=metric.parent_metric_id, + selection_mode=metric.selection_mode, + # Required by the Flow tab to know whether a parent + # opted into discovery; without it the + # DiscoveredLabelsPanel stays hidden even when the + # worker is actively producing discovered_labels. + allow_discovery=bool( + getattr(metric, "allow_discovery", False) + ), + ) + for metric in metrics + ], + status=row.status, + total_rows=row.total_rows, + completed_rows=ui_completed, + failed_rows=ui_failed, + error_message=row.error_message, + llm_provider=row.llm_provider, + llm_model=row.llm_model, + llm_credential_id=row.llm_credential_id, + llm_config=( + row.llm_config if isinstance(getattr(row, "llm_config", None), dict) else None + ), + metric_llm_overrides=( + row.metric_llm_overrides + if isinstance(row.metric_llm_overrides, dict) + else None + ), + stt_provider=row.stt_provider, + stt_model=row.stt_model, + stt_credential_id=row.stt_credential_id, + diarisation_llm_provider=getattr(row, "diarisation_llm_provider", None), + diarisation_llm_model=getattr(row, "diarisation_llm_model", None), + diarisation_llm_credential_id=getattr( + row, "diarisation_llm_credential_id", None + ), + diarisation_prompt=getattr(row, "diarisation_prompt", None), + transcribe_mode=( + (getattr(row, "transcribe_mode", None) or "stt_llm") + ), + transcript_source=(row.transcript_source or "diarised"), + sibling_evaluation_ids=list(sibling_evaluation_ids or []), + started_at=row.started_at, + finished_at=row.finished_at, + created_at=row.created_at, + updated_at=row.updated_at, + created_by_email=created_email, + last_updated_by_email=updated_email, + tldr_summary=_tldr_summary_payload(row), + user_insights=_user_insights_payload(row), + metric_clusters=_metric_clusters_payload(row), + discover_new_metrics=bool( + getattr(row, "discover_new_metrics", False) + ), + bulk_operation=_evaluation_bulk_operation_for_response(row.id), + ) + + +def _normalize_name(value: Optional[str]) -> Optional[str]: + """Trim user-supplied name; empty string becomes ``NULL``.""" + if value is None: + return None + trimmed = value.strip() + return trimmed or None + + +def _rollup_evaluation_status(evaluation: CallImportEvaluation, db: Session) -> None: + """Recompute counters + terminal status after rows are added/removed. + + Uses a single aggregate query instead of loading every row status. + """ + from app.workers.tasks.evaluate_call_import_row_core import ( + _apply_parent_status_from_counters, + reconcile_evaluation_counters, + ) + + reconcile_evaluation_counters(db, evaluation) + _apply_parent_status_from_counters(evaluation) + db.flush() + + if evaluation.status in {"completed", "failed", "partial"}: + from app.models.database import CallImport + from app.services.call_imports.bulk_ops import rollup_call_import_batch_status + + call_import = ( + db.query(CallImport) + .filter(CallImport.id == evaluation.call_import_id) + .first() + ) + if call_import is not None: + rollup_call_import_batch_status(db, call_import) + + +@router.post( + "", + response_model=CallImportEvaluationResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="createCallImportEvaluation", +) +async def create_call_import_evaluation( + call_import_id: UUID, + payload: CallImportEvaluationCreate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationResponse: + del api_key + call_import = _require_import(db, call_import_id, organization_id) + + metric_ids = payload.metric_ids + if not metric_ids: + raise HTTPException( + status_code=400, + detail="Select at least one metric to run the evaluation against.", + ) + + org_metrics = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.id.in_(metric_ids), + ) + .all() + ) + by_id = {metric.id: metric for metric in org_metrics} + unknown_ids = [mid for mid in metric_ids if mid not in by_id] + if unknown_ids: + raise HTTPException( + status_code=400, + detail=( + "These metric ids do not exist in your organization: " + f"{', '.join(str(mid) for mid in unknown_ids)}. " + "Refresh the metrics list and try again." + ), + ) + # Parents themselves are containers, not scored rows, so a disabled + # parent shouldn't block the run as long as it has enabled children. + # We only reject disabled rows that the worker will actually try to + # evaluate (children + standalone leaves). + disabled_leaves = [ + metric + for metric in org_metrics + if not metric.enabled + and not (metric.selection_mode and not metric.parent_metric_id) + ] + if disabled_leaves: + names = ", ".join(metric.name for metric in disabled_leaves) + raise HTTPException( + status_code=400, + detail=( + f"These metrics are disabled and cannot be evaluated: {names}. " + "Enable them on the Metrics page (or pick different ones) and " + "try again." + ), + ) + + # Expand hierarchical selection: parents auto-include their enabled + # children, mixed parent+child selections respect the user's subset. + effective_metrics, parent_to_children = _expand_metric_selection( + db, organization_id, metric_ids + ) + if not effective_metrics: + raise HTTPException( + status_code=400, + detail=( + "None of the selected metrics yielded an enabled leaf to " + "evaluate. Check that parent categories have enabled " + "children, then try again." + ), + ) + + # The effective list (children + standalone leaves) is what gets + # persisted to ``selected_metric_ids`` and scored by the worker. + # The original parents are preserved in ``selected_metric_groups`` + # so the UI can rebuild the tree later. + leaf_metric_ids: List[UUID] = [m.id for m in effective_metrics] + selected_metric_groups: Dict[str, List[str]] = { + str(pid): [str(c.id) for c in children] + for pid, children in parent_to_children.items() + } + metric_rows = effective_metrics + valid_metric_id_strs = {str(m.id) for m in metric_rows} + + # ----- Validate run-level + per-metric LLM config ----- + llm_provider_norm: Optional[str] = None + llm_model_norm: Optional[str] = None + if payload.llm_provider or payload.llm_model: + if not (payload.llm_provider and payload.llm_model): + raise HTTPException( + status_code=400, + detail="Both llm_provider and llm_model are required when overriding the run LLM.", + ) + try: + llm_provider_norm = ModelProvider( + payload.llm_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Unknown LLM provider '{payload.llm_provider}'. " + "Valid keys are documented in ModelProvider." + ), + ) + llm_model_norm = payload.llm_model.strip() or None + if not llm_model_norm: + raise HTTPException( + status_code=400, detail="llm_model cannot be empty." + ) + + if payload.llm_credential_id is not None: + cred = ( + db.query(AIProvider) + .filter( + AIProvider.id == payload.llm_credential_id, + AIProvider.organization_id == organization_id, + ) + .first() + ) + if not cred: + raise HTTPException( + status_code=400, + detail=( + "The provided llm_credential_id does not exist in this " + "organization." + ), + ) + + # Per-metric overrides: keys can be either a leaf metric id (applies + # to that metric only) or a parent metric id (applies to every + # child of that parent). Parent keys are expanded to their + # children so the worker only sees concrete leaf ids. + metric_overrides_payload: Optional[Dict[str, Dict[str, Any]]] = None + if payload.metric_llm_overrides: + metric_overrides_payload = {} + for metric_id, override in payload.metric_llm_overrides.items(): + target_leaf_ids: List[str] = [] + if metric_id in valid_metric_id_strs: + target_leaf_ids = [metric_id] + else: + # Maybe it's a parent id — expand to the children that + # are part of THIS run. + try: + parent_uuid = UUID(metric_id) + except (TypeError, ValueError): + raise HTTPException( + status_code=400, + detail=( + "metric_llm_overrides references metric " + f"{metric_id} which is not a valid UUID." + ), + ) + children_for_parent = parent_to_children.get(parent_uuid) + if not children_for_parent: + raise HTTPException( + status_code=400, + detail=( + "metric_llm_overrides references metric " + f"{metric_id} which is not in metric_ids." + ), + ) + target_leaf_ids = [str(c.id) for c in children_for_parent] + + override_dict: Dict[str, Any] = {} + if override.provider is not None: + if not override.model: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} has a provider " + "but no model." + ), + ) + try: + override_dict["provider"] = ModelProvider( + override.provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} uses unknown " + f"provider '{override.provider}'." + ), + ) + override_dict["model"] = override.model.strip() + elif override.model: + # Model without provider doesn't make sense — treat as 400 + # so the UI can fix it instead of silently falling back. + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} has a model but " + "no provider." + ), + ) + if override.credential_id is not None: + override_dict["credential_id"] = str(override.credential_id) + if override.llm_config is not None: + override_dict["llm_config"] = override.llm_config + if override_dict: + for leaf_id in target_leaf_ids: + metric_overrides_payload[leaf_id] = override_dict + + # ----- Validate auto-transcribe settings ----- + # Diarised runs auto-diarise rows missing a diarised transcript and + # require STT + diariser LLM config. Production runs score the CSV + # transcript directly and skip diarisation entirely. + use_diarised = payload.transcript_sources[0] == "diarised" + auto_transcribe = use_diarised + + transcribe_mode_norm: Optional[str] = None + stt_provider_norm: Optional[str] = None + stt_model_norm: Optional[str] = None + diarisation_llm_provider_norm: Optional[str] = None + diarisation_llm_model_norm: Optional[str] = None + diarisation_prompt_norm: Optional[str] = None + + if use_diarised: + transcribe_mode_norm = (payload.transcribe_mode or "stt_llm").strip().lower() + if transcribe_mode_norm not in {"stt_llm", "llm_only"}: + raise HTTPException( + status_code=400, + detail=( + f"Unknown transcribe_mode '{payload.transcribe_mode}'. " + "Expected 'stt_llm' or 'llm_only'." + ), + ) + + if transcribe_mode_norm == "stt_llm": + if not payload.stt_provider: + raise HTTPException( + status_code=400, + detail=( + "stt_provider is required when " + "transcribe_mode='stt_llm': every evaluation run " + "auto-diarises rows that are missing a diarised " + "transcript." + ), + ) + if not payload.stt_model: + raise HTTPException( + status_code=400, + detail=( + "stt_model is required when transcribe_mode='stt_llm'." + ), + ) + try: + stt_provider_norm = ModelProvider( + payload.stt_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Unknown STT provider '{payload.stt_provider}'.", + ) + stt_model_norm = payload.stt_model.strip() or None + if not stt_model_norm: + raise HTTPException( + status_code=400, detail="stt_model cannot be empty." + ) + else: + # llm_only — explicitly reject lingering STT inputs so the + # contract is unambiguous (the worker would ignore them but + # silent acceptance hides accidental misconfiguration). + if (payload.stt_provider or "").strip() or ( + payload.stt_model or "" + ).strip(): + raise HTTPException( + status_code=400, + detail=( + "stt_provider / stt_model must be omitted when " + "transcribe_mode='llm_only'; the LLM consumes the " + "audio directly." + ), + ) + + # --- Validate LLM diariser settings ----- + if not payload.diarization_llm_provider: + raise HTTPException( + status_code=400, + detail=( + "diarization_llm_provider is required: every evaluation " + "run diarises STT output with an LLM." + ), + ) + if not payload.diarization_llm_model: + raise HTTPException( + status_code=400, + detail=( + "diarization_llm_model is required: every evaluation " + "run diarises STT output with an LLM." + ), + ) + try: + diarisation_llm_provider_norm = ModelProvider( + payload.diarization_llm_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Unknown diarisation LLM provider " + f"'{payload.diarization_llm_provider}'." + ), + ) + diarisation_llm_model_norm = ( + payload.diarization_llm_model.strip() or None + ) + if not diarisation_llm_model_norm: + raise HTTPException( + status_code=400, + detail="diarization_llm_model cannot be empty.", + ) + diarisation_prompt_norm = ( + payload.diarization_prompt.strip() + if isinstance(payload.diarization_prompt, str) + else None + ) or None + + from app.models.enums import CallImportParameterType, CallImportStatus + from app.services.call_imports.bulk_ops import ( + count_all_source_rows, + count_completed_source_rows, + count_source_rows_with_production_transcript, + ) + + starting_from_mapped = False + if call_import.status == CallImportStatus.MAPPED: + if not call_import.source_s3_key or not call_import.source_format: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This batch has no staged source file. Upload and map " + "a CSV/Excel file before running evaluation." + ), + ) + if not call_import.schema_id: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Cannot run evaluation without a mapped schema.", + ) + from app.api.v1.routes.call_imports import ( + _ensure_blob_storage_enabled, + _resolve_schema, + _resolve_telephony_integration, + _validate_direct_url_import_ready, + ) + + workspace_id = call_import.workspace_id + schema = _resolve_schema( + db, organization_id, workspace_id, call_import.schema_id + ) + parameters = list(schema.parameters) + if not use_diarised: + transcript_mapped = any( + param.type == CallImportParameterType.TRANSCRIPT + and (call_import.parameter_mapping or {}).get(param.name) + for param in parameters + ) + if not transcript_mapped: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "No transcript column is mapped in this batch. " + "Map a schema transcript parameter to a CSV column, " + "or choose 'Diarize then evaluate'." + ), + ) + if payload.telephony_integration_id is not None: + integration = _resolve_telephony_integration( + db, + organization_id, + payload.telephony_integration_id, + payload.provider or "", + ) + else: + _validate_direct_url_import_ready( + parameters, dict(call_import.parameter_mapping or {}) + ) + integration = None + + _ensure_blob_storage_enabled() + + if integration is not None: + call_import.provider = integration.provider + call_import.telephony_integration_id = integration.id + else: + call_import.provider = None + call_import.telephony_integration_id = None + + call_import.total_rows = 0 + call_import.completed_rows = 0 + call_import.failed_rows = 0 + call_import.error_message = None + call_import.status = CallImportStatus.PROCESSING + db.commit() + db.refresh(call_import) + starting_from_mapped = True + + if use_diarised: + total_row_count = count_completed_source_rows(db, call_import.id) + else: + # Production runs score CSV text — rows need not wait for + # recording fetch to finish before they are evaluable. + total_row_count = count_source_rows_with_production_transcript( + db, call_import.id + ) + + requested_sources: List[str] = list(payload.transcript_sources) + + if ( + not use_diarised + and not starting_from_mapped + and count_all_source_rows(db, call_import.id) > 0 + and total_row_count == 0 + ): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "No rows have a production transcript. " + "Choose 'Diarize then evaluate' or import rows with " + "a transcript column." + ), + ) + + base_name = _normalize_name(payload.name) + + def _name_for_source(source: str) -> Optional[str]: + # Single-source runs preserve the user's chosen name verbatim. + del source + return base_name + + created_evaluations: List[CallImportEvaluation] = [] + + for source in requested_sources: + evaluation = CallImportEvaluation( + call_import_id=call_import.id, + organization_id=organization_id, + # Mirror the parent CallImport's workspace so listings can + # filter on workspace_id directly without joining. + workspace_id=call_import.workspace_id, + name=_name_for_source(source), + selected_metric_ids=[ + str(metric_id) for metric_id in leaf_metric_ids + ], + selected_metric_groups=selected_metric_groups or None, + status="pending", + total_rows=total_row_count, + completed_rows=0, + failed_rows=0, + llm_provider=llm_provider_norm, + llm_model=llm_model_norm, + llm_credential_id=payload.llm_credential_id, + llm_config=payload.llm_config, + metric_llm_overrides=metric_overrides_payload, + stt_provider=stt_provider_norm, + stt_model=stt_model_norm, + stt_credential_id=( + payload.stt_credential_id if auto_transcribe else None + ), + diarisation_llm_provider=diarisation_llm_provider_norm, + diarisation_llm_model=diarisation_llm_model_norm, + diarisation_llm_credential_id=( + payload.diarization_llm_credential_id if auto_transcribe else None + ), + diarisation_prompt=diarisation_prompt_norm, + transcribe_mode=transcribe_mode_norm, + transcript_source=source, + discover_new_metrics=bool( + getattr(payload, "discover_new_metrics", False) + ), + ) + stamp_evaluation_actor(evaluation, principal, creating=True) + db.add(evaluation) + db.flush() + created_evaluations.append(evaluation) + + db.commit() + for evaluation in created_evaluations: + db.refresh(evaluation) + + primary_evaluation = created_evaluations[0] + sibling_ids = [e.id for e in created_evaluations[1:]] + + if not total_row_count and not starting_from_mapped: + for evaluation in created_evaluations: + evaluation.status = "completed" + db.commit() + for evaluation in created_evaluations: + db.refresh(evaluation) + return _serialize_eval( + db, primary_evaluation, sibling_evaluation_ids=sibling_ids + ) + + if starting_from_mapped: + from app.workers.tasks.call_import_bulk_ops import ( + materialize_mapped_call_import_evaluation_task, + ) + + for evaluation in created_evaluations: + materialize_mapped_call_import_evaluation_task.delay( + str(call_import.id), + str(organization_id), + str(call_import.workspace_id), + str(evaluation.id), + transcribe_overwrite=payload.transcribe_overwrite, + ) + else: + from app.workers.tasks.call_import_bulk_ops import ( + materialize_call_import_evaluation_task, + ) + + for evaluation in created_evaluations: + materialize_call_import_evaluation_task.delay( + str(evaluation.id), + transcribe_overwrite=payload.transcribe_overwrite, + ) + + for evaluation in created_evaluations: + db.refresh(evaluation) + + return _serialize_eval( + db, primary_evaluation, sibling_evaluation_ids=sibling_ids + ) + + +@router.get( + "", + response_model=CallImportEvaluationListResponse, + operation_id="listCallImportEvaluations", +) +async def list_call_import_evaluations( + call_import_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationListResponse: + del api_key + _require_import(db, call_import_id, organization_id) + rows = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .order_by(desc(CallImportEvaluation.created_at)) + .all() + ) + email_map = emails_for_user_ids(db, user_ids_from_evaluations(rows)) + return CallImportEvaluationListResponse( + items=[_serialize_eval(db, row, user_emails=email_map) for row in rows], + total=len(rows), + ) + + +@router.get( + "/{eval_id}", + response_model=CallImportEvaluationResponse, + operation_id="getCallImportEvaluation", +) +async def get_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationResponse: + del api_key + _require_import(db, call_import_id, organization_id) + row = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + return _serialize_eval(db, row) + + +@router.get( + "/{eval_id}/rows", + response_model=CallImportEvaluationRowListResponse, + operation_id="listCallImportEvaluationRows", +) +async def list_call_import_evaluation_rows( + call_import_id: UUID, + eval_id: UUID, + page: int = Query(1, ge=1), + page_size: int = Query(100, ge=1, le=500), + q: Optional[str] = Query( + None, + description=( + "Free-text search across conversation_id and transcript " + "(case-insensitive substring match)." + ), + ), + metric_id: Optional[UUID] = Query( + None, + description=( + "If set, only return rows whose ``metric_scores[metric_id].value`` " + "exactly matches ``metric_value`` (string-compared). " + "Use together with ``metric_value``." + ), + ), + metric_value: Optional[str] = Query( + None, + description="Value to match against metric_id (string compare).", + ), + status_filter: Optional[str] = Query( + None, + alias="status", + description="Restrict to rows with this evaluation row status.", + ), + flow_parent_id: Optional[UUID] = Query( + None, + description=( + "Parent (category) metric whose ``sequence`` array should be " + "checked against ``flow_node`` and ``flow_edge_target``. Used " + "to drill into the calls behind a flow-chart node or edge." + ), + ), + flow_node: Optional[str] = Query( + None, + description=( + "If set together with ``flow_parent_id``, only return rows " + "whose sequence under that parent contains this step. Accepts " + "either a child metric UUID (resolved to slug(name)), a " + "``disc:`` discovered-label id, or a raw slug." + ), + ), + flow_edge_target: Optional[str] = Query( + None, + description=( + "Optional companion to ``flow_node``: when set, restrict to " + "rows whose sequence contains the directed transition " + "``flow_node -> flow_edge_target`` (immediately adjacent). " + "Same id format as ``flow_node``." + ), + ), + discovered_parent_id: Optional[UUID] = Query( + None, + description=( + "Parent (category) metric that defines the discovery scope " + "for ``discovered_label_key`` / ``has_discovered``." + ), + ), + discovered_label_key: Optional[str] = Query( + None, + description=( + "If set together with ``discovered_parent_id``, only return " + "rows whose ``metric_scores[parent].discovered_labels`` " + "list contains an entry with this slug (after applying " + "evaluation-level merge aliases)." + ), + ), + has_discovered: Optional[bool] = Query( + None, + description=( + "If true together with ``discovered_parent_id``, only return " + "rows that have at least one LLM-discovered label for the " + "parent. Useful to triage which calls produced novel labels." + ), + ), + sort_by: Optional[str] = Query( + None, + description=( + "Column to sort by. Accepted values: ``row_index`` (default " + "when omitted), ``conversation_id``, ``status`` (the " + "evaluation-row status), or ``metric:`` to sort " + "by ``metric_scores[].value``. Metric sorts compare " + "the extracted JSON text — adequate for booleans, enum " + "labels, and 0-1 ratings; large integer values may sort " + "lexicographically (10 before 2)." + ), + ), + sort_dir: Optional[str] = Query( + "asc", + description="Sort direction: ``asc`` (default) or ``desc``.", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationRowListResponse: + del api_key + _require_import(db, call_import_id, organization_id) + + eval_row = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not eval_row: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + query = ( + db.query(CallImportEvaluationRow, CallImportRow) + .join(CallImportRow, CallImportRow.id == CallImportEvaluationRow.call_import_row_id) + .filter(CallImportEvaluationRow.evaluation_id == eval_id) + ) + + # --- Filters ---------------------------------------------------------- + if q and q.strip(): + needle = f"%{q.strip()}%" + # Search across both transcript columns so a hit in either the + # production or the diarised version surfaces the row, + # independent of which source the evaluation actually scored. + query = query.filter( + or_( + CallImportRow.conversation_id.ilike(needle), + CallImportRow.transcript.ilike(needle), + CallImportRow.diarised_transcript.ilike(needle), + ) + ) + + if status_filter: + # The CallImportEvaluationRow.status column is a string in PG so a + # plain == filter works; we lowercase to match the stored values. + query = query.filter( + CallImportEvaluationRow.status == status_filter.strip().lower() + ) + + if metric_id is not None and metric_value is not None: + # ``metric_scores`` is a JSONB column shaped like + # ``{"": {"value": , "type": "boolean", ...}}``. We + # extract the nested ``value`` as text and compare to the user + # input as a string — that handles bool/int/enum without needing + # per-type casts. ``metric_value`` is matched case-insensitively + # so chart clicks on labels like "True" survive any casing drift + # between worker output and the chart label. + path_value = func.json_extract_path_text( + CallImportEvaluationRow.metric_scores, + str(metric_id), + "value", + ) + query = query.filter(func.lower(path_value) == metric_value.strip().lower()) + + # --- Flow chart drilldown filter ------------------------------------- + # Translates a clicked node (or edge) on the flow chart into a + # SQL filter against ``metric_scores[].sequence``. The + # frontend sends either a child UUID, a ``disc:`` discovered + # node id, or a raw slug — we normalize all three to the slug that + # actually appears in stored ``sequence`` arrays. + if flow_parent_id is not None and flow_node and flow_node.strip(): + parent_id_str_local = str(flow_parent_id) + alias_map_flow = _alias_map_for_parent(eval_row, flow_parent_id) + + def _flow_node_to_slug(raw: str) -> Optional[str]: + raw_clean = raw.strip() + if not raw_clean: + return None + if raw_clean == _FLOW_START_NODE_ID: + # The synthetic START node isn't a real sequence entry; + # filtering on it is meaningless so we skip silently. + return None + if raw_clean.startswith(_DISCOVERED_NODE_PREFIX): + return _resolve_alias( + alias_map_flow, + _slug_label(raw_clean[len(_DISCOVERED_NODE_PREFIX) :]), + ) + # Try to interpret as a child metric UUID first; fall back + # to treating it as a slug. + try: + child_uuid = UUID(raw_clean) + except (TypeError, ValueError): + return _resolve_alias(alias_map_flow, _slug_label(raw_clean)) + child = ( + db.query(Metric.name) + .filter( + Metric.id == child_uuid, + Metric.organization_id == organization_id, + ) + .first() + ) + if child and child[0]: + return _resolve_alias(alias_map_flow, _slug_label(child[0])) + return _resolve_alias(alias_map_flow, _slug_label(raw_clean)) + + from_slug = _flow_node_to_slug(flow_node) + target_slug: Optional[str] = None + if flow_edge_target and flow_edge_target.strip(): + target_slug = _flow_node_to_slug(flow_edge_target) + + if from_slug: + # The ``metric_scores`` column is declared as ``Column(JSON)`` + # in the model so on databases where the table was created + # from the model (rather than the migration) the physical + # type is ``json``, not ``jsonb``. The JSONB-only operators + # below (``jsonb_exists``, ``jsonb_array_elements_text``, + # ``@>``) require a JSONB input — we cast once up front so + # the same SQL works regardless of which path created the + # table. + scores_jsonb = ( + "(call_import_evaluation_rows.metric_scores)::jsonb" + ) + if target_slug: + # Edge filter: rows whose sequence under this parent + # contains ``from_slug`` immediately followed by + # ``target_slug``. Implemented as a correlated EXISTS + # over ``jsonb_array_elements_text`` with ORDINALITY, + # which is the portable way to express "next array + # index" against a JSONB array in Postgres. + edge_filter_sql = text( + f""" + EXISTS ( + SELECT 1 + FROM jsonb_array_elements_text( + COALESCE( + {scores_jsonb} -> :p_id -> 'sequence', + '[]'::jsonb + ) + ) WITH ORDINALITY AS s1(elem, ord) + JOIN jsonb_array_elements_text( + COALESCE( + {scores_jsonb} -> :p_id -> 'sequence', + '[]'::jsonb + ) + ) WITH ORDINALITY AS s2(elem, ord) + ON s2.ord = s1.ord + 1 + WHERE s1.elem = :from_slug + AND s2.elem = :to_slug + ) + """ + ).bindparams( + p_id=parent_id_str_local, + from_slug=from_slug, + to_slug=target_slug, + ) + query = query.filter(edge_filter_sql) + else: + # Node filter: rows whose ``metric_scores -> parent -> + # 'sequence'`` array contains ``from_slug``. We use the + # function form ``jsonb_exists`` rather than the ``?`` + # operator to avoid psycopg2 mistaking the question + # mark for a parameter placeholder. + node_filter_sql = text( + f""" + jsonb_exists( + COALESCE( + {scores_jsonb} -> :p_id -> 'sequence', + '[]'::jsonb + ), + :slug + ) + """ + ).bindparams(p_id=parent_id_str_local, slug=from_slug) + query = query.filter(node_filter_sql) + + # --- Discovered label filters --------------------------------------- + # Surfaces "which calls produced THIS LLM-discovered label" and the + # broader "which calls produced ANY LLM-discovered label". Both + # operate on ``metric_scores[].discovered_labels`` (a list + # of dicts) plus the same ``sequence`` array — covering both legacy + # rows where the slug only made it into ``sequence`` and newer + # rows where it landed in both. + if discovered_parent_id is not None and ( + discovered_label_key or has_discovered + ): + d_parent_str = str(discovered_parent_id) + alias_map_disc = _alias_map_for_parent(eval_row, discovered_parent_id) + # See note above: cast once so the JSONB operators don't reject + # the column when it's typed as ``json`` in the database. + scores_jsonb = "(call_import_evaluation_rows.metric_scores)::jsonb" + if discovered_label_key and discovered_label_key.strip(): + target = _resolve_alias( + alias_map_disc, _slug_label(discovered_label_key) + ) + if target: + # Match rows whose discovered_labels list has an entry + # ``{"key": }`` OR whose sequence array still + # contains the slug. The latter covers older rows that + # were rewritten by a merge in the discovered_labels + # blob but whose sequence may have lagged. + contains_json = json.dumps( + {d_parent_str: {"discovered_labels": [{"key": target}]}} + ) + disc_filter_sql = text( + f""" + ( + {scores_jsonb} @> CAST(:contains AS JSONB) + OR + jsonb_exists( + COALESCE( + {scores_jsonb} -> :p_id -> 'sequence', + '[]'::jsonb + ), + :slug + ) + ) + """ + ).bindparams( + contains=contains_json, + p_id=d_parent_str, + slug=target, + ) + query = query.filter(disc_filter_sql) + elif has_discovered: + # No specific slug — just rows that surfaced any candidate + # under this parent. We coalesce missing paths to ``[]`` so + # ``jsonb_array_length`` always sees an array (it raises on + # non-array inputs, but our shape guarantees a list when + # the key is present). + has_disc_sql = text( + f""" + jsonb_array_length( + COALESCE( + {scores_jsonb} -> :p_id -> 'discovered_labels', + '[]'::jsonb + ) + ) > 0 + """ + ).bindparams(p_id=d_parent_str) + query = query.filter(has_disc_sql) + + # --- Sorting ---------------------------------------------------------- + # Column-click sorting from the UI. Falls back to ``row_index`` so + # paging stays stable when the user clears the sort. We always add a + # secondary ``row_index`` tiebreaker so duplicate sort keys (e.g. + # many rows with ``status = 'completed'``) keep a deterministic + # order across page boundaries — without this, pagination can + # double-show or skip rows when Postgres picks a different physical + # order on each query. + direction_desc = (sort_dir or "asc").strip().lower() == "desc" + + def _apply_direction(column_expr): + return column_expr.desc() if direction_desc else column_expr.asc() + + # Whether the caller's ``sort_by`` resolved to a known column. We + # use this flag to decide whether ``sort_dir`` is honoured on the + # fallback path: unrecognized columns (typos, stale UI state) fall + # back to the implicit ``row_index ASC`` default and intentionally + # ignore ``sort_dir`` so users don't get a surprise reverse order + # from a typo'd column name. + sort_recognized = False + sort_by_clean = (sort_by or "").strip() + primary_sort = None + metric_uuid: Optional[UUID] = None + if sort_by_clean == "row_index": + sort_recognized = True + # Falls through to the default ``order_by`` below with + # ``primary_sort`` still None — but ``sort_recognized=True`` + # tells the fallback branch to apply the requested direction. + elif sort_by_clean == "conversation_id": + sort_recognized = True + primary_sort = _apply_direction(CallImportRow.conversation_id) + elif sort_by_clean == "status": + sort_recognized = True + primary_sort = _apply_direction(CallImportEvaluationRow.status) + elif sort_by_clean.startswith("metric:"): + raw_metric_id = sort_by_clean.split(":", 1)[1].strip() + try: + metric_uuid = UUID(raw_metric_id) + except (TypeError, ValueError): + metric_uuid = None + if metric_uuid is not None: + sort_recognized = True + # ``metric_scores`` is JSON-typed but the helper functions + # for path extraction differ between Postgres (production) + # and SQLite (default test backend). Branch on the active + # dialect so we can use the right primitive: + # * Postgres → ``json_extract_path_text(col, key, "value")`` + # which returns the value as TEXT for both ``json`` and + # ``jsonb`` columns. + # * SQLite → ``json_extract(col, '$."".value')`` + # using JSONPath syntax. ``metric_uuid`` is already + # validated above (``UUID(raw_metric_id)``), so the + # interpolated path is safe from injection. + # NULL values (rows where the metric wasn't scored) sort + # to the END regardless of direction so un-scored rows + # don't crowd the top of an ascending sort. + dialect_name = ( + db.bind.dialect.name if db.bind is not None else "postgresql" + ) + if dialect_name == "sqlite": + json_path = f'$."{metric_uuid}".value' + path_value = func.json_extract( + CallImportEvaluationRow.metric_scores, + json_path, + ) + else: + path_value = func.json_extract_path_text( + CallImportEvaluationRow.metric_scores, + str(metric_uuid), + "value", + ) + primary_sort = ( + path_value.desc().nullslast() + if direction_desc + else path_value.asc().nullslast() + ) + + if primary_sort is not None: + query = query.order_by(primary_sort, CallImportRow.row_index.asc()) + elif sort_recognized: + # Explicit ``sort_by=row_index`` request — honour direction. + query = query.order_by(_apply_direction(CallImportRow.row_index)) + else: + # No sort requested OR unrecognized column — safe default of + # ``row_index ASC``. We deliberately ignore ``sort_dir`` here + # so a typo'd / stale ``sort_by`` doesn't quietly invert the + # default order. + query = query.order_by(CallImportRow.row_index.asc()) + from app.db_sharding.eval_rows import fetch_evaluation_row_pairs_page + from app.db_sharding.sessions import is_sharding_enabled + + def _pair_row_index( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> int: + return int(pair[1].row_index or 0) + + def _directed_string(value: Optional[str], desc: bool) -> Tuple[int, ...]: + text = value or "" + if not desc: + return (0, *text.encode("utf-8")) + return (1, *(-byte for byte in text.encode("utf-8"))) + + if sort_by_clean == "conversation_id": + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[Any, ...]: + return ( + _directed_string(pair[1].conversation_id, direction_desc), + _pair_row_index(pair), + ) + elif sort_by_clean == "status": + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[Any, ...]: + return ( + _directed_string(pair[0].status, direction_desc), + _pair_row_index(pair), + ) + elif sort_by_clean.startswith("metric:") and metric_uuid is not None: + metric_id_str = str(metric_uuid) + + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[Any, ...]: + scores = pair[0].metric_scores or {} + entry = scores.get(metric_id_str, {}) + raw_value = entry.get("value") if isinstance(entry, dict) else None + null_rank = 1 if raw_value is None else 0 + return ( + null_rank, + _directed_string( + str(raw_value) if raw_value is not None else None, + direction_desc, + ), + _pair_row_index(pair), + ) + elif sort_recognized and sort_by_clean == "row_index": + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[int, ...]: + idx = _pair_row_index(pair) + return (-idx,) if direction_desc else (idx,) + else: + def _pair_sort_key( + pair: Tuple[CallImportEvaluationRow, CallImportRow], + ) -> Tuple[int, ...]: + return (_pair_row_index(pair),) + + if is_sharding_enabled(): + def _build_query(session: Session): + return query.with_session(session) + + total, rows = fetch_evaluation_row_pairs_page( + db, + _build_query, + page=page, + page_size=page_size, + sort_key=_pair_sort_key, + bounded_shard_fetch=( + not sort_recognized or sort_by_clean == "row_index" + ), + ) + else: + total = query.count() + rows = query.offset((page - 1) * page_size).limit(page_size).all() + + # Row detail shows the transcript for this run's chosen source. + items: List[CallImportEvaluationRowResponse] = [ + _to_evaluation_row_response(eval_row_obj, source_row, eval_row) + for eval_row_obj, source_row in rows + ] + + return CallImportEvaluationRowListResponse( + items=items, + total=total, + page=page, + page_size=page_size, + ) + + +@router.get( + "/{eval_id}/export", + operation_id="exportCallImportEvaluationCsv", +) +async def export_call_import_evaluation_csv( + call_import_id: UUID, + eval_id: UUID, + format: Literal["csv", "xlsx"] = Query( + "csv", + description=( + "Output format. ``csv`` returns a UTF-8 BOM CSV; ``xlsx`` " + "returns a native Excel workbook (single sheet)." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> StreamingResponse: + del api_key + call_import = _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + selected_metric_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) + # Include parent metric ids referenced in selected_metric_groups so + # the export shows a parent "Chosen Label" column next to its + # children's true/false columns. + lookup_ids: List[UUID] = list(selected_metric_ids) + groups_raw = ( + evaluation.selected_metric_groups + if isinstance(evaluation.selected_metric_groups, dict) + else {} + ) + for parent_str in groups_raw.keys(): + try: + pid = UUID(parent_str) + if pid not in lookup_ids: + lookup_ids.append(pid) + except (TypeError, ValueError): + continue + metrics = _metrics_for_ids(db, organization_id, lookup_ids) + metric_names = {str(metric.id): metric.name for metric in metrics} + metrics_by_id = {str(metric.id): metric for metric in metrics} + + # Two export-time modes depending on how the batch was uploaded: + # + # * Schema-driven (new): ``call_imports.schema_id`` is set, + # ``parameter_mapping`` records which CSV header fed each + # parameter, and ``raw_columns`` on each row is keyed by + # parameter NAME. Export headers are the parameter names. + # * Legacy (pre-schema): ``column_mapping`` / ``extra_columns`` / + # ``custom_column_mapping`` drive the columns and + # ``raw_columns`` is keyed by the original CSV header. + # + # We bucket entries into ``standard_export_headers`` (raw_columns + # key == export header) and ``custom_export`` (export header + # differs from the raw_columns key) so the row-projection loop + # below stays mode-agnostic. + standard_export_headers: List[str] = [] + custom_export: List[tuple[str, str]] = [] # [(export_header, raw_columns_key)] + + if call_import.schema_id is not None: + # Use the live schema parameter list for column ordering. Falls + # back to whatever's in ``parameter_mapping`` if the schema was + # deleted (defensive - the FK is ON DELETE RESTRICT, but tests + # / future cascades may still hit this branch). + from app.models.database import CallImportSchema as _ImportSchema + + schema_obj = ( + db.query(_ImportSchema) + .filter(_ImportSchema.id == call_import.schema_id) + .first() + ) + if schema_obj is not None: + params_sorted = sorted( + schema_obj.parameters, key=lambda p: p.ordering or 0 + ) + for param in params_sorted: + if param.name and param.name not in standard_export_headers: + standard_export_headers.append(param.name) + else: + for param_name in (call_import.parameter_mapping or {}).keys(): + if param_name and param_name not in standard_export_headers: + standard_export_headers.append(param_name) + else: + mapping = call_import.column_mapping or {} + mapped_headers = [ + mapping.get("external_call_id"), + mapping.get("transcript"), + mapping.get("recording_url"), + ] + for header in [*mapped_headers, *(call_import.extra_columns or [])]: + if ( + isinstance(header, str) + and header + and header not in standard_export_headers + ): + standard_export_headers.append(header) + + custom_mapping = call_import.custom_column_mapping or {} + if isinstance(custom_mapping, dict): + for name, csv_header in custom_mapping.items(): + if not isinstance(name, str) or not isinstance(csv_header, str): + continue + if not name or not csv_header: + continue + if name in standard_export_headers: + continue # would clobber a real column + custom_export.append((name, csv_header)) + + if ( + call_import.source_format == "audio" + and "conversation_id" not in standard_export_headers + ): + standard_export_headers.insert(0, "conversation_id") + + # Build the metric columns: each parent (if any) gets a value column + # and (when capture_rationale=true) a " - LLM Rationale" + # column. The per-child boolean columns are intentionally suppressed + # — categorization metrics now collapse to exactly two columns in + # the export, mirroring the in-app table. + child_ids_in_groups: set[str] = set() + for parent_str, child_strs in groups_raw.items(): + for child_str in child_strs: + if isinstance(child_str, str): + child_ids_in_groups.add(child_str) + + metric_headers: List[str] = [] + rationale_headers: Dict[str, str] = {} # metric_id_str -> rationale column name + seen_metric_ids: set[str] = set() + + def _add_metric_column(metric: Metric) -> None: + mid_str = str(metric.id) + if mid_str in seen_metric_ids: + return + # Skip any child whose parent is part of this run — the parent + # column above already shows the chosen child name as its + # value. + if mid_str in child_ids_in_groups: + return + seen_metric_ids.add(mid_str) + header = metric_names[mid_str] + metric_headers.append(header) + if bool(getattr(metric, "capture_rationale", False)): + rationale_header = f"{header} - LLM Rationale" + metric_headers.append(rationale_header) + rationale_headers[mid_str] = rationale_header + + for parent_str in groups_raw.keys(): + parent = metrics_by_id.get(parent_str) + if parent: + _add_metric_column(parent) + # Children of an in-run parent are deliberately not emitted — + # the ``child_ids_in_groups`` guard inside ``_add_metric_column`` + # is what enforces this. We still iterate the keys above (not + # ``.items()``) so the parent-only emission is explicit. + # Append anything left over (standalone metrics not in any group, or + # legacy runs without ``selected_metric_groups``). + for metric in metrics: + if metric.selection_mode and not metric.parent_metric_id: + continue # already handled above + if str(metric.id) in seen_metric_ids: + continue + _add_metric_column(metric) + + # Three new fixed columns surface the two transcript fields and the + # evaluation's transcript_source as live values pulled from the + # ``CallImportRow`` (not from the frozen ``raw_columns`` snapshot). + # The user can now compare "what was in the CSV" vs "what the + # diarisation worker produced" without round-tripping through the + # UI, and downstream tools can verify which transcript the metrics + # were computed against. + PRODUCTION_TRANSCRIPT_HEADER = "Production Transcript" + DIARISED_TRANSCRIPT_HEADER = "Diarised Transcript" + EVAL_SOURCE_HEADER = "Evaluated Transcript Source" + + fieldnames = [ + *standard_export_headers, + *[h for h, _ in custom_export], + PRODUCTION_TRANSCRIPT_HEADER, + DIARISED_TRANSCRIPT_HEADER, + EVAL_SOURCE_HEADER, + *metric_headers, + ] + + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + rows = sorted( + load_evaluation_row_pairs(db, eval_id), + key=lambda pair: int(pair[1].row_index or 0), + ) + else: + rows = ( + db.query(CallImportEvaluationRow, CallImportRow) + .join( + CallImportRow, + CallImportRow.id == CallImportEvaluationRow.call_import_row_id, + ) + .filter(CallImportEvaluationRow.evaluation_id == eval_id) + .order_by(CallImportRow.row_index.asc()) + .all() + ) + + def _project_rows() -> Iterator[Dict[str, str]]: + for eval_row, source_row in rows: + row_out: Dict[str, str] = {} + raw = ( + source_row.raw_columns + if isinstance(source_row.raw_columns, dict) + else {} + ) + for header in standard_export_headers: + value = raw.get(header) + if value is None and header == "conversation_id": + value = source_row.conversation_id + row_out[header] = "" if value is None else str(value) + for export_header, csv_header in custom_export: + value = raw.get(csv_header) + row_out[export_header] = "" if value is None else str(value) + + # Live transcripts pulled from the row, NOT from raw_columns, + # so re-diarised values are always reflected in the export. + # Both transcript columns are flattened to a single line so the + # spreadsheet cell doesn't balloon vertically — the in-app + # ``TranscriptView`` still has the DB copy with line breaks + # intact for chat-bubble rendering. + row_out[PRODUCTION_TRANSCRIPT_HEADER] = _flatten_transcript( + source_row.transcript + ) + row_out[DIARISED_TRANSCRIPT_HEADER] = _flatten_transcript( + source_row.diarised_transcript + ) + row_out[EVAL_SOURCE_HEADER] = _evaluated_transcript_source_label( + evaluation, + source_row, + ) + + scores = ( + eval_row.metric_scores + if isinstance(eval_row.metric_scores, dict) + else {} + ) + for metric in metrics: + metric_score = ( + scores.get(str(metric.id)) + if isinstance(scores, dict) + else None + ) + value = ( + metric_score.get("value") + if isinstance(metric_score, dict) + else None + ) + # Parent metrics (selection_mode set) render the chosen + # child name for single_choice or the ";"-joined list of + # true child names for multi_label. + if ( + metric.selection_mode + and not metric.parent_metric_id + and isinstance(metric_score, dict) + ): + if metric.selection_mode == "multi_label": + selected = metric_score.get("selected_child_names") + if isinstance(selected, list): + value = ";".join(str(s) for s in selected) + else: + value = ( + metric_score.get("chosen_child_name") + or metric_score.get("value") + ) + row_out[metric.name] = "" if value is None else str(value) + rationale_header = rationale_headers.get(str(metric.id)) + if rationale_header is not None: + rationale = ( + metric_score.get("rationale") + if isinstance(metric_score, dict) + else None + ) + row_out[rationale_header] = ( + "" if rationale is None else str(rationale) + ) + yield row_out + + base_filename = f"call-import-{call_import_id}-evaluation-{eval_id}" + + if format == "xlsx": + # xlsx is unicode-native (Hindi/Devanagari, emoji, etc.) so the + # UTF-8-BOM dance isn't needed here. ``write_only`` mode keeps + # peak memory bounded for large evaluations because openpyxl + # only buffers the current row. + try: + from openpyxl import Workbook # type: ignore + from openpyxl.cell import WriteOnlyCell # type: ignore + from openpyxl.styles import Font # type: ignore + except ImportError as exc: # pragma: no cover - exercised by pyproject lock + raise HTTPException( + status_code=500, + detail=( + "Excel export requires the 'openpyxl' package which is " + "not installed." + ), + ) from exc + + workbook = Workbook(write_only=True) + worksheet = workbook.create_sheet(title="Evaluation") + + bold_font = Font(bold=True) + header_cells = [] + for header in fieldnames: + cell = WriteOnlyCell(worksheet, value=header) + cell.font = bold_font + header_cells.append(cell) + worksheet.append(header_cells) + + for row_dict in _project_rows(): + worksheet.append([row_dict.get(h, "") for h in fieldnames]) + + buffer = io.BytesIO() + workbook.save(buffer) + xlsx_bytes = buffer.getvalue() + filename = f"{base_filename}.xlsx" + return StreamingResponse( + iter([xlsx_bytes]), + media_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + output = io.StringIO() + writer = csv.DictWriter(output, fieldnames=fieldnames, extrasaction="ignore") + writer.writeheader() + for row_dict in _project_rows(): + writer.writerow(row_dict) + + # Excel on Windows defaults to the system ANSI codepage (Windows-1252) + # when a CSV has no encoding marker, which turns UTF-8 Hindi/Devanagari + # / any non-ASCII text into mojibake (e.g. ``ठीक`` → ``ठीक``). + # A UTF-8 BOM tells Excel to switch to UTF-8 decoding and is silently + # skipped by every other UTF-8-aware reader (pandas, LibreOffice, + # Google Sheets, etc.), so the data round-trips correctly everywhere. + csv_text = output.getvalue() + # ``utf-8-sig`` adds the UTF-8 BOM so Excel on Windows decodes the file + # as UTF-8 instead of the system codepage. We also declare the same + # codec in the Content-Type header so well-behaved HTTP clients (incl. + # ``httpx`` / ``requests`` in our tests) strip the BOM during decode. + csv_bytes = csv_text.encode("utf-8-sig") + filename = f"{base_filename}.csv" + return StreamingResponse( + iter([csv_bytes]), + media_type="text/csv; charset=utf-8-sig", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + +def _report_filename_slug(value: str) -> str: + slug = re.sub(r"[^a-zA-Z0-9]+", "-", value.strip().lower()).strip("-") + return slug or "client" + + +def _report_branding_for_import_workspace( + db: Session, + organization_id: UUID, + workspace_id: UUID, + *, + internal_brand_image_id: Optional[str] = None, + external_brand_image_id: Optional[str] = None, +) -> tuple[dict[str, str] | list[str], Optional[str]]: + workspace = ( + db.query(Workspace) + .filter( + Workspace.id == workspace_id, + Workspace.organization_id == organization_id, + ) + .first() + ) + raw = workspace.report_branding if workspace and isinstance(workspace.report_branding, dict) else {} + images = raw.get("images") if isinstance(raw.get("images"), list) else [] + loaded_images: list[dict[str, str]] = [] + for item in images: + if not isinstance(item, dict) or not item.get("s3_key"): + continue + content_type = str(item.get("content_type") or "image/png") + try: + from app.services.storage.s3_service import s3_service + + image_bytes = s3_service.download_file_by_key(str(item["s3_key"])) + except Exception as exc: # noqa: BLE001 + logger.warning( + "Unable to load report branding image for workspace {}: {}", + workspace_id, + exc, + ) + continue + encoded = base64.b64encode(image_bytes).decode("ascii") + role = str(item.get("role") or "generic") + if role not in {"internal", "external", "generic"}: + role = "generic" + loaded_images.append( + { + "id": str(item.get("id") or ""), + "role": role, + "data_uri": f"data:{content_type};base64,{encoded}", + } + ) + + def _pick(role: str, selected_id: Optional[str]) -> Optional[str]: + if selected_id: + for loaded in loaded_images: + if loaded["id"] == selected_id: + return loaded["data_uri"] + for loaded in loaded_images: + if loaded["role"] == role: + return loaded["data_uri"] + return None + + logo_data_uris: dict[str, str] = {} + internal_uri = _pick("internal", internal_brand_image_id) + external_uri = _pick("external", external_brand_image_id) + if internal_uri: + logo_data_uris["internal"] = internal_uri + if external_uri: + logo_data_uris["external"] = external_uri + if ( + not logo_data_uris + and not internal_brand_image_id + and not external_brand_image_id + ): + # Backward compatibility for workspaces that only had a generic logo + # library before the two-slot report header existed. + generic_uris = [ + loaded["data_uri"] + for loaded in loaded_images + if loaded.get("data_uri") + ] + if generic_uris: + heading = raw.get("heading") if isinstance(raw.get("heading"), str) else None + return generic_uris[:4], heading + heading = raw.get("heading") if isinstance(raw.get("heading"), str) else None + return logo_data_uris, heading + + +def _display_metrics_for_pdf_report( + db: Session, + organization_id: UUID, + evaluation: CallImportEvaluation, +) -> list[Metric]: + selected_metric_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) + lookup_ids: List[UUID] = list(selected_metric_ids) + groups_raw = ( + evaluation.selected_metric_groups + if isinstance(evaluation.selected_metric_groups, dict) + else {} + ) + for parent_str in groups_raw.keys(): + try: + parent_id = UUID(parent_str) + except (TypeError, ValueError): + continue + if parent_id not in lookup_ids: + lookup_ids.append(parent_id) + + metrics = _metrics_for_ids(db, organization_id, lookup_ids) + child_ids_in_groups: set[str] = set() + for child_strs in groups_raw.values(): + if not isinstance(child_strs, list): + continue + child_ids_in_groups.update(str(child_id) for child_id in child_strs) + + metrics_by_id = {str(metric.id): metric for metric in metrics} + display: list[Metric] = [] + seen: set[str] = set() + + for parent_str in groups_raw.keys(): + parent = metrics_by_id.get(str(parent_str)) + if parent and str(parent.id) not in seen: + display.append(parent) + seen.add(str(parent.id)) + + for metric in metrics: + metric_id = str(metric.id) + if metric_id in seen or metric_id in child_ids_in_groups: + continue + if metric.selection_mode and not metric.parent_metric_id: + continue + display.append(metric) + seen.add(metric_id) + + return display + + +def _metrics_for_clustering( + db: Session, + evaluation: CallImportEvaluation, + eval_rows: List[CallImportEvaluationRow], +) -> List[Metric]: + """All enabled quality metrics scored in this run, normalized for clustering. + + Hierarchical children are collapsed to their parent metric so cluster + groups render at the category level (e.g. ``AI reveal``) instead of the + child label level (e.g. ``Yes`` / ``No``). + """ + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + aggregate_metric_ids: List[UUID] = [] + for agg in aggregates: + if (agg.metric_category or "quality") == "user_insight": + continue + try: + aggregate_metric_ids.append(UUID(agg.metric_id)) + except (TypeError, ValueError): + continue + if not aggregate_metric_ids: + return [] + + aggregate_metrics = _metrics_for_ids( + db, evaluation.organization_id, aggregate_metric_ids + ) + by_id = {metric.id: metric for metric in aggregate_metrics} + + normalized_ids: List[UUID] = [] + seen: set[UUID] = set() + for metric_id in aggregate_metric_ids: + metric = by_id.get(metric_id) + target_id = ( + metric.parent_metric_id + if metric is not None and metric.parent_metric_id + else metric_id + ) + if target_id in seen: + continue + seen.add(target_id) + normalized_ids.append(target_id) + + metrics = _metrics_for_ids(db, evaluation.organization_id, normalized_ids) + return [ + metric + for metric in metrics + if getattr(metric, "enabled", True) and not _metric_is_user_insight(metric) + ] + + +def _metric_is_user_insight(metric: Metric) -> bool: + if (getattr(metric, "metric_category", "quality") or "quality") == "user_insight": + return True + text_value = " ".join( + str(part or "").lower() + for part in (getattr(metric, "name", ""), getattr(metric, "description", "")) + ) + normalized = text_value.replace("-", " ").replace("_", " ") + phrases = ( + "call context", + "caller context", + "product identification", + "out of scope", + "identity match", + "user identity", + "caller identity", + "frustration trigger", + "video call offer", + "video call reception", + ) + return any(phrase in normalized for phrase in phrases) + + +def _evaluation_rows_for_period( + db: Session, + evaluation_id: UUID, +) -> list[tuple[CallImportEvaluationRow, CallImportRow]]: + return ( + db.query(CallImportEvaluationRow, CallImportRow) + .join(CallImportRow, CallImportRow.id == CallImportEvaluationRow.call_import_row_id) + .filter(CallImportEvaluationRow.evaluation_id == evaluation_id) + .order_by(CallImportRow.row_index.asc()) + .all() + ) + + +def _baseline_candidate_evaluations( + db: Session, + organization_id: UUID, + workspace_id: UUID, + current_evaluation: CallImportEvaluation, + current_period_start: Optional[date], + *, + limit: int = 20, +) -> list[dict[str, Any]]: + candidates = ( + db.query(CallImportEvaluation, CallImport) + .join(CallImport, CallImport.id == CallImportEvaluation.call_import_id) + .filter( + CallImportEvaluation.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + CallImportEvaluation.id != current_evaluation.id, + CallImportEvaluation.status == "completed", + CallImportEvaluation.completed_rows > 0, + ) + .order_by(desc(CallImportEvaluation.created_at)) + .limit(limit * 3) + .all() + ) + items: list[dict[str, Any]] = [] + for candidate_eval, candidate_import in candidates: + rows = _evaluation_rows_for_period(db, candidate_eval.id) + period_start, period_end, period_label, period_display = _report_period_from_rows(rows) + if current_period_start and period_start and period_start >= current_period_start: + continue + dataset = ( + (candidate_import.dataset or "").strip() + or (candidate_import.original_filename or candidate_import.filename or "").strip() + or "Unknown dataset" + ) + evaluation_name = ( + (candidate_eval.name or "").strip() + or str(candidate_eval.id)[:8] + ) + items.append( + { + "evaluation_id": str(candidate_eval.id), + "name": evaluation_name, + "dataset": dataset, + "period_label": period_label, + "period_start": period_start, + "period_end": period_end, + "period_display": period_display, + "completed_rows": int(candidate_eval.completed_rows or 0), + "created_at": candidate_eval.created_at, + "is_default": False, + } + ) + if len(items) >= limit: + break + items.sort( + key=lambda item: ( + item["period_start"] or date.min, + item["created_at"] or datetime.min.replace(tzinfo=timezone.utc), + ), + reverse=True, + ) + if items: + items[0]["is_default"] = True + return items + + +def _resolve_baseline_evaluation( + db: Session, + organization_id: UUID, + workspace_id: UUID, + current_evaluation: CallImportEvaluation, + current_period_start: Optional[date], + baseline_evaluation_id: Optional[str], +) -> Optional[CallImportEvaluation]: + candidates = _baseline_candidate_evaluations( + db, + organization_id, + workspace_id, + current_evaluation, + current_period_start, + ) + allowed_ids = {item["evaluation_id"] for item in candidates} + if baseline_evaluation_id: + baseline_id = str(baseline_evaluation_id).strip() + if baseline_id not in allowed_ids: + raise HTTPException( + status_code=400, + detail="Selected baseline evaluation is not a valid prior run for this report.", + ) + return ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == UUID(baseline_id), + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not candidates: + return None + default_id = candidates[0]["evaluation_id"] + return ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == UUID(default_id), + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + + +def _benchmark_context_for_evaluation( + db: Session, + baseline_evaluation: Optional[CallImportEvaluation], +) -> Optional[dict[str, str]]: + if baseline_evaluation is None: + return None + baseline_import = ( + db.query(CallImport) + .filter(CallImport.id == baseline_evaluation.call_import_id) + .first() + ) + rows = _evaluation_rows_for_period(db, baseline_evaluation.id) + period_start, _period_end, period_label, _period_display = _report_period_from_rows(rows) + dataset = ( + (baseline_import.dataset or "").strip() + if baseline_import and baseline_import.dataset + else None + ) + filename = ( + (baseline_import.original_filename or baseline_import.filename or "").strip() + if baseline_import + else None + ) + evaluation_label = ( + (baseline_evaluation.name or "").strip() + if baseline_evaluation.name + else str(baseline_evaluation.id)[:8] + ) + period = period_label or ( + period_start.isoformat() if period_start else "previous report" + ) + return { + "dataset": dataset or filename or "Unknown dataset", + "evaluation": evaluation_label, + "evaluation_id": str(baseline_evaluation.id), + "period": period, + } + + +def _period_deltas_from_evaluation( + db: Session, + baseline_evaluation: CallImportEvaluation, + current_metric_aggregates: list[dict[str, Any]], + current_evaluation: CallImportEvaluation, + current_eval_rows: List[CallImportEvaluationRow], +) -> dict[str, dict[str, str]]: + baseline_rows = _evaluation_rows_for_period(db, baseline_evaluation.id) + baseline_eval_rows = [eval_row for eval_row, _source_row in baseline_rows] + baseline_aggregate_models = _compute_metric_aggregates( + db, + baseline_evaluation, + baseline_eval_rows, + ) + baseline_metric_aggregates = [ + _aggregate_to_dict(aggregate) for aggregate in baseline_aggregate_models + ] + _metrics, _aggs, policies, _source, _child_map = _clustering_context( + db, current_evaluation, current_eval_rows + ) + metric_by_id = {str(m.id): m for m in _metrics} + current_by_id = { + str(item.get("metric_id")): item for item in current_metric_aggregates + } + previous_by_id = { + str(item.get("metric_id")): item + for item in baseline_metric_aggregates + if isinstance(item, dict) + } + deltas: dict[str, dict[str, str]] = {} + for metric_id, current in current_by_id.items(): + metric = metric_by_id.get(metric_id) + policy = policies.get(metric_id) + previous_raw = previous_by_id.get(metric_id) + if metric is None or policy is None: + deltas[metric_id] = { + "label": "No previous-week baseline", + "detail": "No comparable prior report snapshot was found.", + } + continue + current_pct = failure_rate_percent_from_rows( + current_eval_rows, metric, policy + ) + previous_pct = failure_rate_percent_from_rows( + baseline_eval_rows, metric, policy + ) + if current_pct is None or previous_pct is None: + current_pct = current_pct or _aggregate_primary_percent(current, policy) + previous_pct = ( + previous_pct or _aggregate_primary_percent(previous_raw, policy) + if previous_raw + else None + ) + if current_pct is None or previous_pct is None: + deltas[metric_id] = { + "label": "No previous-week baseline", + "detail": "No comparable prior report snapshot was found.", + } + continue + delta = current_pct - previous_pct + sign = "+" if delta >= 0 else "" + deltas[metric_id] = { + "label": f"{sign}{delta:.1f} pp", + "detail": ( + f"Current report {current_pct:.1f}% vs previous report " + f"{previous_pct:.1f}%" + ), + } + return deltas + + +_DELTA_EXPLANATION_SYSTEM_PROMPT = ( + "You are a senior conversation-analytics reviewer. You will receive " + "week-over-week metric failure-rate deltas plus reconciled failure " + "cluster context per metric.\n\n" + "Return STRICT JSON only:\n" + "{\n" + ' "explanations": {"": "<1-2 sentence explanation of why the delta likely occurred>"}\n' + "}\n\n" + "Constraints:\n" + "- Only include metrics supplied in the prompt.\n" + "- Cluster labels are generated independently each run and are NOT stable " + "IDs. Never compare an unmatched current label to 0% baseline.\n" + "- Use matched_theme_shifts for label-aligned comparisons, " + "gap_label_shifts for structural shifts, and new_themes_current_period " + "for themes that emerged without a baseline match.\n" + "- If reconciliation is uncertain, explain using the numeric delta and " + "gap_label_shifts only.\n" + "- Keep each explanation to 1-2 short sentences (~220 chars).\n" + "- Vendor-safe, factual language; no markdown." +) + + +def _period_delta_explanation_cache_key( + baseline_evaluation_id: UUID, + *, + completed_rows: int, + baseline_completed_rows: int, +) -> str: + return ( + f"{baseline_evaluation_id}:{completed_rows}:" + f"{baseline_completed_rows}:reconciled-v2" + ) + + +def _normalize_cluster_label(label: str) -> str: + return re.sub(r"[^a-z0-9]+", " ", (label or "").lower()).strip() + + +_CLUSTER_LABEL_STOPWORDS = frozenset( + { + "a", + "an", + "the", + "and", + "or", + "during", + "while", + "with", + "for", + "from", + "into", + "general", + "user", + "bot", + "agent", + } +) + + +def _cluster_label_tokens(label: str) -> set[str]: + return { + token + for token in _normalize_cluster_label(label).split() + if token and token not in _CLUSTER_LABEL_STOPWORDS and len(token) > 2 + } + + +def _cluster_label_similarity(left: str, right: str) -> float: + tokens_left = _cluster_label_tokens(left) + tokens_right = _cluster_label_tokens(right) + if not tokens_left or not tokens_right: + return 0.0 + intersection = tokens_left & tokens_right + if not intersection: + return 0.0 + union = tokens_left | tokens_right + jaccard = len(intersection) / len(union) + smaller = tokens_left if len(tokens_left) <= len(tokens_right) else tokens_right + overlap_ratio = len(intersection) / len(smaller) + return max(jaccard, overlap_ratio * 0.85) + + +def _group_clusters_by_gap_label( + clusters: list[dict[str, Any]], +) -> dict[str, list[dict[str, Any]]]: + grouped: dict[str, list[dict[str, Any]]] = {} + for cluster in clusters: + gap_label = str(cluster.get("gap_label") or "UNKNOWN") + grouped.setdefault(gap_label, []).append(cluster) + return grouped + + +def _append_matched_cluster_pair( + matched: list[dict[str, Any]], + current: dict[str, Any], + baseline: dict[str, Any], + *, + match_confidence: float, + match_method: str, +) -> None: + matched.append( + { + "current_label": current.get("label"), + "baseline_label": baseline.get("label"), + "gap_label": current.get("gap_label") or baseline.get("gap_label"), + "current_share_pct": current.get("share_pct"), + "baseline_share_pct": baseline.get("share_pct"), + "share_delta_pp": round( + float(current.get("share_pct") or 0.0) + - float(baseline.get("share_pct") or 0.0), + 1, + ), + "match_confidence": round(match_confidence, 2), + "match_method": match_method, + } + ) + + +def _aggregate_share_by_gap_label( + clusters: list[dict[str, Any]], +) -> dict[str, float]: + totals: dict[str, float] = {} + for cluster in clusters: + gap_label = str(cluster.get("gap_label") or "UNKNOWN") + totals[gap_label] = totals.get(gap_label, 0.0) + float( + cluster.get("share_pct") or 0.0 + ) + return {gap: round(share, 1) for gap, share in totals.items()} + + +def _reconcile_cluster_periods( + current_clusters: list[dict[str, Any]], + baseline_clusters: list[dict[str, Any]], + *, + similarity_threshold: float = 0.35, +) -> dict[str, Any]: + """Align independently-generated cluster labels before delta explanation.""" + matched: list[dict[str, Any]] = [] + current_unmatched = list(current_clusters) + remaining_baseline = list(baseline_clusters) + + current_by_gap = _group_clusters_by_gap_label(current_unmatched) + baseline_by_gap = _group_clusters_by_gap_label(remaining_baseline) + for gap_label in list(current_by_gap): + current_group = current_by_gap.get(gap_label) or [] + baseline_group = baseline_by_gap.get(gap_label) or [] + if len(current_group) != 1 or len(baseline_group) != 1: + continue + current = current_group[0] + baseline = baseline_group[0] + _append_matched_cluster_pair( + matched, + current, + baseline, + match_confidence=0.75, + match_method="single_cluster_per_gap_label", + ) + current_unmatched.remove(current) + remaining_baseline.remove(baseline) + current_by_gap[gap_label] = [] + baseline_by_gap[gap_label] = [] + + for current in list(current_unmatched): + best_idx: Optional[int] = None + best_score = 0.0 + for idx, baseline in enumerate(remaining_baseline): + score = _cluster_label_similarity( + str(current.get("label") or ""), + str(baseline.get("label") or ""), + ) + if current.get("gap_label") == baseline.get("gap_label"): + score += 0.1 + if score > best_score: + best_score = score + best_idx = idx + + if best_idx is not None and best_score >= similarity_threshold: + baseline = remaining_baseline.pop(best_idx) + _append_matched_cluster_pair( + matched, + current, + baseline, + match_confidence=best_score, + match_method="label_similarity", + ) + + matched_current_labels = { + str(item.get("current_label") or "") for item in matched + } + matched_baseline_labels = { + str(item.get("baseline_label") or "") for item in matched + } + current_unmatched = [ + cluster + for cluster in current_clusters + if str(cluster.get("label") or "") not in matched_current_labels + ] + remaining_baseline = [ + cluster + for cluster in baseline_clusters + if str(cluster.get("label") or "") not in matched_baseline_labels + ] + + new_themes = [ + { + "label": cluster.get("label"), + "gap_label": cluster.get("gap_label"), + "share_pct": cluster.get("share_pct"), + "note": "New theme in current period (no close baseline match).", + } + for cluster in current_unmatched + ] + + retired_themes = [ + { + "label": baseline.get("label"), + "gap_label": baseline.get("gap_label"), + "share_pct": baseline.get("share_pct"), + "note": "Theme present in baseline only (retired or renamed).", + } + for baseline in remaining_baseline + ] + + current_gap = _aggregate_share_by_gap_label(current_clusters) + baseline_gap = _aggregate_share_by_gap_label(baseline_clusters) + gap_label_shifts: dict[str, dict[str, float]] = {} + for gap_label in set(current_gap) | set(baseline_gap): + current_share = current_gap.get(gap_label, 0.0) + baseline_share = baseline_gap.get(gap_label, 0.0) + if abs(current_share - baseline_share) >= 0.5: + gap_label_shifts[gap_label] = { + "current_share_pct": current_share, + "baseline_share_pct": baseline_share, + "share_delta_pp": round(current_share - baseline_share, 1), + } + + return { + "matched_theme_shifts": matched, + "new_themes_current_period": new_themes, + "retired_themes_baseline_period": retired_themes, + "gap_label_shifts": gap_label_shifts, + "reconciliation_note": ( + "Cluster labels are generated independently each run and may " + "rename the same failure mode. Do not treat unmatched current " + "labels as 0% in the baseline period." + ), + } + + +def _load_period_delta_explanations_cache( + evaluation: CallImportEvaluation, + cache_key: str, +) -> Optional[dict[str, str]]: + raw = getattr(evaluation, "period_delta_explanations", None) + if not isinstance(raw, dict): + return None + entry = raw.get(cache_key) + if not isinstance(entry, dict): + return None + explanations_raw = entry.get("explanations") + if not isinstance(explanations_raw, dict): + return None + return { + str(metric_id): str(why).strip() + for metric_id, why in explanations_raw.items() + if str(metric_id).strip() and isinstance(why, str) and why.strip() + } + + +def _save_period_delta_explanations_cache( + db: Session, + evaluation: CallImportEvaluation, + cache_key: str, + explanations: dict[str, str], +) -> None: + raw = evaluation.period_delta_explanations + if not isinstance(raw, dict): + raw = {} + updated = dict(raw) + updated[cache_key] = { + "explanations": explanations, + "generated_at": datetime.now(timezone.utc).isoformat(), + } + evaluation.period_delta_explanations = updated + flag_modified(evaluation, "period_delta_explanations") + db.commit() + + +def _cluster_summary_for_metric( + state: Optional[EvaluationMetricClustersState], + metric_id: str, +) -> list[dict[str, Any]]: + if state is None or state.status != "completed": + return [] + for group in state.groups: + if str(group.metric_id) != metric_id: + continue + return [ + { + "label": cluster.label, + "gap_label": cluster.gap_label, + "share_pct": round(cluster.share_pct, 1), + "count": cluster.count, + } + for cluster in group.clusters[:5] + ] + return [] + + +def _merge_delta_why( + raw_deltas: dict[str, dict[str, str]], + explanations: dict[str, str], +) -> dict[str, dict[str, str]]: + if not explanations: + return raw_deltas + merged: dict[str, dict[str, str]] = {} + for metric_id, delta in raw_deltas.items(): + updated = dict(delta) + why = explanations.get(metric_id) + if why: + updated["why"] = why + merged[metric_id] = updated + return merged + + +def _explain_period_deltas( + db: Session, + organization_id: UUID, + evaluation: CallImportEvaluation, + baseline_evaluation: CallImportEvaluation, + raw_deltas: dict[str, dict[str, str]], + *, + min_delta_pp: float = 0.5, +) -> dict[str, dict[str, str]]: + """Attach ``why`` explanations to period deltas using cached LLM output.""" + if not raw_deltas: + return raw_deltas + + cache_key = _period_delta_explanation_cache_key( + baseline_evaluation.id, + completed_rows=evaluation.completed_rows, + baseline_completed_rows=baseline_evaluation.completed_rows, + ) + cached = _load_period_delta_explanations_cache(evaluation, cache_key) + if cached is not None: + return _merge_delta_why(raw_deltas, cached) + + current_clusters = _metric_clusters_payload(evaluation) + baseline_clusters = _metric_clusters_payload(baseline_evaluation) + metrics_for_prompt: list[dict[str, Any]] = [] + for metric_id, delta in raw_deltas.items(): + label = delta.get("label") or "" + if "No previous-week baseline" in label: + continue + match = re.search(r"([+-]?\d+(?:\.\d+)?)\s*pp", label) + if match and abs(float(match.group(1))) < min_delta_pp: + continue + current_summary = _cluster_summary_for_metric(current_clusters, metric_id) + baseline_summary = _cluster_summary_for_metric(baseline_clusters, metric_id) + if not current_summary and not baseline_summary: + continue + cluster_reconciliation = _reconcile_cluster_periods( + current_summary, + baseline_summary, + ) + metrics_for_prompt.append( + { + "metric_id": metric_id, + "delta_label": label, + "delta_detail": delta.get("detail") or "", + "cluster_reconciliation": cluster_reconciliation, + } + ) + + if not metrics_for_prompt: + return raw_deltas + + provider_hint: Optional[str] = None + model_hint: Optional[str] = None + tldr_raw = evaluation.tldr_summary + if isinstance(tldr_raw, dict): + if isinstance(tldr_raw.get("provider"), str): + provider_hint = tldr_raw["provider"] + if isinstance(tldr_raw.get("model"), str): + model_hint = tldr_raw["model"] + + from app.services.ai.llm_resolver import get_llm_provider_and_model + from app.services.call_import_user_insights import _call_llm, _parse_json_object + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, provider_hint, model_hint + ) + try: + text = _call_llm( + db, + organization_id, + provider_enum, + model_str, + [ + {"role": "system", "content": _DELTA_EXPLANATION_SYSTEM_PROMPT}, + { + "role": "user", + "content": json.dumps( + {"metrics": metrics_for_prompt}, + ensure_ascii=False, + default=str, + ), + }, + ], + temperature=0.3, + max_tokens=900, + ) + except Exception as exc: + logger.warning("[PeriodDeltaExplain] LLM call failed: {}", exc) + return raw_deltas + + parsed = _parse_json_object(text) + explanations_raw = parsed.get("explanations") + explanations: dict[str, str] = {} + if isinstance(explanations_raw, dict): + for metric_id, why in explanations_raw.items(): + if isinstance(why, str) and why.strip(): + explanations[str(metric_id)] = why.strip() + + if explanations: + _save_period_delta_explanations_cache( + db, evaluation, cache_key, explanations + ) + return _merge_delta_why(raw_deltas, explanations) + + +def _period_deltas_with_explanations( + db: Session, + organization_id: UUID, + evaluation: CallImportEvaluation, + baseline_evaluation: CallImportEvaluation, + raw_deltas: dict[str, dict[str, str]], +) -> dict[str, dict[str, str]]: + return _explain_period_deltas( + db, + organization_id, + evaluation, + baseline_evaluation, + raw_deltas, + ) + + +def _benchmark_context_for_snapshot( + db: Session, + previous_snapshot: Optional[CallImportEvaluationReportSnapshot], +) -> Optional[dict[str, str]]: + if previous_snapshot is None: + return None + previous_import = ( + db.query(CallImport) + .filter(CallImport.id == previous_snapshot.call_import_id) + .first() + ) + previous_eval = ( + db.query(CallImportEvaluation) + .filter(CallImportEvaluation.id == previous_snapshot.evaluation_id) + .first() + ) + dataset = ( + (previous_import.dataset or "").strip() + if previous_import and previous_import.dataset + else None + ) + filename = ( + (previous_import.original_filename or previous_import.filename or "").strip() + if previous_import + else None + ) + evaluation_label = ( + (previous_eval.name or "").strip() + if previous_eval and previous_eval.name + else str(previous_snapshot.evaluation_id)[:8] + ) + period = previous_snapshot.period_label or ( + previous_snapshot.period_start.isoformat() + if previous_snapshot.period_start + else "previous report" + ) + return { + "dataset": dataset or filename or "Unknown dataset", + "evaluation": evaluation_label, + "evaluation_id": str(previous_snapshot.evaluation_id), + "period": period, + } + + +def _clamp_prose_to_sentences( + text: str, + *, + max_sentences: int = 3, + max_chars: int = 300, +) -> str: + """Keep concise audit/TLDR prose within sentence and character limits.""" + cleaned = (text or "").strip() + if not cleaned: + return cleaned + cleaned = re.sub(r"\s*\n+\s*", " ", cleaned).strip() + sentences = [ + sentence.strip() + for sentence in re.split(r"(?<=[.!?])\s+", cleaned) + if sentence.strip() + ] + if sentences: + result = " ".join(sentences[:max_sentences]).strip() + else: + result = cleaned + if len(result) > max_chars: + trimmed = result[: max_chars - 3].rsplit(" ", 1)[0].rstrip(".,;:") + result = f"{trimmed}..." if trimmed else result[:max_chars] + return result + + +def _audit_summary_text_from_tldr( + summary: Optional[EvaluationTldrSummary], +) -> Optional[str]: + if summary is None: + return None + narrative = _clamp_prose_to_sentences(summary.narrative.strip()) + return narrative or None + + +def _metric_insights_from_tldr( + summary: Optional[EvaluationTldrSummary], +) -> dict[str, str]: + if summary is None: + return {} + return { + str(metric_id): insight.strip() + for metric_id, insight in summary.metric_insights.items() + if str(metric_id).strip() and insight.strip() + } + + +def _report_period_from_rows( + rows: list[tuple[CallImportEvaluationRow, CallImportRow]], +) -> tuple[Optional[date], Optional[date], Optional[str], str]: + dates = [ + source_row.recording_date + for eval_row, source_row in rows + if eval_row.status == "completed" and source_row.recording_date + ] + if not dates: + return None, None, None, "Not specified" + start = min(dates) + end = max(dates) + week_anchor = max(dates) + week_start = week_anchor - timedelta(days=week_anchor.weekday()) + week_end = week_start + timedelta(days=6) + iso_year, iso_week, _ = week_anchor.isocalendar() + label = f"{iso_year}-W{iso_week:02d}" + if week_start.year == week_end.year: + week_range = f"{week_start.strftime('%b %d')}–{week_end.strftime('%b %d, %Y')}" + else: + week_range = ( + f"{week_start.strftime('%b %d, %Y')}–{week_end.strftime('%b %d, %Y')}" + ) + display = f"W{iso_week:02d} · {week_range}" + return start, end, label, display + + +def _aggregate_to_dict(aggregate: CallImportMetricAggregate) -> dict[str, Any]: + if hasattr(aggregate, "model_dump"): + return aggregate.model_dump(mode="json") + return aggregate.dict() + + +def _aggregate_primary_percent( + raw: dict[str, Any], + policy: Optional[MetricFailurePolicy] = None, +) -> Optional[float]: + return aggregate_primary_percent(raw, policy) + + +def _child_names_by_parent( + db: Session, + organization_id: UUID, + parent_metric_ids: Sequence[UUID], +) -> Dict[str, List[str]]: + if not parent_metric_ids: + return {} + children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id.in_(list(parent_metric_ids)), + ) + .all() + ) + out: Dict[str, List[str]] = {} + for child in children: + pid = str(child.parent_metric_id) + out.setdefault(pid, []).append(child.name) + return out + + +def _clustering_context( + db: Session, + evaluation: CallImportEvaluation, + eval_rows: List[CallImportEvaluationRow], +) -> Tuple[ + List[Metric], + List[CallImportMetricAggregate], + Dict[str, MetricFailurePolicy], + Literal["inferred", "user"], + Dict[str, List[str]], +]: + metrics = _metrics_for_clustering(db, evaluation, eval_rows) + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + parent_ids = [ + m.id + for m in metrics + if getattr(m, "selection_mode", None) and not getattr(m, "parent_metric_id", None) + ] + child_names_by_parent = _child_names_by_parent( + db, evaluation.organization_id, parent_ids + ) + policies, source = effective_policies( + evaluation, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + return metrics, aggregates, policies, source, child_names_by_parent + + +def _period_deltas_from_aggregates( + previous_metric_aggregates: list[dict[str, Any]], + current_metric_aggregates: list[dict[str, Any]], + policies: Optional[Dict[str, MetricFailurePolicy]] = None, +) -> dict[str, dict[str, str]]: + current_by_id = {str(item.get("metric_id")): item for item in current_metric_aggregates} + previous_by_id = { + str(item.get("metric_id")): item + for item in previous_metric_aggregates + if isinstance(item, dict) + } + deltas: dict[str, dict[str, str]] = {} + for metric_id, current in current_by_id.items(): + previous_raw = previous_by_id.get(metric_id) + policy = (policies or {}).get(metric_id) + current_pct = _aggregate_primary_percent(current, policy) + previous_pct = ( + _aggregate_primary_percent(previous_raw, policy) + if previous_raw + else None + ) + if current_pct is None or previous_pct is None: + deltas[metric_id] = { + "label": "No previous-week baseline", + "detail": "No comparable prior report snapshot was found.", + } + continue + delta = current_pct - previous_pct + sign = "+" if delta >= 0 else "" + deltas[metric_id] = { + "label": f"{sign}{delta:.1f} pp", + "detail": f"Current report {current_pct:.1f}% vs previous report {previous_pct:.1f}%", + } + return deltas + + +def _period_deltas_from_snapshot( + previous: Optional[CallImportEvaluationReportSnapshot], + current_metric_aggregates: list[dict[str, Any]], +) -> dict[str, dict[str, str]]: + previous_items = ( + previous.metric_aggregates + if previous and isinstance(previous.metric_aggregates, list) + else [] + ) + return _period_deltas_from_aggregates(previous_items, current_metric_aggregates) + + +def _sample_evidence_for_metrics( + rows: list[tuple[CallImportEvaluationRow, CallImportRow]], + metric_ids: set[str], +) -> dict[str, list[dict[str, str]]]: + samples: dict[str, list[dict[str, str]]] = {metric_id: [] for metric_id in metric_ids} + for eval_row, source_row in rows: + scores = eval_row.metric_scores if isinstance(eval_row.metric_scores, dict) else {} + for metric_id in metric_ids: + if len(samples.get(metric_id, [])) >= 4: + continue + score = scores.get(metric_id) + if not isinstance(score, dict): + continue + rationale = score.get("rationale") + transcript = source_row.diarised_transcript or source_row.transcript or "" + quote = rationale if isinstance(rationale, str) and rationale.strip() else transcript[:350] + if quote: + samples.setdefault(metric_id, []).append( + { + "conversation_id": source_row.conversation_id, + "quote": str(quote).strip()[:500], + } + ) + return samples + + +def _fallback_report_narrative( + insight_aggregates: list[dict[str, Any]], + evidence_samples: dict[str, list[dict[str, str]]], +) -> dict[str, Any]: + observations: dict[str, str] = {} + evidence: dict[str, dict[str, str]] = {} + design_notes: list[str] = [] + for aggregate in insight_aggregates: + metric_id = str(aggregate.get("metric_id") or "") + name = str(aggregate.get("metric_name") or "Insight") + counts = aggregate.get("value_counts") if isinstance(aggregate.get("value_counts"), list) else [] + if counts: + top = counts[0] + total = int(aggregate.get("count") or 0) or sum( + int(item.get("count") or 0) for item in counts if isinstance(item, dict) + ) + pct = (int(top.get("count") or 0) / total) * 100 if total else 0 + observations[metric_id] = ( + f"{top.get('label')} is the dominant {name.lower()} category at {pct:.1f}% of classified calls." + ) + design_notes.append( + f"{name}: {top.get('label')} is the largest segment and should be reviewed for workflow or prompt improvements." + ) + sample = (evidence_samples.get(metric_id) or [{}])[0] + if sample: + evidence[metric_id] = sample + return { + "observations": observations, + "evidence": evidence, + "design_notes": design_notes[:7], + "audit_summary": None, + } + + +def _generate_report_narrative( + db: Session, + organization_id: UUID, + *, + metric_aggregates: list[dict[str, Any]], + insight_aggregates: list[dict[str, Any]], + period_delta_by_metric: dict[str, dict[str, str]], + evidence_samples: dict[str, list[dict[str, str]]], + report_config: dict[str, Any], +) -> dict[str, Any]: + if not insight_aggregates: + return {"observations": {}, "evidence": {}, "design_notes": [], "audit_summary": None} + try: + from app.services.ai.llm_resolver import get_llm_provider_and_model + from app.services.ai.llm_service import llm_service + + provider_enum, model_str = get_llm_provider_and_model(organization_id, db, None, None) + prompt = ( + "You are writing a vendor-safe external call quality audit report. " + "Return strict JSON with keys observations (object keyed by metric_id), " + "evidence (object keyed by metric_id with conversation_id and quote), " + "design_notes (array of concise numbered-note strings), and audit_summary (string). " + "Use only the supplied aggregates and evidence samples.\n\n" + + json.dumps( + { + "metric_aggregates": metric_aggregates[:30], + "insight_aggregates": insight_aggregates, + "period_deltas": period_delta_by_metric, + "evidence_samples": evidence_samples, + "report_config": report_config, + }, + default=str, + ) + ) + llm_result = llm_service.generate_response( + messages=[ + {"role": "system", "content": "Return JSON only. No markdown."}, + {"role": "user", "content": prompt}, + ], + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + temperature=0.2, + max_tokens=1200, + ) + parsed = json.loads(str(llm_result.content or "{}")) + if isinstance(parsed, dict): + fallback = _fallback_report_narrative(insight_aggregates, evidence_samples) + return { + "observations": parsed.get("observations") or fallback["observations"], + "evidence": parsed.get("evidence") or fallback["evidence"], + "design_notes": parsed.get("design_notes") or fallback["design_notes"], + "audit_summary": parsed.get("audit_summary") or fallback["audit_summary"], + } + except Exception as exc: # noqa: BLE001 + logger.warning("Report narrative LLM generation fell back to deterministic text: {}", exc) + return _fallback_report_narrative(insight_aggregates, evidence_samples) + + +@router.get( + "/{eval_id}/baseline-candidates", + response_model=CallImportEvaluationBaselineCandidatesResponse, + operation_id="listCallImportEvaluationBaselineCandidates", +) +async def list_call_import_evaluation_baseline_candidates( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationBaselineCandidatesResponse: + del api_key + call_import = _require_import(db, call_import_id, organization_id) + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + rows = _evaluation_rows_for_period(db, evaluation.id) + period_start, _period_end, _derived_period_label, _period_display = _report_period_from_rows( + rows + ) + candidates = _baseline_candidate_evaluations( + db, + organization_id, + call_import.workspace_id, + evaluation, + period_start, + ) + default_evaluation_id = next( + (item["evaluation_id"] for item in candidates if item.get("is_default")), + None, + ) + return CallImportEvaluationBaselineCandidatesResponse( + items=[CallImportEvaluationBaselineCandidate(**item) for item in candidates], + default_evaluation_id=default_evaluation_id, + ) + + +@router.post( + "/{eval_id}/pdf-report", + operation_id="generateCallImportEvaluationPdfReport", + dependencies=[Depends(require_call_import_capability(REPORTS_GENERATE))], +) +async def generate_call_import_evaluation_pdf_report( + call_import_id: UUID, + eval_id: UUID, + payload: CallImportEvaluationPdfReportRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> StreamingResponse: + del api_key + call_import = _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + is_internal = payload.report_type == "internal" + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + rows = sorted( + load_evaluation_row_pairs(db, eval_id), + key=lambda pair: int(pair[1].row_index or 0), + ) + else: + rows = ( + db.query(CallImportEvaluationRow, CallImportRow) + .join( + CallImportRow, + CallImportRow.id == CallImportEvaluationRow.call_import_row_id, + ) + .filter(CallImportEvaluationRow.evaluation_id == eval_id) + .order_by(CallImportRow.row_index.asc()) + .all() + ) + report_config = payload.report_config if isinstance(payload.report_config, dict) else {} + metrics = _display_metrics_for_pdf_report(db, organization_id, evaluation) + configured_quality_ids = { + str(item) + for item in report_config.get("quality_metric_ids", []) + if item + } + configured_insight_ids = { + str(item.get("metric_id") or item) + for item in report_config.get("insights", []) + if item + } + if configured_quality_ids or configured_insight_ids: + allowed_ids = configured_quality_ids | configured_insight_ids + metrics = [metric for metric in metrics if str(metric.id) in allowed_ids] + + eval_rows = [eval_row for eval_row, _source_row in rows] + aggregate_models = _compute_metric_aggregates(db, evaluation, eval_rows) + selected_report_metric_ids = {str(metric.id) for metric in metrics} + aggregate_dicts = [ + _aggregate_to_dict(aggregate) + for aggregate in aggregate_models + if aggregate.metric_id in selected_report_metric_ids + ] + insight_metric_ids = { + str(metric.id) + for metric in metrics + if _metric_is_user_insight(metric) + } + metric_aggregates = [ + item for item in aggregate_dicts if str(item.get("metric_id")) not in insight_metric_ids + ] + insight_aggregates = [ + item for item in aggregate_dicts if str(item.get("metric_id")) in insight_metric_ids + ] + period_start, period_end, derived_period_label, period_display = _report_period_from_rows(rows) + period_label = (payload.period_label or derived_period_label or "").strip() or None + include_period_delta = ( + payload.include_period_delta or payload.include_weekly_delta + ) + previous_snapshot = None + period_delta_by_metric: dict[str, dict[str, str]] = {} + baseline_evaluation: Optional[CallImportEvaluation] = None + if include_period_delta and period_start: + baseline_evaluation = _resolve_baseline_evaluation( + db, + organization_id, + call_import.workspace_id, + evaluation, + period_start, + payload.baseline_evaluation_id, + ) + if baseline_evaluation: + period_delta_by_metric = _period_deltas_from_evaluation( + db, + baseline_evaluation, + metric_aggregates, + evaluation, + [eval_row for eval_row, _ in rows], + ) + period_delta_by_metric = _period_deltas_with_explanations( + db, + organization_id, + evaluation, + baseline_evaluation, + period_delta_by_metric, + ) + benchmark_context = _benchmark_context_for_evaluation(db, baseline_evaluation) + evidence_samples = _sample_evidence_for_metrics(rows, insight_metric_ids) + cached_tldr_summary = _tldr_summary_payload(evaluation) + cached_user_insights = _user_insights_payload(evaluation) + cached_metric_clusters = _metric_clusters_payload(evaluation) + cached_prompt_improvements = _prompt_improvements_payload(evaluation) + generated_insights_for_pdf = _selected_generated_user_insights( + cached_user_insights, + report_config, + ) + metric_clusters_for_pdf = _selected_metric_clusters_for_pdf( + cached_metric_clusters, + report_config, + ) + prompt_improvements_for_pdf = _selected_prompt_improvements_for_pdf( + cached_prompt_improvements, + report_config, + ) + narrative = _generate_report_narrative( + db, + organization_id, + metric_aggregates=metric_aggregates, + insight_aggregates=insight_aggregates if is_internal else [], + period_delta_by_metric=period_delta_by_metric, + evidence_samples=evidence_samples if is_internal else {}, + report_config=report_config, + ) + + generated_at = datetime.now(timezone.utc) + branding_images, custom_heading = _report_branding_for_import_workspace( + db, + organization_id, + call_import.workspace_id, + internal_brand_image_id=payload.internal_brand_image_id, + external_brand_image_id=payload.external_brand_image_id, + ) + eval_row_list = [eval_row for eval_row, _ in rows] + pdf_aggregates = _compute_metric_aggregates(db, evaluation, eval_row_list) + pdf_parent_ids = [ + m.id + for m in metrics + if getattr(m, "selection_mode", None) + and not getattr(m, "parent_metric_id", None) + ] + pdf_child_map = _child_names_by_parent( + db, evaluation.organization_id, pdf_parent_ids + ) + failure_policies_for_pdf, _fp_source = effective_policies( + evaluation, + metrics, + pdf_aggregates, + child_names_by_parent=pdf_child_map, + ) + try: + pdf_started = datetime.now(timezone.utc) + pdf_bytes = await asyncio.to_thread( + call_import_evaluation_pdf_report_service.render_pdf, + vendor_name=payload.vendor_name, + call_import=call_import, + evaluation=evaluation, + metrics=metrics, + rows=rows, + failure_policies=failure_policies_for_pdf, + generated_at=generated_at, + internal=is_internal, + logo_data_uris=branding_images, + custom_heading=custom_heading, + include_weekly_delta=include_period_delta, + period_delta_by_metric=period_delta_by_metric, + use_case=payload.use_case, + period_display=period_display, + total_metric_count=db.query(Metric) + .filter(Metric.organization_id == organization_id, Metric.enabled.is_(True)) + .count(), + report_config=report_config, + narrative=narrative, + audit_summary=_audit_summary_text_from_tldr(cached_tldr_summary), + metric_insights=_metric_insights_from_tldr(cached_tldr_summary), + benchmark_context=benchmark_context, + generated_user_insights=generated_insights_for_pdf, + user_insights_overview=( + cached_user_insights.overview if cached_user_insights else None + ), + metric_clusters=metric_clusters_for_pdf, + metric_clusters_overview=( + cached_metric_clusters.overview if cached_metric_clusters else None + ), + prompt_improvements=prompt_improvements_for_pdf, + platform_base_url=payload.platform_base_url, + ) + logger.info( + "PDF report render finished in {:.1f}s for evaluation {}", + (datetime.now(timezone.utc) - pdf_started).total_seconds(), + eval_id, + ) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to generate PDF report for call import {} evaluation {}", + call_import_id, + eval_id, + ) + raise HTTPException( + status_code=500, + detail=f"Failed to generate PDF report: {exc}", + ) from exc + + snapshot = CallImportEvaluationReportSnapshot( + evaluation_id=evaluation.id, + call_import_id=call_import.id, + organization_id=organization_id, + workspace_id=call_import.workspace_id, + period_label=period_label, + period_start=period_start, + period_end=period_end, + report_config=report_config, + selected_metric_ids=[str(metric.id) for metric in metrics], + metric_aggregates=metric_aggregates, + insight_aggregates=insight_aggregates, + narrative=narrative, + total_calls=evaluation.total_rows, + selected_metric_count=len(metrics), + total_metric_count=db.query(Metric) + .filter(Metric.organization_id == organization_id, Metric.enabled.is_(True)) + .count(), + ) + db.add(snapshot) + db.commit() + + filename = ( + f"{_report_filename_slug(payload.vendor_name)}-" + f"{payload.report_type}-quality-metric-audit-{eval_id}.pdf" + ) + return StreamingResponse( + iter([pdf_bytes]), + media_type="application/pdf", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + +@router.patch( + "/{eval_id}", + response_model=CallImportEvaluationResponse, + operation_id="updateCallImportEvaluation", +) +async def update_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + payload: CallImportEvaluationUpdate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationResponse: + """Edit metadata on an existing evaluation run (currently just ``name``).""" + + del api_key + _require_import(db, call_import_id, organization_id) + + row = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + # Treat unset vs explicit ``None`` differently: unset = leave alone, + # explicit ``None`` or empty string = clear the name. + payload_data = payload.model_dump(exclude_unset=True) + if "name" in payload_data: + row.name = _normalize_name(payload_data["name"]) + + stamp_evaluation_actor(row, principal) + db.commit() + db.refresh(row) + return _serialize_eval(db, row) + + +def _revoke_pending_tasks(evaluation: CallImportEvaluation) -> None: + """Best-effort cancel of any in-flight Celery tasks for an evaluation.""" + + if not evaluation.celery_group_id and not any( + r.celery_task_id for r in evaluation.row_results + ): + return + try: + from app.workers.celery_app import celery_app + + pending_task_ids = [ + eval_row.celery_task_id + for eval_row in evaluation.row_results + if eval_row.celery_task_id + and eval_row.status in {"pending", "running"} + ] + if pending_task_ids: + celery_app.control.revoke(pending_task_ids, terminate=False) + except Exception: + # Best effort — DB delete remains the source of truth. + pass + + +# --------------------------------------------------------------------------- +# User-initiated cancel for in-flight evaluation rows +# --------------------------------------------------------------------------- +# +# Evaluation rows can sit in ``running`` for many minutes when the underlying +# LLM / audio metric call is slow or wedged (the worker carries an 8 min +# soft / 10 min hard time limit). Without a cancel affordance the operator's +# only recourse is to wait for Celery's time limit to fire — or to manually +# mutate the DB. These helpers + the two endpoints below give the UI a +# first-class "Abort" button mirroring the diarisation cancel pattern at +# ``app.api.v1.routes.call_imports`` (``_apply_diarisation_cancel`` etc.). +# +# Why ``terminate=True``: the legacy ``_revoke_pending_tasks`` above uses +# ``terminate=False`` because it's called from delete-flow paths where the +# task may simply not get to run (a worker pulls it off the queue and drops +# it). For a user-initiated cancel we want SIGTERM to interrupt the worker +# mid-LLM/audio call so the in-flight HTTP request actually aborts. +# ``terminate=True`` routes the signal to the executing process; we spell +# ``signal="SIGTERM"`` out for clarity even though it's the default. + +# Sentinel error message stamped on cancelled rows. Read by the eval worker's +# ``_was_cancelled_externally`` guard (see +# :mod:`app.workers.tasks.evaluate_call_import_row`) so a worker that's already +# past its slowest operation can't overwrite the cancelled state with its own +# terminal status. Touching either copy means touching both. +EVAL_CANCELLED_BY_USER_ERROR: str = "Evaluation cancelled by user" + + +def _cancellable_eval_states() -> Tuple[str, ...]: + """States that an evaluation row can be cancelled from. + + Kept as a tiny helper so adding a future ``"queued"`` / ``"retrying"`` + state only needs one edit. + """ + return ("pending", "running") + + +def _revoke_eval_task(eval_row: CallImportEvaluationRow) -> None: + """Best-effort revoke of a single eval row's Celery task. + + Always swallows control-plane exceptions — Celery's control bus is + inherently best-effort and a missed revoke is not catastrophic + because the DB row is already flipped to ``failed`` by the caller + before this runs (so the UI immediately reflects the cancel; if + the task happens to finish anyway, the worker's finaliser skips + over the row via :data:`EVAL_CANCELLED_BY_USER_ERROR`). + """ + task_id = (eval_row.celery_task_id or "").strip() + if not task_id: + return + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke( + task_id, terminate=True, signal="SIGTERM" + ) + logger.info( + "Revoked evaluation task {} for eval row {}", + task_id, + eval_row.id, + ) + except Exception as exc: # noqa: BLE001 — revoke is best-effort + logger.warning( + "Failed to revoke evaluation task {} for eval row {}: {}", + task_id, + eval_row.id, + exc, + ) + + +def _apply_evaluation_cancel( + eval_rows: List[CallImportEvaluationRow], +) -> Tuple[int, int]: + """Cancel every cancellable row in ``eval_rows``. + + Returns ``(cancelled, skipped)`` so the caller can build a typed + response without re-querying the DB. The caller is responsible for + ``db.commit()`` after this returns — we deliberately don't commit + here so a batch endpoint can flush all rows in one transaction. + """ + cancellable_states = _cancellable_eval_states() + cancelled = 0 + skipped = 0 + now = datetime.now(timezone.utc) + for eval_row in eval_rows: + if (eval_row.status or "").lower() not in cancellable_states: + skipped += 1 + continue + # Flip the row state BEFORE we revoke so the UI's next poll + # already shows the cancel, even if Celery's control plane is + # slow to ack. + eval_row.status = "failed" + eval_row.error_message = EVAL_CANCELLED_BY_USER_ERROR + eval_row.finished_at = now + _revoke_eval_task(eval_row) + # Drop the task id so a follow-up retry (or a stale poll) can't + # accidentally re-revoke or get confused. + eval_row.celery_task_id = None + cancelled += 1 + return cancelled, skipped + + +def _claim_evaluation_bulk_operation( + evaluation_id: UUID, + operation: str, +) -> None: + """Reserve the run for a single bulk worker pass; 409 if one is active.""" + from app.services.call_imports.evaluation_bulk_op import ( + get_evaluation_bulk_operation, + try_set_evaluation_bulk_operation, + ) + + if try_set_evaluation_bulk_operation(evaluation_id, operation): # type: ignore[arg-type] + return + existing = get_evaluation_bulk_operation(evaluation_id) or operation + raise HTTPException( + status_code=409, + detail=( + f"A bulk {existing.replace('_', ' ')} operation is already in " + "progress for this evaluation. Wait for it to finish before " + "starting another action." + ), + ) + + +def _require_no_evaluation_bulk_operation(evaluation_id: UUID) -> None: + from app.services.call_imports.evaluation_bulk_op import ( + get_evaluation_bulk_operation, + ) + + existing = get_evaluation_bulk_operation(evaluation_id) + if existing: + raise HTTPException( + status_code=409, + detail=( + f"A bulk {existing.replace('_', ' ')} operation is already in " + "progress for this evaluation. Wait for it to finish before " + "starting another action." + ), + ) + + +@router.post( + "/{eval_id}/cancel", + response_model=CallImportEvaluationBulkActionResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="cancelCallImportEvaluation", +) +async def cancel_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationBulkActionResponse: + """Abort all in-flight (or queued) rows in a single evaluation run. + + Idempotent: calling on a run whose rows are already terminal returns + ``target_count=0`` with 202 so the UI can fire this from an + "Abort" button without having to pre-check the state. + + Heavy row resets and Celery revokes run in a background worker so + large batches do not block the API thread. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + from app.services.call_imports.bulk_ops import count_evaluation_cancel_targets + + target_count = count_evaluation_cancel_targets(db, eval_id, mode="abort") + if target_count == 0: + return CallImportEvaluationBulkActionResponse( + accepted=True, + target_count=0, + evaluation_id=eval_id, + ) + + _claim_evaluation_bulk_operation(eval_id, "abort") + evaluation.status = "cancelled" + stamp_evaluation_actor(evaluation, principal) + db.commit() + + from app.workers.tasks.call_import_bulk_ops import ( + cancel_call_import_evaluation_task, + ) + + cancel_call_import_evaluation_task.delay(str(eval_id), mode="abort") + return CallImportEvaluationBulkActionResponse( + accepted=True, + target_count=target_count, + evaluation_id=eval_id, + ) + + +@router.post( + "/{eval_id}/force-fail-pending", + response_model=CallImportEvaluationBulkActionResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="forceFailCallImportEvaluationPending", +) +async def force_fail_pending_call_import_evaluation_rows( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationBulkActionResponse: + """Force-fail only rows currently in ``pending`` for a single run. + + This is narrower than :func:`cancel_call_import_evaluation`: it leaves + ``running`` rows untouched so operators can clear permanently queued rows + without interrupting in-flight evaluations. + + Row updates run in a background worker so large batches do not block + the API thread. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + from app.services.call_imports.bulk_ops import count_evaluation_cancel_targets + + target_count = count_evaluation_cancel_targets( + db, eval_id, mode="force_fail_pending" + ) + if target_count == 0: + return CallImportEvaluationBulkActionResponse( + accepted=True, + target_count=0, + evaluation_id=eval_id, + ) + + _claim_evaluation_bulk_operation(eval_id, "force_fail_pending") + stamp_evaluation_actor(evaluation, principal) + db.commit() + + from app.workers.tasks.call_import_bulk_ops import ( + cancel_call_import_evaluation_task, + ) + + cancel_call_import_evaluation_task.delay( + str(eval_id), mode="force_fail_pending" + ) + return CallImportEvaluationBulkActionResponse( + accepted=True, + target_count=target_count, + evaluation_id=eval_id, + ) + + +@router.post( + "/{eval_id}/rows/{eval_row_id}/cancel", + response_model=CallImportEvaluationRowResponse, + status_code=status.HTTP_200_OK, + operation_id="cancelCallImportEvaluationRow", +) +async def cancel_call_import_evaluation_row( + call_import_id: UUID, + eval_id: UUID, + eval_row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationRowResponse: + """Abort an in-flight (or queued) evaluation for a single row. + + Idempotent: calling on a row that's already terminal (``completed`` + / ``failed``) returns the row unchanged with a 200 so the UI can + wire this to a "Stop" button without having to pre-check the + state. Updates the parent run's rollup so its counters reflect + the cancel immediately. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + _require_no_evaluation_bulk_operation(eval_id) + + from app.db_sharding.eval_rows import evaluation_row_session + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + try: + with evaluation_row_session(eval_row_id) as ( + row_db, + _catalog_db, + eval_row, + source_row, + _shard_id, + ): + if eval_row.evaluation_id != eval_id: + raise HTTPException( + status_code=404, + detail="Evaluation row not found in this run", + ) + _apply_evaluation_cancel([eval_row]) + row_db.commit() + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + row_db.refresh(eval_row) + return _to_evaluation_row_response(eval_row, source_row, evaluation) + except LookupError as exc: + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) from exc + + eval_row = ( + db.query(CallImportEvaluationRow) + .filter( + CallImportEvaluationRow.id == eval_row_id, + CallImportEvaluationRow.evaluation_id == eval_id, + ) + .first() + ) + if not eval_row: + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) + + _apply_evaluation_cancel([eval_row]) + db.flush() + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(eval_row) + + source_row = ( + db.query(CallImportRow) + .filter(CallImportRow.id == eval_row.call_import_row_id) + .first() + ) + + return _to_evaluation_row_response(eval_row, source_row, evaluation) + + +@router.delete( + "/{eval_id}", + status_code=status.HTTP_204_NO_CONTENT, + operation_id="deleteCallImportEvaluation", +) +async def delete_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> Response: + del api_key + _require_import(db, call_import_id, organization_id) + + row = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + _revoke_pending_tasks(row) + + db.delete(row) + db.commit() + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +@router.post( + "/bulk-delete", + status_code=status.HTTP_200_OK, + operation_id="bulkDeleteCallImportEvaluations", +) +async def bulk_delete_call_import_evaluations( + call_import_id: UUID, + payload: CallImportEvaluationBulkDelete, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> Dict[str, int]: + """Delete multiple evaluation runs scoped to one call import. + + Mirrors :func:`delete_call_import_evaluation` but in bulk so the UI + can clear out a multi-select. Unknown ids (already deleted, or + belonging to a different org/import) are silently skipped — the + response just reports how many actually went away. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + if not payload.evaluation_ids: + return {"deleted": 0} + + rows = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id.in_(payload.evaluation_ids), + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .all() + ) + deleted = 0 + for row in rows: + _revoke_pending_tasks(row) + db.delete(row) + deleted += 1 + db.commit() + return {"deleted": deleted} + + +# --------------------------------------------------------------------------- +# Aggregation: turns per-row metric scores into histograms / value counts. +# +# Designed to be cheap enough to call on every page load: we read each +# evaluation row once, bucket numeric values into a fixed 10-bin +# histogram, and tally the top categorical values. Scaling concerns +# (millions of rows) are deferred — at that point we'd push this into a +# Postgres aggregate query, but for typical CSV imports (<10k rows) the +# Python pass is fast enough and dramatically simpler. +# --------------------------------------------------------------------------- + + +_HISTOGRAM_BUCKETS = 10 +_TOP_VALUE_COUNTS = 10 + + +def _coerce_numeric(value: Any) -> Optional[float]: + """Return ``value`` as ``float`` when it's numeric; ``None`` otherwise.""" + if isinstance(value, bool): + # Booleans are ints in Python; treat them as categorical so + # pass/fail metrics show up in value_counts instead of becoming + # a degenerate {0,1} histogram. + return None + if isinstance(value, (int, float)) and math.isfinite(value): + return float(value) + if isinstance(value, str): + try: + f = float(value) + if math.isfinite(f): + return f + except ValueError: + return None + return None + + +def _coerce_category(value: Any) -> Optional[str]: + """Render ``value`` as a label suitable for a value_counts bucket.""" + if value is None: + return None + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, str): + text = value.strip() + return text or None + # Lists / dicts: stringify so they still group sensibly without + # exploding the cardinality (worst case: everything is "[…]" once). + return str(value) + + +def _build_histogram( + values: List[float], +) -> List[CallImportMetricHistogramBucket]: + """Fixed-bin histogram over ``values``; returns [] for <2 values.""" + if len(values) < 2: + return [] + lo = min(values) + hi = max(values) + if lo == hi: + # All values identical — render a single bucket so the UI shows a + # spike rather than empty space. + return [ + CallImportMetricHistogramBucket(x0=lo, x1=hi, count=len(values)) + ] + width = (hi - lo) / _HISTOGRAM_BUCKETS + buckets: List[List[float]] = [[] for _ in range(_HISTOGRAM_BUCKETS)] + for v in values: + # Right-edge inclusive on the last bucket so ``hi`` doesn't fall + # off into a non-existent bucket index. + idx = int((v - lo) / width) + if idx >= _HISTOGRAM_BUCKETS: + idx = _HISTOGRAM_BUCKETS - 1 + buckets[idx].append(v) + return [ + CallImportMetricHistogramBucket( + x0=lo + i * width, + x1=lo + (i + 1) * width, + count=len(bucket), + ) + for i, bucket in enumerate(buckets) + ] + + +def _percentile(values: List[float], pct: float) -> Optional[float]: + """Linear-interpolated percentile compatible with NumPy default.""" + if not values: + return None + sorted_vals = sorted(values) + if len(sorted_vals) == 1: + return sorted_vals[0] + rank = (pct / 100.0) * (len(sorted_vals) - 1) + lo = int(math.floor(rank)) + hi = int(math.ceil(rank)) + if lo == hi: + return sorted_vals[lo] + frac = rank - lo + return sorted_vals[lo] + (sorted_vals[hi] - sorted_vals[lo]) * frac + + +def _compute_metric_aggregates( + db: Session, + evaluation: CallImportEvaluation, + eval_rows: List[CallImportEvaluationRow], +) -> List[CallImportMetricAggregate]: + """Collapse per-row ``metric_scores`` into one aggregate per metric. + + Selected metrics are read fresh from the DB so the response always + surfaces the current ``metric.name`` / ``metric_type`` even when a + metric was renamed after the run finished. + """ + + selected_ids = _serialize_selected_metric_ids(evaluation.selected_metric_ids) + # Include parent metrics from selected_metric_groups so they appear + # alongside their children in the aggregate response. Use ``getattr`` + # with a default so the helper still works for callers that pass + # lightweight objects (tests, in-memory shims) that don't carry the + # attribute at all. + groups_raw_candidate = getattr(evaluation, "selected_metric_groups", None) + groups_raw = ( + groups_raw_candidate if isinstance(groups_raw_candidate, dict) else {} + ) + for parent_str in groups_raw.keys(): + try: + pid = UUID(parent_str) + if pid not in selected_ids: + selected_ids.append(pid) + except (TypeError, ValueError): + continue + + metrics = _metrics_for_ids(db, evaluation.organization_id, selected_ids) + metric_meta: Dict[str, Metric] = {str(m.id): m for m in metrics} + + # Default to selected metrics, but also include any metric ids that + # surface in row scores even if missing from the metric registry — + # otherwise renaming/deleting a metric mid-run would silently drop + # results from the chart. + discovered_ids: List[str] = list(metric_meta.keys()) + for row in eval_rows: + scores = row.metric_scores if isinstance(row.metric_scores, dict) else {} + for metric_id_str in scores.keys(): + if metric_id_str not in metric_meta and metric_id_str not in discovered_ids: + discovered_ids.append(metric_id_str) + + results: List[CallImportMetricAggregate] = [] + + for metric_id_str in discovered_ids: + meta = metric_meta.get(metric_id_str) + numeric_values: List[float] = [] + category_counts: Dict[str, int] = {} + # For multi-label parents we still need to know how many rows + # were scored (each row votes for >=1 label) so the n-badge in + # the UI shows "n=50" instead of the misleading "n=208" sum. + multi_label_rows_scored = 0 + # Unordered pair tally for the co-occurrence heatmap. Keys are + # ``(label_a, label_b)`` with ``a < b`` so we never double-count + # the same unordered pair. Only populated for multi-label + # parents — every other metric leaves this empty. + multi_label_pair_counts: Dict[Tuple[str, str], int] = {} + skipped = 0 + errored = 0 + observed_metric_type: Optional[str] = None + observed_name: Optional[str] = None + + # ``meta`` is a real ``Metric`` row in production, but tests + # frequently pass a lightweight stub. Pull the two attributes + # we need via ``getattr`` so a stub that only sets ``id`` / + # ``name`` / ``metric_type`` doesn't blow up here. + is_multi_label_parent = bool( + meta + and getattr(meta, "selection_mode", None) == "multi_label" + and not getattr(meta, "parent_metric_id", None) + ) + + for row in eval_rows: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else {} + ) + entry = scores.get(metric_id_str) + if not isinstance(entry, dict): + continue + if entry.get("metric_name"): + observed_name = entry.get("metric_name") + if entry.get("type"): + observed_metric_type = entry.get("type") + if entry.get("skipped"): + skipped += 1 + continue + if entry.get("error"): + errored += 1 + continue + + # Multi-label parents store a comma-joined value that + # isn't useful as a single category; instead tally each + # selected child individually so the chart shows per-label + # counts that mirror the children's own boolean histograms. + if is_multi_label_parent: + selected = entry.get("selected_child_names") + if isinstance(selected, list) and selected: + multi_label_rows_scored += 1 + cleaned: List[str] = [] + for label in selected: + text_label = str(label).strip() or None + if text_label: + cleaned.append(text_label) + category_counts[text_label] = ( + category_counts.get(text_label, 0) + 1 + ) + # Emit one increment per unordered pair of distinct + # labels that fired together on this row. ``cleaned`` + # is deduplicated first because the LLM occasionally + # repeats a label inside ``selected_child_names``. + distinct = sorted(set(cleaned)) + for i in range(len(distinct)): + for j in range(i + 1, len(distinct)): + pair = (distinct[i], distinct[j]) + multi_label_pair_counts[pair] = ( + multi_label_pair_counts.get(pair, 0) + 1 + ) + continue + + value = entry.get("value") + numeric = _coerce_numeric(value) + if numeric is not None: + numeric_values.append(numeric) + continue + category = _coerce_category(value) + if category is not None: + category_counts[category] = category_counts.get(category, 0) + 1 + + # ``count`` is "rows scored". For numeric / single-choice + # metrics that's the same as ``len(numeric) + sum(categories)`` + # because each scored row contributes exactly one observation. + # Multi-label parents however contribute one observation per + # selected child, so summing ``category_counts`` over-counts — + # we tracked rows-scored separately above and use it here. + rows_scored = ( + multi_label_rows_scored + if is_multi_label_parent + else len(numeric_values) + sum(category_counts.values()) + ) + + # Build numeric stats first, then categorical (both can coexist). + agg = CallImportMetricAggregate( + metric_id=metric_id_str, + metric_name=( + (meta.name if meta else observed_name) or "Unknown metric" + ), + metric_type=( + meta.metric_type if meta else observed_metric_type + ), + metric_category=( + "user_insight" + if meta is not None and _metric_is_user_insight(meta) + else "quality" + ) + or "quality", + is_multi_label_parent=is_multi_label_parent, + count=rows_scored, + skipped_count=skipped, + error_count=errored, + ) + if numeric_values: + agg.mean = float(statistics.fmean(numeric_values)) + agg.median = float(statistics.median(numeric_values)) + agg.min = min(numeric_values) + agg.max = max(numeric_values) + agg.stddev = ( + float(statistics.pstdev(numeric_values)) + if len(numeric_values) > 1 + else 0.0 + ) + agg.p25 = _percentile(numeric_values, 25) + agg.p75 = _percentile(numeric_values, 75) + agg.p95 = _percentile(numeric_values, 95) + agg.histogram_buckets = _build_histogram(numeric_values) + if category_counts: + sorted_counts = sorted( + category_counts.items(), key=lambda kv: kv[1], reverse=True + ) + agg.value_counts = [ + CallImportMetricValueCount(label=label, count=count) + for label, count in sorted_counts[:_TOP_VALUE_COUNTS] + ] + # Restrict the heatmap to pairs of labels we actually + # rendered above so the frontend never has to match + # against truncated/missing rows. Sorted desc by pair + # count to keep the most informative cells in the + # response when ``_TOP_VALUE_COUNTS`` clipped the matrix. + if is_multi_label_parent and multi_label_pair_counts: + kept_labels = { + label for label, _ in sorted_counts[:_TOP_VALUE_COUNTS] + } + pair_items = [ + (a, b, count) + for (a, b), count in multi_label_pair_counts.items() + if a in kept_labels and b in kept_labels + ] + pair_items.sort(key=lambda t: t[2], reverse=True) + agg.co_occurrence = [ + CallImportMetricLabelPair(a=a, b=b, count=count) + for a, b, count in pair_items + ] + + results.append(agg) + + # Sort so each parent metric immediately precedes its children. + # The Visualizations grid renders metrics top-to-bottom in this + # order, so multi-label parents (the "summary" chart) sit above + # the per-child boolean histograms that drill into them. Metrics + # whose ``meta`` row was deleted mid-run (``meta is None``) sink + # to the bottom but keep their relative order. + enumerated = list(enumerate(results)) + + def _sort_key(item: Tuple[int, CallImportMetricAggregate]): + original_idx, agg = item + meta = metric_meta.get(agg.metric_id) + if meta is None: + return (1, "", 1, "", original_idx) + parent_id = getattr(meta, "parent_metric_id", None) + # Group key: a child shares its parent's UUID; a parent + # uses its own UUID. Within a group, depth=0 (parent) sorts + # before depth=1 (child); ties break alphabetically by name + # so children render in a stable order regardless of which + # row scored which label first. + if parent_id is None: + group_key = str(meta.id) + depth = 0 + else: + group_key = str(parent_id) + depth = 1 + return ( + 0, + group_key, + depth, + (getattr(meta, "name", "") or "").lower(), + original_idx, + ) + + enumerated.sort(key=_sort_key) + return [agg for _idx, agg in enumerated] + + +@router.get( + "/{eval_id}/aggregate", + response_model=CallImportEvaluationAggregateResponse, + operation_id="getCallImportEvaluationAggregate", +) +async def get_call_import_evaluation_aggregate( + call_import_id: UUID, + eval_id: UUID, + baseline_evaluation_id: Optional[UUID] = Query(None), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationAggregateResponse: + """Return per-metric distributions for the Visualizations tab. + + The shape is intentionally chart-friendly: histograms for numeric + metrics, top-N value counts for categorical/text metrics, plus + summary stats (mean/p50/p95) so the UI can render summary cards + without recomputing on the client. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + eval_rows = _load_eval_rows(db, eval_id) + + metrics = _compute_metric_aggregates(db, evaluation, eval_rows) + + period_deltas: dict[str, MetricPeriodDelta] = {} + resolved_baseline_id: Optional[UUID] = None + if baseline_evaluation_id is not None: + call_import = _require_import(db, call_import_id, organization_id) + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + + rows = load_evaluation_row_pairs(db, eval_id) + period_start, _, _, _ = _report_period_from_rows(rows) + baseline_evaluation = _resolve_baseline_evaluation( + db, + organization_id, + call_import.workspace_id, + evaluation, + period_start, + str(baseline_evaluation_id), + ) + if baseline_evaluation: + resolved_baseline_id = baseline_evaluation.id + metric_aggregates_dicts = [ + _aggregate_to_dict(agg) for agg in metrics + ] + raw_deltas = _period_deltas_from_evaluation( + db, + baseline_evaluation, + metric_aggregates_dicts, + evaluation, + eval_rows, + ) + raw_deltas = _period_deltas_with_explanations( + db, + organization_id, + evaluation, + baseline_evaluation, + raw_deltas, + ) + period_deltas = { + metric_id: MetricPeriodDelta( + label=delta.get("label") or "", + detail=delta.get("detail") or "", + why=(delta.get("why") or "").strip() or None, + ) + for metric_id, delta in raw_deltas.items() + } + + _fp_stored, failure_policies_source = policies_from_evaluation_raw( + evaluation.metric_clusters + ) + return CallImportEvaluationAggregateResponse( + evaluation_id=eval_id, + total_rows=evaluation.total_rows, + completed_rows=evaluation.completed_rows, + failed_rows=evaluation.failed_rows, + metrics=metrics, + period_deltas=period_deltas, + baseline_evaluation_id=resolved_baseline_id, + failure_policies_source=failure_policies_source, + ) + + +# --------------------------------------------------------------------------- +# TLDR insights: LLM-generated narrative + bullet patterns rendered above +# the Visualizations charts. Cached on ``CallImportEvaluation.tldr_summary`` +# so the page never auto-burns LLM tokens; the user explicitly clicks +# "Generate summary" or "Regenerate" from the empty-state CTA. +# --------------------------------------------------------------------------- + + +_INSIGHTS_SYSTEM_PROMPT = ( + "You are a senior conversation-analytics reviewer. You will be " + "given aggregated metric statistics + a sample of rationales for " + "the rows of a single call-import evaluation. Identify the most " + "useful PATTERNS that hold ACROSS the calls -- not just per-metric " + "numbers. Look for combinations (e.g. `when X happens, Y also " + "tends to happen`), notable outliers, frequent failure modes, and " + "any signal that would change how a reviewer triages the run.\n\n" + "Return STRICT JSON only, with this shape and no extra keys:\n" + "{\n" + ' "narrative": "",\n' + ' "patterns": ["", "", ...],\n' + ' "metric_insights": {"": "<2-3 line business meaning>"}\n' + "}\n\n" + "Constraints:\n" + "- narrative is the ONLY text shown in the external audit summary and " + "Visualizations TLDR; keep it to at most 3 short sentences (~300 chars).\n" + "- patterns are optional supporting notes and are NOT rendered in the " + "audit summary; keep 0 to 3 bullets if supplied, each <= 120 characters.\n" + "- metric_insights must include one entry for each top-level metric id supplied.\n" + "- Each metric insight should explain what the metric means for the business and what the current distribution suggests, not restate the metric rubric.\n" + "- Avoid restating raw counts unless they reveal a pattern.\n" + "- Use neutral, factual language ('frustration appeared in...') " + "rather than judgemental ('the agents failed to...')." +) + + +def _tldr_summary_payload( + evaluation: CallImportEvaluation, +) -> Optional[EvaluationTldrSummary]: + """Return the cached TLDR (with ``is_stale`` set) or ``None``. + + ``CallImportEvaluation.tldr_summary`` is a ``JSON`` column so we + have to validate shape defensively -- a half-written or hand-edited + blob should not break the aggregate response. Returns ``None`` when + no cached summary exists. + """ + raw = evaluation.tldr_summary + if not isinstance(raw, dict): + return None + narrative = raw.get("narrative") + if not isinstance(narrative, str) or not narrative.strip(): + return None + patterns_raw = raw.get("patterns") + patterns = ( + [str(p) for p in patterns_raw if isinstance(p, str) and p.strip()] + if isinstance(patterns_raw, list) + else [] + ) + metric_insights_raw = raw.get("metric_insights") + metric_insights = ( + { + str(metric_id): str(insight).strip() + for metric_id, insight in metric_insights_raw.items() + if str(metric_id).strip() + and isinstance(insight, str) + and insight.strip() + } + if isinstance(metric_insights_raw, dict) + else {} + ) + generated_at_raw = raw.get("generated_at") + try: + generated_at = ( + datetime.fromisoformat(generated_at_raw) + if isinstance(generated_at_raw, str) + else evaluation.updated_at or datetime.now(timezone.utc) + ) + except ValueError: + generated_at = evaluation.updated_at or datetime.now(timezone.utc) + snapshot = raw.get("generated_at_completed_rows") + snapshot_int = int(snapshot) if isinstance(snapshot, (int, float)) else 0 + return EvaluationTldrSummary( + narrative=_clamp_prose_to_sentences(narrative.strip()), + patterns=patterns, + metric_insights=metric_insights, + generated_at=generated_at, + generated_at_completed_rows=snapshot_int, + provider=raw.get("provider") if isinstance(raw.get("provider"), str) else None, + model=raw.get("model") if isinstance(raw.get("model"), str) else None, + is_stale=evaluation.completed_rows > snapshot_int, + ) + + +def _sample_rationales_per_metric( + eval_rows: List[CallImportEvaluationRow], + *, + per_metric_cap: int = 3, + rationale_char_cap: int = 600, +) -> Dict[str, List[str]]: + """Collect up to ``per_metric_cap`` distinct rationales per metric. + + Distinctness is case- and whitespace-insensitive. We truncate each + rationale to ``rationale_char_cap`` so a few unusually verbose rows + can't dominate the prompt budget. Empty / non-string rationales are + skipped. + """ + out: Dict[str, List[str]] = {} + seen: Dict[str, set[str]] = {} + for row in eval_rows: + scores = row.metric_scores if isinstance(row.metric_scores, dict) else {} + for metric_id, entry in scores.items(): + if not isinstance(entry, dict): + continue + rationale = entry.get("rationale") + if not isinstance(rationale, str): + continue + text = rationale.strip() + if not text: + continue + bucket = out.setdefault(metric_id, []) + if len(bucket) >= per_metric_cap: + continue + key = " ".join(text.lower().split()) + seen_set = seen.setdefault(metric_id, set()) + if key in seen_set: + continue + seen_set.add(key) + bucket.append(text[:rationale_char_cap]) + return out + + +def _build_insights_messages( + evaluation: CallImportEvaluation, + aggregate: List[CallImportMetricAggregate], + rationale_samples: Dict[str, List[str]], + metric_meta: Dict[str, Metric], +) -> List[Dict[str, str]]: + """Render the user prompt fed to the LLM. + + The shape is plain markdown-ish text instead of JSON so the LLM can + skim it without us spending tokens on verbose schema delimiters. + Parent metrics surface their child metrics nested underneath so the + model sees the hierarchy and can talk about "X often co-occurred + with Y" rather than treating sub-labels as standalone metrics. + """ + name = evaluation.name or f"Run {str(evaluation.id)[:8]}" + lines: List[str] = [ + f"Evaluation: {name}", + ( + f"Rows: total={evaluation.total_rows} " + f"completed={evaluation.completed_rows} " + f"failed={evaluation.failed_rows}" + ), + "", + "## Per-metric aggregate", + ] + + # Group metrics by parent so the prompt mirrors the hierarchy. Any + # aggregate row whose ``metric_id`` is missing from ``metric_meta`` + # is rendered as a leaf at the top-level list (handles renamed / + # deleted parents). + children_by_parent: Dict[str, List[CallImportMetricAggregate]] = {} + top_level: List[CallImportMetricAggregate] = [] + for agg in aggregate: + meta = metric_meta.get(agg.metric_id) + parent_id = ( + str(meta.parent_metric_id) + if meta is not None and getattr(meta, "parent_metric_id", None) + else None + ) + if parent_id: + children_by_parent.setdefault(parent_id, []).append(agg) + else: + top_level.append(agg) + + def _format_metric_block(agg: CallImportMetricAggregate, indent: int) -> List[str]: + prefix = " " * indent + "- " + bits: List[str] = [f"{prefix}{agg.metric_name} [id={agg.metric_id}] (n={agg.count}"] + if agg.skipped_count: + bits.append(f", skipped={agg.skipped_count}") + if agg.error_count: + bits.append(f", errors={agg.error_count}") + bits.append(")") + meta = metric_meta.get(agg.metric_id) + description = (meta.description or "").strip() if meta else "" + if description: + bits.append(f" | definition={description[:500]}") + if agg.mean is not None: + mean_s = f"{agg.mean:.2f}" + stddev_s = f"{agg.stddev:.2f}" if agg.stddev is not None else "-" + bits.append(f" | mean={mean_s} stddev={stddev_s}") + if agg.min is not None and agg.max is not None: + bits.append(f" range=[{agg.min:.2f}, {agg.max:.2f}]") + if agg.value_counts: + total = sum(v.count for v in agg.value_counts) or 1 + top = agg.value_counts[:3] + shares = ", ".join( + f'"{v.label}"={v.count}/{total}' for v in top + ) + bits.append(f" | top={shares}") + result = ["".join(bits)] + rationales = rationale_samples.get(agg.metric_id, []) + for r in rationales: + result.append(" " * (indent + 1) + f"- rationale: {r}") + return result + + for agg in top_level: + lines.extend(_format_metric_block(agg, indent=0)) + meta = metric_meta.get(agg.metric_id) + children = children_by_parent.get(str(meta.id), []) if meta else [] + for child in children: + lines.extend(_format_metric_block(child, indent=1)) + + lines.append("") + top_level_ids = [agg.metric_id for agg in top_level] + if top_level_ids: + lines.append( + "metric_insights keys must exactly use these top-level metric ids: " + + ", ".join(top_level_ids) + ) + lines.append("") + lines.append( + "Write the JSON object as instructed. Do not include " + "preamble, code fences, or trailing commentary." + ) + + return [ + {"role": "system", "content": _INSIGHTS_SYSTEM_PROMPT}, + {"role": "user", "content": "\n".join(lines)}, + ] + + +def _parse_insights_response(text: str) -> EvaluationTldrSummary: + """Coerce the LLM response into ``narrative`` + ``patterns``. + + Matches the JSON-with-fallback pattern used by + ``app.api.v1.routes.metrics._parse_metric_generation_response``: try + ``json.loads`` first, then fall back to regex extraction of the + first ``{...}`` block. Raises ``HTTPException`` with a 502 when the + response can't be parsed at all. + """ + cleaned = (text or "").strip() + if not cleaned: + raise HTTPException( + status_code=502, detail="LLM returned an empty insights response" + ) + try: + parsed = json.loads(cleaned) + except json.JSONDecodeError: + import re + + match = re.search(r"\{.*\}", cleaned, re.DOTALL) + if not match: + raise HTTPException( + status_code=502, + detail="Could not parse LLM insights response as JSON", + ) + try: + parsed = json.loads(match.group(0)) + except json.JSONDecodeError as e: + raise HTTPException( + status_code=502, + detail=f"Could not parse LLM insights response: {e}", + ) + + if not isinstance(parsed, dict): + raise HTTPException( + status_code=502, detail="LLM insights JSON was not an object" + ) + + narrative = parsed.get("narrative") + if not isinstance(narrative, str) or not narrative.strip(): + raise HTTPException( + status_code=502, + detail="LLM insights JSON missing 'narrative' string", + ) + + patterns_raw = parsed.get("patterns") + if patterns_raw is None: + patterns: List[str] = [] + elif isinstance(patterns_raw, list): + patterns = [ + str(p).strip() + for p in patterns_raw + if isinstance(p, str) and p.strip() + ] + else: + raise HTTPException( + status_code=502, + detail="LLM insights JSON 'patterns' must be a list of strings", + ) + metric_insights_raw = parsed.get("metric_insights") + if metric_insights_raw is None: + metric_insights: Dict[str, str] = {} + elif isinstance(metric_insights_raw, dict): + metric_insights = { + str(metric_id): str(insight).strip() + for metric_id, insight in metric_insights_raw.items() + if str(metric_id).strip() + and isinstance(insight, str) + and insight.strip() + } + else: + raise HTTPException( + status_code=502, + detail="LLM insights JSON 'metric_insights' must be an object", + ) + + return EvaluationTldrSummary( + narrative=_clamp_prose_to_sentences(narrative.strip()), + patterns=patterns, + metric_insights=metric_insights, + generated_at=datetime.now(timezone.utc), + generated_at_completed_rows=0, # filled in by caller + is_stale=False, + ) + + +def _generate_and_persist_tldr_summary( + db: Session, + evaluation: CallImportEvaluation, + *, + organization_id: UUID, + provider: Optional[str] = None, + model: Optional[str] = None, +) -> EvaluationTldrSummary: + """LLM TLDR generation used by the imports-queue Celery worker.""" + eval_id = evaluation.id + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + + pairs = load_evaluation_row_pairs(db, eval_id) + eval_rows = [eval_row for eval_row, _ in pairs] + aggregate = _compute_metric_aggregates(db, evaluation, eval_rows) + if not aggregate: + raise HTTPException( + status_code=400, + detail=( + "No metric data yet. Wait for at least one row to " + "finish scoring before generating a summary." + ), + ) + + metric_ids: List[UUID] = [] + for agg in aggregate: + try: + metric_ids.append(UUID(agg.metric_id)) + except (TypeError, ValueError): + continue + metrics = _metrics_for_ids(db, organization_id, metric_ids) + metric_meta: Dict[str, Metric] = {str(m.id): m for m in metrics} + + rationale_samples = _sample_rationales_per_metric(eval_rows) + messages = _build_insights_messages( + evaluation, aggregate, rationale_samples, metric_meta + ) + + from app.services.ai.llm_resolver import get_llm_provider_and_model + from app.services.ai.llm_service import llm_service + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, provider, model + ) + + try: + llm_result = llm_service.generate_response( + messages=messages, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + temperature=0.4, + max_tokens=1400, + ) + except Exception as e: + logger.error(f"[CallImportInsights] LLM call failed: {e}") + raise HTTPException( + status_code=502, detail=f"LLM call failed: {e}" + ) from e + + summary = _parse_insights_response(llm_result.get("text", "")) + total = int(evaluation.total_rows or 0) + ui_completed = min(int(evaluation.completed_rows or 0), total) if total else int( + evaluation.completed_rows or 0 + ) + summary.generated_at_completed_rows = ui_completed + summary.provider = provider_enum.value + summary.model = model_str + summary.is_stale = False + + evaluation.tldr_summary = { + "narrative": summary.narrative, + "patterns": summary.patterns, + "metric_insights": summary.metric_insights, + "generated_at": summary.generated_at.isoformat(), + "generated_at_completed_rows": summary.generated_at_completed_rows, + "provider": summary.provider, + "model": summary.model, + } + flag_modified(evaluation, "tldr_summary") + db.commit() + db.refresh(evaluation) + return summary + + +@router.get( + "/{eval_id}/insights", + response_model=Optional[EvaluationTldrSummary], + operation_id="getCallImportEvaluationInsights", +) +async def get_call_import_evaluation_insights( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> Optional[EvaluationTldrSummary]: + """Return the cached TLDR (or ``null``) without contacting the LLM. + + Used by the Visualizations tab on first paint so the empty-state + CTA can show up before the user opts into generation. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + return _tldr_summary_payload(evaluation) + + +@router.post( + "/{eval_id}/insights", + response_model=EvaluationTldrSummary, + operation_id="generateCallImportEvaluationInsights", +) +async def generate_call_import_evaluation_insights( + call_import_id: UUID, + eval_id: UUID, + body: EvaluationInsightsRequest = Body(default_factory=EvaluationInsightsRequest), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationTldrSummary: + """Generate (or return-cached) the LLM TLDR for an evaluation run. + + Behavior: + + * ``body.regenerate=False`` and a cached summary at the current + ``completed_rows`` watermark exists -> return it as-is. + * ``body.regenerate=False`` and a stale cached summary exists + (``generated_at_completed_rows < completed_rows``) -> return it + with ``is_stale=True``; the UI prompts the user to regenerate. + * Otherwise -> resolve provider+model (auto-detect when omitted), + call the LLM, persist the new summary, return it. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + if not body.regenerate: + cached = _tldr_summary_payload(evaluation) + if cached is not None: + return cached + + # Run the TLDR LLM on the imports worker (not the default worker or API). + from app.workers.tasks.generate_evaluation_tldr_insights import ( + generate_evaluation_tldr_insights_task, + ) + + try: + task_result = generate_evaluation_tldr_insights_task.apply_async( + kwargs={ + "evaluation_id": str(eval_id), + "call_import_id": str(call_import_id), + "organization_id": str(organization_id), + "provider": body.provider, + "model": body.model, + }, + ).get(timeout=25 * 60) + except Exception as exc: + logger.error( + "[CallImportInsights] TLDR task failed for evaluation {}: {}", + eval_id, + exc, + ) + raise HTTPException( + status_code=502, + detail=f"Summary generation failed: {exc}", + ) from exc + + if isinstance(task_result, dict) and task_result.get("error"): + status_code = int(task_result.get("status_code") or 502) + raise HTTPException( + status_code=status_code, + detail=str(task_result["error"]), + ) + + summary = EvaluationTldrSummary.model_validate(task_result) + db.refresh(evaluation) + + from app.services.ai.llm_resolver import get_llm_provider_and_model + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, body.provider, body.model, body.credential_id + ) + + _enqueue_user_insights_job( + evaluation, + provider=summary.provider or provider_enum.value, + model=summary.model or model_str, + force=body.regenerate, + max_llm_calls=body.max_llm_calls, + db=db, + ) + + return summary + + +def _user_insights_payload( + evaluation: CallImportEvaluation, +) -> Optional[EvaluationUserInsightsState]: + raw = getattr(evaluation, "user_insights", None) + if raw is None: + return None + return user_insights_state_from_raw( + raw, + completed_rows=evaluation.completed_rows, + ) + + +def _selected_generated_user_insights( + state: Optional[EvaluationUserInsightsState], + report_config: dict[str, Any], +) -> list[dict[str, Any]]: + """Filter and order generated insights for PDF section 03.""" + if state is None or state.status != "completed" or not state.insights: + return [] + + selected_ids = report_config.get("user_insight_ids") + if isinstance(selected_ids, list) and selected_ids: + allowed = {str(item) for item in selected_ids if item} + items = [item for item in state.insights if item.id in allowed] + else: + items = list(state.insights) + + order_raw = report_config.get("order") + order_ids: list[str] = [] + if isinstance(order_raw, dict): + user_order = order_raw.get("user_insights") + if isinstance(user_order, list): + order_ids = [str(item) for item in user_order if item] + + if order_ids: + by_id = {item.id: item for item in items} + ordered = [by_id[iid] for iid in order_ids if iid in by_id] + seen = set(order_ids) + ordered.extend(item for item in items if item.id not in seen) + items = ordered + + return [item.model_dump(mode="json") for item in items] + + +def _enqueue_user_insights_job( + evaluation: CallImportEvaluation, + *, + provider: Optional[str] = None, + model: Optional[str] = None, + force: bool = False, + max_llm_calls: Optional[int] = None, + db: Optional[Session] = None, +) -> None: + """Enqueue background user-insights generation unless already running.""" + current = _user_insights_payload(evaluation) + if current is not None and current.status == "running" and not force: + return + + llm_budget = normalize_max_llm_calls(max_llm_calls) + + completed_count = ( + _count_completed_eval_rows(db, evaluation.id) + if db is not None + else evaluation.completed_rows + ) + total_calls = total_llm_calls_for_rows(completed_count, max_llm_calls=llm_budget) + evaluation.user_insights = { + "status": "running", + "insights": ( + (evaluation.user_insights or {}).get("insights", []) + if isinstance(evaluation.user_insights, dict) + else [] + ), + "generated_at": datetime.now(timezone.utc).isoformat(), + "generated_at_completed_rows": evaluation.completed_rows, + "progress": {"completed_llm_calls": 0, "total_llm_calls": total_calls}, + "provider": provider, + "model": model, + "max_llm_calls": llm_budget, + "llm_calls_used": 0, + "error_message": None, + } + if db is not None: + flag_modified(evaluation, "user_insights") + db.commit() + + from app.workers.tasks.generate_evaluation_user_insights import ( + generate_evaluation_user_insights_task, + ) + + generate_evaluation_user_insights_task.delay( + str(evaluation.id), + provider=provider, + model=model, + max_llm_calls=llm_budget, + ) + + +@router.get( + "/{eval_id}/user-insights", + response_model=Optional[EvaluationUserInsightsState], + operation_id="getCallImportEvaluationUserInsights", +) +async def get_call_import_evaluation_user_insights( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> Optional[EvaluationUserInsightsState]: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + return _user_insights_payload(evaluation) + + +@router.post( + "/{eval_id}/user-insights", + response_model=EvaluationUserInsightsState, + operation_id="generateCallImportEvaluationUserInsights", +) +async def generate_call_import_evaluation_user_insights( + call_import_id: UUID, + eval_id: UUID, + body: EvaluationUserInsightsRequest = Body( + default_factory=EvaluationUserInsightsRequest + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationUserInsightsState: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + if not body.regenerate and not body.force: + cached = _user_insights_payload(evaluation) + if cached is not None and cached.status in {"running", "completed"}: + return cached + + eval_rows = _load_eval_rows(db, eval_id) + if not any(row.status == "completed" for row in eval_rows): + raise HTTPException( + status_code=400, + detail=( + "No completed rows yet. Wait for at least one row to " + "finish scoring before generating user insights." + ), + ) + + from app.services.ai.llm_resolver import get_llm_provider_and_model + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, body.provider, body.model, body.credential_id + ) + + _enqueue_user_insights_job( + evaluation, + provider=provider_enum.value, + model=model_str, + force=body.force or body.regenerate, + max_llm_calls=body.max_llm_calls, + db=db, + ) + + db.refresh(evaluation) + return _user_insights_payload(evaluation) or EvaluationUserInsightsState( + status="running" + ) + + +def _metric_clusters_payload( + evaluation: CallImportEvaluation, +) -> Optional[EvaluationMetricClustersState]: + raw = getattr(evaluation, "metric_clusters", None) + if raw is None: + return None + return metric_clusters_state_from_raw( + raw, + completed_rows=evaluation.completed_rows, + ) + + +def _selected_metric_clusters_for_pdf( + state: Optional[EvaluationMetricClustersState], + report_config: dict[str, Any], +) -> dict[str, Any]: + if state is None or state.status != "completed": + return {} + sections = report_config.get("sections") + if isinstance(sections, dict) and sections.get("failure_diagnostics") is False: + return {} + payload: dict[str, Any] = { + "groups": [g.model_dump(mode="json") for g in state.groups], + "discovered_problems": [ + d.model_dump(mode="json") for d in state.discovered_problems + ], + } + if state.rca_summary is not None: + payload["rca_summary"] = state.rca_summary.model_dump(mode="json") + return payload + + +def _prompt_improvements_payload( + evaluation: CallImportEvaluation, +) -> Optional[EvaluationPromptImprovementsState]: + from app.services.call_import_prompt_improvements import ( + prompt_improvements_state_from_raw, + ) + + raw = getattr(evaluation, "prompt_improvements", None) + if raw is None: + return None + return prompt_improvements_state_from_raw( + raw, + completed_rows=evaluation.completed_rows, + ) + + +def _selected_prompt_improvements_for_pdf( + state: Optional[EvaluationPromptImprovementsState], + report_config: dict[str, Any], +) -> dict[str, Any]: + if state is None or state.status != "completed": + return {} + sections = report_config.get("sections") + if isinstance(sections, dict) and sections.get("prompt_improvements") is False: + return {} + return { + "imported_agent_id": state.imported_agent_id, + "imported_agent_name": state.imported_agent_name, + "overview": state.overview, + "suggestions": [s.model_dump(mode="json") for s in state.suggestions], + } + + +def _enqueue_prompt_improvements_job( + evaluation: CallImportEvaluation, + *, + imported_agent_id: UUID, + imported_agent_name: str, + provider: Optional[str] = None, + model: Optional[str] = None, + credential_id: Optional[UUID] = None, + force: bool = False, + db: Optional[Session] = None, +) -> None: + current = _prompt_improvements_payload(evaluation) + if current is not None and current.status == "running" and not force: + return + + evaluation.prompt_improvements = { + "status": "running", + "imported_agent_id": str(imported_agent_id), + "imported_agent_name": imported_agent_name, + "suggestions": [], + "generated_at": datetime.now(timezone.utc).isoformat(), + "generated_at_completed_rows": evaluation.completed_rows, + "provider": provider, + "model": model, + "error_message": None, + } + if db is not None: + flag_modified(evaluation, "prompt_improvements") + db.commit() + + from app.workers.tasks.generate_evaluation_prompt_improvements import ( + generate_evaluation_prompt_improvements_task, + ) + + async_result = generate_evaluation_prompt_improvements_task.apply_async( + kwargs={ + "evaluation_id": str(evaluation.id), + "imported_agent_id": str(imported_agent_id), + "provider": provider, + "model": model, + "credential_id": str(credential_id) if credential_id else None, + }, + queue="imports", + ) + if db is not None and isinstance(evaluation.prompt_improvements, dict): + evaluation.prompt_improvements["celery_task_id"] = async_result.id + flag_modified(evaluation, "prompt_improvements") + db.commit() + + +def _load_eval_rows(db: Session, evaluation_id: UUID) -> List[CallImportEvaluationRow]: + from app.db_sharding.eval_rows import load_evaluation_rows_for_run + + return load_evaluation_rows_for_run(db, evaluation_id) + + +def _count_completed_eval_rows(db: Session, evaluation_id: UUID) -> int: + from app.db_sharding.eval_rows import count_evaluation_rows_for_run + + return count_evaluation_rows_for_run( + db, evaluation_id, statuses=["completed"] + ) + + +def _completed_row_pairs_for_evaluation( + db: Session, + evaluation_id: UUID, +) -> List[Tuple[CallImportEvaluationRow, CallImportRow]]: + from app.db_sharding.scatter_gather import load_evaluation_row_pairs + + row_pairs = load_evaluation_row_pairs(db, evaluation_id) + return [ + (eval_row, source_row) + for eval_row, source_row in row_pairs + if eval_row.status == "completed" + ] + + +def _resolve_metric_cluster_row_selection( + db: Session, + evaluation: CallImportEvaluation, + eval_rows: List[CallImportEvaluationRow], + evaluation_row_ids: Optional[List[UUID]], + *, + row_limit: Optional[int] = None, + policies: Optional[Dict[str, MetricFailurePolicy]] = None, +) -> Tuple[List[Tuple[CallImportEvaluationRow, CallImportRow]], List[str]]: + """Return filtered completed row pairs and the selected row id strings.""" + completed_pairs = _completed_row_pairs_for_evaluation(db, evaluation.id) + metrics = _metrics_for_clustering(db, evaluation, eval_rows) + if policies is None: + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + parent_ids = [ + m.id + for m in metrics + if getattr(m, "selection_mode", None) + and not getattr(m, "parent_metric_id", None) + ] + child_names_by_parent = _child_names_by_parent( + db, evaluation.organization_id, parent_ids + ) + policies, _ = effective_policies( + evaluation, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + eligible = list_eligible_cluster_rows( + evaluation, completed_pairs, metrics, policies + ) + eligible_ordered_ids = [str(item["evaluation_row_id"]) for item in eligible] + eligible_id_set = set(eligible_ordered_ids) + + if evaluation_row_ids is None and row_limit is not None: + selected_ids = eligible_ordered_ids[:row_limit] + filtered = filter_completed_row_pairs( + completed_pairs, + [UUID(rid) for rid in selected_ids], + ) + return filtered, selected_ids + + if evaluation_row_ids is None: + selected_ids = eligible_ordered_ids + filtered = filter_completed_row_pairs( + completed_pairs, + [UUID(rid) for rid in selected_ids], + ) + return filtered, selected_ids + + requested = {str(rid) for rid in evaluation_row_ids} + completed_id_set = {str(eval_row.id) for eval_row, _ in completed_pairs} + unknown = sorted(requested - completed_id_set) + if unknown: + raise HTTPException( + status_code=400, + detail=( + "One or more evaluation_row_ids are missing or not completed: " + + ", ".join(unknown[:5]) + + ("…" if len(unknown) > 5 else "") + ), + ) + not_eligible = sorted(requested - eligible_id_set) + if not_eligible: + raise HTTPException( + status_code=400, + detail=( + "Each selected row must have at least one flagged quality metric. " + "Ineligible row(s): " + + ", ".join(not_eligible[:5]) + + ("…" if len(not_eligible) > 5 else "") + ), + ) + selected_ids = sorted(requested) + filtered = filter_completed_row_pairs(completed_pairs, evaluation_row_ids) + return filtered, selected_ids + + +def _enqueue_metric_clusters_job( + evaluation: CallImportEvaluation, + *, + provider: Optional[str] = None, + model: Optional[str] = None, + credential_id: Optional[UUID] = None, + force: bool = False, + max_llm_calls: Optional[int] = None, + evaluation_row_ids: Optional[List[UUID]] = None, + selected_evaluation_row_ids: Optional[List[str]] = None, + failure_policies: Optional[Dict[str, MetricFailurePolicy]] = None, + db: Optional[Session] = None, +) -> None: + current = _metric_clusters_payload(evaluation) + if current is not None and current.status == "running" and not force: + return + + llm_budget = normalize_max_llm_calls(max_llm_calls) + total_calls = 1 + row_ids_for_task: Optional[List[str]] = None + if db is not None: + eval_rows = _load_eval_rows(db, evaluation.id) + if selected_evaluation_row_ids is None: + _, selected_evaluation_row_ids = _resolve_metric_cluster_row_selection( + db, + evaluation, + eval_rows, + evaluation_row_ids, + ) + completed_pairs = filter_completed_row_pairs( + _completed_row_pairs_for_evaluation(db, evaluation.id), + [UUID(rid) for rid in selected_evaluation_row_ids], + ) + metrics = _metrics_for_clustering(db, evaluation, eval_rows) + policies_for_estimate = failure_policies + if policies_for_estimate is None: + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + parent_ids = [ + m.id + for m in metrics + if getattr(m, "selection_mode", None) + and not getattr(m, "parent_metric_id", None) + ] + child_names_by_parent = _child_names_by_parent( + db, evaluation.organization_id, parent_ids + ) + policies_for_estimate, _ = effective_policies( + evaluation, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + _, total_calls = estimate_metric_clusters_llm_calls( + evaluation, + metrics, + completed_pairs, + policies_for_estimate, + max_llm_calls=llm_budget, + ) + row_ids_for_task = list(selected_evaluation_row_ids) + + prior_raw = ( + evaluation.metric_clusters + if isinstance(evaluation.metric_clusters, dict) + else {} + ) + policy_blob: Dict[str, Any] = {} + if failure_policies: + policy_blob = failure_policies_to_db(failure_policies, source="user") + + evaluation.metric_clusters = { + "status": "running", + "groups": prior_raw.get("groups", []) if isinstance(prior_raw, dict) else [], + "discovered_problems": ( + prior_raw.get("discovered_problems", []) + if isinstance(prior_raw, dict) + else [] + ), + "generated_at": datetime.now(timezone.utc).isoformat(), + "generated_at_completed_rows": evaluation.completed_rows, + "progress": {"completed_llm_calls": 0, "total_llm_calls": total_calls}, + "provider": provider, + "model": model, + "max_llm_calls": llm_budget, + "llm_calls_used": 0, + "error_message": None, + "selected_evaluation_row_ids": selected_evaluation_row_ids or [], + **policy_blob, + } + if db is not None: + flag_modified(evaluation, "metric_clusters") + db.commit() + + from app.workers.tasks.generate_evaluation_metric_clusters import ( + generate_evaluation_metric_clusters_task, + ) + + async_result = generate_evaluation_metric_clusters_task.apply_async( + kwargs={ + "evaluation_id": str(evaluation.id), + "provider": provider, + "model": model, + "credential_id": str(credential_id) if credential_id else None, + "max_llm_calls": llm_budget, + "evaluation_row_ids": row_ids_for_task, + }, + queue="imports", + ) + if db is not None and isinstance(evaluation.metric_clusters, dict): + evaluation.metric_clusters["celery_task_id"] = async_result.id + flag_modified(evaluation, "metric_clusters") + db.commit() + + +def _revoke_metric_clusters_task(evaluation: CallImportEvaluation) -> None: + """Best-effort SIGTERM revoke of the in-flight clustering Celery task.""" + raw = evaluation.metric_clusters + if not isinstance(raw, dict): + return + task_id = str(raw.get("celery_task_id") or "").strip() + if not task_id: + return + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke(task_id, terminate=True, signal="SIGTERM") + logger.info( + "Revoked metric-clusters task {} for evaluation {}", + task_id, + evaluation.id, + ) + except Exception as exc: # noqa: BLE001 + logger.warning( + "Failed to revoke metric-clusters task {} for evaluation {}: {}", + task_id, + evaluation.id, + exc, + ) + + +def _apply_metric_clusters_cancel(evaluation: CallImportEvaluation) -> bool: + """Mark clustering as cancelled and revoke the worker task. + + Returns True if a running job was cancelled, False if already terminal. + """ + raw = evaluation.metric_clusters + if not isinstance(raw, dict): + return False + if (raw.get("status") or "").lower() != "running": + return False + + _revoke_metric_clusters_task(evaluation) + progress = raw.get("progress") if isinstance(raw.get("progress"), dict) else {} + evaluation.metric_clusters = { + **raw, + "status": "cancelled", + "error_message": METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + "progress": progress, + "celery_task_id": None, + } + return True + + +@router.get( + "/{eval_id}/metric-clusters/failure-policies", + response_model=MetricFailurePoliciesResponse, + operation_id="getCallImportEvaluationMetricClusterFailurePolicies", +) +async def get_call_import_evaluation_metric_cluster_failure_policies( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> MetricFailurePoliciesResponse: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + eval_rows = _load_eval_rows(db, eval_id) + metrics, aggregates, policies, source, child_names_by_parent = _clustering_context( + db, evaluation, eval_rows + ) + previews = build_failure_policy_previews( + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + effective=policies, + ) + updated_at = None + raw_mc = evaluation.metric_clusters + if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): + try: + updated_at = datetime.fromisoformat( + str(raw_mc["failure_policies_updated_at"]) + ) + except ValueError: + updated_at = None + return MetricFailurePoliciesResponse( + previews=previews, + policies=policies, + source=source, + updated_at=updated_at, + ) + + +@router.put( + "/{eval_id}/metric-clusters/failure-policies", + response_model=MetricFailurePoliciesResponse, + operation_id="saveCallImportEvaluationMetricClusterFailurePolicies", +) +async def save_call_import_evaluation_metric_cluster_failure_policies( + call_import_id: UUID, + eval_id: UUID, + body: MetricFailurePoliciesSaveRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> MetricFailurePoliciesResponse: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + eval_rows = _load_eval_rows(db, eval_id) + metrics, aggregates, _existing, _source, child_names_by_parent = _clustering_context( + db, evaluation, eval_rows + ) + try: + validate_failure_policies_for_metrics(body.policies, metrics) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + prior = ( + evaluation.metric_clusters + if isinstance(evaluation.metric_clusters, dict) + else {} + ) + evaluation.metric_clusters = merge_failure_policies_into_raw( + prior, + body.policies, + source="user", + ) + flag_modified(evaluation, "metric_clusters") + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(evaluation) + + policies, source = policies_from_evaluation_raw(evaluation.metric_clusters) + if source != "user": + source = "user" + previews = build_failure_policy_previews( + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + effective=policies, + ) + updated_at = None + raw_mc = evaluation.metric_clusters + if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): + try: + updated_at = datetime.fromisoformat( + str(raw_mc["failure_policies_updated_at"]) + ) + except ValueError: + updated_at = None + return MetricFailurePoliciesResponse( + previews=previews, + policies=policies, + source="user", + updated_at=updated_at, + ) + + +@router.get( + "/{eval_id}/metric-clusters/eligible-rows", + response_model=MetricClusterEligibleRowsResponse, + operation_id="listCallImportEvaluationMetricClusterEligibleRows", +) +async def list_call_import_evaluation_metric_cluster_eligible_rows( + call_import_id: UUID, + eval_id: UUID, + limit: Optional[int] = Query(default=None, ge=1), + count_only: bool = Query(default=False), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> MetricClusterEligibleRowsResponse: + """Completed rows that have at least one flagged quality metric.""" + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + eval_rows = _load_eval_rows(db, eval_id) + completed_pairs = _completed_row_pairs_for_evaluation(db, eval_id) + metrics, _aggregates, policies, _source, _child_map = _clustering_context( + db, evaluation, eval_rows + ) + all_eligible = list_eligible_cluster_rows( + evaluation, completed_pairs, metrics, policies + ) + total = len(all_eligible) + if count_only: + return MetricClusterEligibleRowsResponse(items=[], total=total) + raw_items = all_eligible if limit is None else all_eligible[:limit] + items = [MetricClusterEligibleRow.model_validate(item) for item in raw_items] + return MetricClusterEligibleRowsResponse(items=items, total=total) + + +@router.get( + "/{eval_id}/metric-clusters", + response_model=Optional[EvaluationMetricClustersState], + operation_id="getCallImportEvaluationMetricClusters", +) +async def get_call_import_evaluation_metric_clusters( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> Optional[EvaluationMetricClustersState]: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + return _metric_clusters_payload(evaluation) + + +@router.post( + "/{eval_id}/metric-clusters", + response_model=EvaluationMetricClustersState, + operation_id="generateCallImportEvaluationMetricClusters", +) +async def generate_call_import_evaluation_metric_clusters( + call_import_id: UUID, + eval_id: UUID, + body: EvaluationMetricClustersRequest = Body( + default_factory=EvaluationMetricClustersRequest + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationMetricClustersState: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + if not body.regenerate and not body.force: + cached = _metric_clusters_payload(evaluation) + if cached is not None and cached.status in {"running", "completed"}: + return cached + + eval_rows = _load_eval_rows(db, eval_id) + if not any(row.status == "completed" for row in eval_rows): + raise HTTPException( + status_code=400, + detail=( + "No completed rows yet. Wait for at least one row to " + "finish scoring before generating metric clusters." + ), + ) + + if body.evaluation_row_ids and body.row_limit is not None: + raise HTTPException( + status_code=400, + detail="Specify either evaluation_row_ids or row_limit, not both.", + ) + + if body.evaluation_row_ids: + completed_pairs = _completed_row_pairs_for_evaluation(db, evaluation.id) + completed_id_set = {str(eval_row.id) for eval_row, _ in completed_pairs} + requested = {str(rid) for rid in body.evaluation_row_ids} + unknown = sorted(requested - completed_id_set) + if unknown: + raise HTTPException( + status_code=400, + detail=( + "One or more evaluation_row_ids are missing or not completed: " + + ", ".join(unknown[:5]) + + ("…" if len(unknown) > 5 else "") + ), + ) + + from app.services.ai.llm_resolver import get_llm_provider_and_model + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, body.provider, body.model, body.credential_id + ) + + metrics, aggregates, _inferred, _source, child_names_by_parent = _clustering_context( + db, evaluation, eval_rows + ) + merged_policies = merge_clustering_policies( + body.failure_policies, + evaluation, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + try: + validate_failure_policies_for_metrics( + body.failure_policies or merged_policies, metrics + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + if not has_clusterable_metrics(metrics, merged_policies, eval_rows): + raise HTTPException( + status_code=400, + detail=( + "No calls match any failure policy. Select failure values on " + "metrics that have matching rows, or leave metrics with no " + "failures unchecked — they are skipped automatically." + ), + ) + + filtered_pairs, selected_row_ids = _resolve_metric_cluster_row_selection( + db, + evaluation, + eval_rows, + body.evaluation_row_ids, + row_limit=body.row_limit, + policies=merged_policies, + ) + if not selected_row_ids: + raise HTTPException( + status_code=400, + detail=( + "No eligible rows to cluster. Select completed calls that match " + "at least one configured failure policy." + ), + ) + if not filtered_pairs: + raise HTTPException( + status_code=400, + detail="No completed rows match the selected evaluation_row_ids.", + ) + + _enqueue_metric_clusters_job( + evaluation, + provider=provider_enum.value, + model=model_str, + credential_id=body.credential_id, + force=body.force or body.regenerate, + max_llm_calls=body.max_llm_calls, + evaluation_row_ids=body.evaluation_row_ids, + selected_evaluation_row_ids=selected_row_ids, + failure_policies=merged_policies, + db=db, + ) + + db.refresh(evaluation) + return _metric_clusters_payload(evaluation) or EvaluationMetricClustersState( + status="running" + ) + + +@router.post( + "/{eval_id}/metric-clusters/cancel", + response_model=EvaluationMetricClustersState, + operation_id="cancelCallImportEvaluationMetricClusters", +) +async def cancel_call_import_evaluation_metric_clusters( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationMetricClustersState: + """Abort in-flight failure-diagnostics clustering. + + Idempotent: if clustering is not ``running``, returns the current state + unchanged. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + _apply_metric_clusters_cancel(evaluation) + flag_modified(evaluation, "metric_clusters") + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(evaluation) + + return _metric_clusters_payload(evaluation) or EvaluationMetricClustersState( + status="idle" + ) + + +@router.get( + "/{eval_id}/prompt-improvements", + response_model=Optional[EvaluationPromptImprovementsState], + operation_id="getCallImportEvaluationPromptImprovements", +) +async def get_call_import_evaluation_prompt_improvements( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> Optional[EvaluationPromptImprovementsState]: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + return _prompt_improvements_payload(evaluation) + + +@router.post( + "/{eval_id}/prompt-improvements", + response_model=EvaluationPromptImprovementsState, + operation_id="generateCallImportEvaluationPromptImprovements", +) +async def generate_call_import_evaluation_prompt_improvements( + call_import_id: UUID, + eval_id: UUID, + body: EvaluationPromptImprovementsRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> EvaluationPromptImprovementsState: + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + clusters = _metric_clusters_payload(evaluation) + if clusters is None or clusters.status != "completed": + raise HTTPException( + status_code=400, + detail=( + "Metric clusters must be completed before generating prompt " + "improvements. Run failure diagnostics first." + ), + ) + + from app.services.call_import_prompt_improvements import is_imported_agent + from app.services.ai.llm_resolver import get_llm_provider_and_model + + imported_agent = ( + db.query(PromptPartial) + .filter( + PromptPartial.id == body.imported_agent_id, + PromptPartial.organization_id == organization_id, + PromptPartial.workspace_id == workspace_id, + ) + .first() + ) + if imported_agent is None or not is_imported_agent(imported_agent): + raise HTTPException( + status_code=404, + detail="Imported agent not found in the active workspace", + ) + + if not body.regenerate and not body.force: + cached = _prompt_improvements_payload(evaluation) + if ( + cached is not None + and cached.status in {"running", "completed"} + and cached.imported_agent_id == str(body.imported_agent_id) + ): + return cached + + provider_enum, model_str = get_llm_provider_and_model( + organization_id, db, body.provider, body.model, body.credential_id + ) + + _enqueue_prompt_improvements_job( + evaluation, + imported_agent_id=body.imported_agent_id, + imported_agent_name=imported_agent.name, + provider=provider_enum.value, + model=model_str, + credential_id=body.credential_id, + force=body.force or body.regenerate, + db=db, + ) + + db.refresh(evaluation) + return _prompt_improvements_payload(evaluation) or EvaluationPromptImprovementsState( + status="running", + imported_agent_id=str(body.imported_agent_id), + imported_agent_name=imported_agent.name, + ) + + +# --------------------------------------------------------------------------- +# Flow chart: turns per-row LLM-inferred ``sequence`` arrays into a +# directed graph of (label -> label) transitions across the whole run. +# Powers the aggregate Sankey-style React Flow chart on the evaluation +# overview; per-call flow charts are built client-side from the same +# ``sequence`` field on a single row's metric_scores entry. +# --------------------------------------------------------------------------- + + +_FLOW_TERMINAL_THRESHOLD = 0.2 # Mark as terminal when >=20% of sequences end here. +_FLOW_START_NODE_ID = "__START__" +_DISCOVERED_NODE_PREFIX = "disc:" + + +def _slug_label(value: Any) -> str: + """Lowercase + whitespace-collapse + underscore-join. + + Used everywhere we need a stable key for a metric/label name — + matching the same convention the worker uses when emitting + ``sequence`` entries and discovered keys. + """ + if value is None: + return "" + return "_".join(str(value).strip().lower().split()) + + +def _resolve_alias(alias_map: Dict[str, str], key: str) -> str: + """Walk the alias map until we hit a slug that doesn't redirect. + + The merge endpoint stores ``from_slug -> to_slug`` pairs. The delete + endpoint stores ``from_slug -> ""`` (empty string sentinel) to mark + a slug as tombstoned. Chains can accumulate when the user merges + A→B and later merges B→C; this helper collapses them so callers + always land on the final canonical slug. + + Returns: + * the canonical slug if it still resolves to a real label, + * an empty string if the slug has been tombstoned (callers MUST + treat an empty result as "drop this entry entirely"), + * the input ``key`` if it isn't aliased. + + Cycles are guarded by a hard step limit since the alias map is + user-driven. + """ + if not key: + return "" + if not alias_map: + return key + current = key + seen: set[str] = set() + for _ in range(16): + if current in seen: + return current + seen.add(current) + if current not in alias_map: + return current + nxt = alias_map[current] + if nxt == current: + return current + if nxt == "": + # Deletion sentinel — the user has explicitly retired this + # slug. Propagate the empty string up so callers drop it. + return "" + current = nxt + return current + + +# Reserved JSON key under which the worker stores top-level metric +# discoveries on each row's ``metric_scores`` dict. Mirrors the constant +# in ``app/workers/tasks/helpers/llm_evaluation.py`` — kept local here to +# avoid a worker import cycle from the routes module. +DISCOVERED_METRICS_KEY = "__discovered_metrics__" + +# Allowed values for an LLM-suggested top-level metric type. Kept in +# sync with ``DiscoveredMetricSuggestedType`` in +# ``app/models/schemas.py``. +_DISCOVERED_METRIC_TYPES = ("boolean", "rating", "category") + + +def normalize_scores_with_aliases( + metric_scores: Dict[str, Any], + evaluation: CallImportEvaluation, + db: Session, + organization_id: UUID, +) -> Dict[str, Any]: + """Rewrite per-row ``metric_scores`` to honor merges + promotions. + + Called by the worker right after ``evaluate_with_llm`` returns so + every row that finishes AFTER a user has merged or promoted a + discovered label persists data already reflecting that decision. + Without this hook, a worker holding a stale prompt could re-emit a + ``from_key`` slug long after the user merged it away. + + For every parent entry (``selection_mode != null`` and a + ``discovered_labels`` / ``sequence`` field) we: + + * resolve discovered slugs through the evaluation's + ``discovered_label_aliases`` map (transitively), + * drop any discovered_labels entry whose canonical slug now + matches a real promoted child of the parent (merging them out + of the panel for free), and + * collapse adjacent duplicate sequence entries that result. + + Returns ``metric_scores`` (mutated in place) for chaining. + """ + if not isinstance(metric_scores, dict): + return metric_scores + + aliases_top = ( + evaluation.discovered_label_aliases + if isinstance(evaluation.discovered_label_aliases, dict) + else {} + ) + + # Identify the parent entries inside metric_scores. They're the + # dicts that carry a ``selection_mode`` key (set by the LLM + # hierarchy parser) and either a ``sequence`` or a + # ``discovered_labels`` list. + for key, entry in list(metric_scores.items()): + if not isinstance(entry, dict): + continue + if entry.get("type") != "category" and not entry.get("selection_mode"): + continue + try: + parent_uuid = UUID(str(key)) + except (TypeError, ValueError): + continue + + alias_map = {} + sub = aliases_top.get(str(parent_uuid)) + if isinstance(sub, dict): + alias_map = { + str(k): str(v) + for k, v in sub.items() + if isinstance(k, str) and isinstance(v, str) + } + promoted = _promoted_child_slugs(db, parent_uuid, organization_id) + + # Rewrite discovered_labels: alias-resolve keys, drop duplicates + # post-resolution, and drop entries that have been promoted. + discovered = entry.get("discovered_labels") + if isinstance(discovered, list): + kept_disc: List[Dict[str, Any]] = [] + seen: set[str] = set() + for d in discovered: + if not isinstance(d, dict): + continue + slug = _slug_label(d.get("key") or d.get("name")) + slug = _resolve_alias(alias_map, slug) + if not slug or slug in promoted or slug in seen: + continue + seen.add(slug) + new_entry = dict(d) + new_entry["key"] = slug + kept_disc.append(new_entry) + entry["discovered_labels"] = kept_disc + + # Rewrite sequence: alias-resolve every entry; collapse adjacent + # duplicates that result. We DON'T drop slugs that match + # promoted children — the promoted child slug is still a valid + # sequence entry; the flow chart will resolve it to the real + # child node. + seq = entry.get("sequence") + if isinstance(seq, list): + new_seq: List[str] = [] + last: Optional[str] = None + for item in seq: + if not isinstance(item, str): + continue + slug = _resolve_alias(alias_map, _slug_label(item)) + if not slug or slug == last: + continue + new_seq.append(slug) + last = slug + entry["sequence"] = new_seq + + # Top-level metric discoveries live alongside the parent entries + # under the reserved ``DISCOVERED_METRICS_KEY`` slot. Apply the + # flat evaluation-level alias/tombstone map + suppress slugs that + # already correspond to a real top-level Metric so workers that + # finish AFTER the user has merged / deleted / promoted can't + # resurrect a retired candidate. + discovered_metrics_payload = metric_scores.get(DISCOVERED_METRICS_KEY) + if isinstance(discovered_metrics_payload, list): + flat_alias_map = ( + evaluation.discovered_metric_aliases + if isinstance(evaluation.discovered_metric_aliases, dict) + else {} + ) + promoted_metric_slugs = _promoted_top_level_metric_slugs( + db, organization_id + ) + kept_metrics: List[Dict[str, Any]] = [] + seen_metrics: set[str] = set() + for d in discovered_metrics_payload: + if not isinstance(d, dict): + continue + slug = _slug_label(d.get("key") or d.get("name")) + slug = _resolve_alias(flat_alias_map, slug) + if ( + not slug + or slug in promoted_metric_slugs + or slug in seen_metrics + ): + continue + seen_metrics.add(slug) + new_entry = dict(d) + new_entry["key"] = slug + kept_metrics.append(new_entry) + if kept_metrics: + metric_scores[DISCOVERED_METRICS_KEY] = kept_metrics + else: + # No survivors — drop the empty array so empty-discovery rows + # keep their pre-feature payload shape. + metric_scores.pop(DISCOVERED_METRICS_KEY, None) + + return metric_scores + + +def _alias_map_for_parent( + evaluation: CallImportEvaluation, parent_metric_id: UUID +) -> Dict[str, str]: + """Pull ``{from_slug: to_slug}`` for one parent out of the eval's blob. + + Stored shape on the evaluation row is + ``{parent_id_str: {from_slug: to_slug, ...}}``. Returns an empty + dict for parents that have never had a merge applied. + """ + raw = getattr(evaluation, "discovered_label_aliases", None) + if not isinstance(raw, dict): + return {} + submap = raw.get(str(parent_metric_id)) + if not isinstance(submap, dict): + return {} + return { + str(k): str(v) + for k, v in submap.items() + if isinstance(k, str) and isinstance(v, str) + } + + +def _promoted_child_slugs( + db: Session, parent_metric_id: UUID, organization_id: UUID +) -> set[str]: + """Slugs of every real child currently sitting under the parent. + + The Discovered Labels panel hides any candidate whose slug already + matches a real child — that covers both freshly-promoted candidates + and legacy children the LLM happened to re-discover. We pull from + the live ``metrics`` table rather than the eval's + ``selected_metric_groups`` snapshot so newly-promoted children take + effect immediately, even on evaluations that ran before the + promotion. + """ + children = ( + db.query(Metric.name) + .filter( + Metric.parent_metric_id == parent_metric_id, + Metric.organization_id == organization_id, + ) + .all() + ) + out: set[str] = set() + for (name,) in children: + slug = _slug_label(name) + if slug: + out.add(slug) + return out + + +def _promoted_top_level_metric_slugs( + db: Session, organization_id: UUID +) -> set[str]: + """Slugs of every top-level (non-child) Metric in the organization. + + Used to suppress discovered-metric candidates whose slug already + matches a real standalone metric. We intentionally include both + standalone metrics AND parent category metrics — a top-level + discovery that collides with either name is a duplicate by + definition. + """ + rows = ( + db.query(Metric.name) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id.is_(None), + ) + .all() + ) + out: set[str] = set() + for (name,) in rows: + slug = _slug_label(name) + if slug: + out.add(slug) + return out + + +def _get_running_discovered_labels( + db: Session, + eval_id: UUID, + parent_metric_id: UUID, + organization_id: Optional[UUID] = None, + alias_map: Optional[Dict[str, str]] = None, +) -> List[Dict[str, Any]]: + """Slug-deduped view of every discovered label seen in this eval so far. + + Walks each ``call_import_evaluation_rows`` row's + ``metric_scores[parent_id]["discovered_labels"]`` and folds entries + that share the same slug. Returns a list ordered by descending + count and stable on label key, shaped like:: + + [{"key": "customer_on_hold", "name": "Customer put on hold", + "description": "...", "sample_rationale": "...", "count": 12}] + + Powers two callers: + * The worker prompt builder ("REUSE the existing key if it fits") + — invoked just before each row's LLM call to feed the model the + running list of previously-discovered labels in this evaluation. + * The ``/discovered-labels`` API surface used by the frontend + Discovered Labels panel to render candidates with counts + + sample rationales. + + Non-completed rows are skipped: an in-flight row's discoveries are + not yet reliable (the row could fail and never produce final + metric_scores). We accept the tradeoff that rows running + concurrently won't see each other's labels — slug-collision dedup + catches identical re-inventions, and near-paraphrases surface in + the UI panel where the user can manually merge. + """ + + parent_id_str = str(parent_metric_id) + from app.db_sharding.eval_rows import load_evaluation_rows_for_run + + eval_rows = load_evaluation_rows_for_run(db, eval_id) + rows = [ + (row.metric_scores,) + for row in eval_rows + if row.status == CallImportRowStatus.COMPLETED.value + ] + + # Suppress slugs that have either: + # * been promoted to a real child of the parent (so the panel doesn't + # keep nagging the user about a candidate they've already + # accepted), or + # * been merged INTO another slug (the "from" side of a merge) — + # those occurrences fold into the canonical target instead. + promoted_slugs: set[str] = set() + if organization_id is not None: + promoted_slugs = _promoted_child_slugs( + db, parent_metric_id, organization_id + ) + aliases = alias_map or {} + + by_key: Dict[str, Dict[str, Any]] = {} + for (scores,) in rows: + if not isinstance(scores, dict): + continue + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + continue + discovered = parent_entry.get("discovered_labels") + if not isinstance(discovered, list): + continue + for entry in discovered: + if not isinstance(entry, dict): + continue + raw_key = entry.get("key") or entry.get("name") + key = _slug_label(raw_key) + if not key: + continue + # Apply user merges + deletions first, THEN drop anything + # that ended up on a real child slug. Order matters: a + # candidate that was merged into a slug which has since + # been promoted should disappear, not show up at the + # canonical slug. An empty resolved key means the slug was + # tombstoned via the delete endpoint. + key = _resolve_alias(aliases, key) + if not key or key in promoted_slugs: + continue + name = (entry.get("name") or "").strip() or key.replace("_", " ") + description = (entry.get("description") or "").strip() or None + sample = (entry.get("rationale") or "").strip() or None + + existing = by_key.get(key) + if existing is None: + # Track up to N=3 distinct rationales per candidate so + # the Promote-to-child flow can pre-fill the new + # sub-metric's rubric with concrete LLM examples + # without the user copy-pasting from the row table. + # ``sample_rationale`` is preserved for back-compat + # with older clients; ``examples`` is the new field. + examples = [sample] if sample else [] + by_key[key] = { + "key": key, + "name": name, + "description": description, + "sample_rationale": sample, + "examples": examples, + "count": 1, + } + continue + + existing["count"] += 1 + if not existing["description"] and description: + existing["description"] = description + if not existing["sample_rationale"] and sample: + existing["sample_rationale"] = sample + # Append distinct rationales (case-insensitive trim) up + # to a small cap. Headroom is intentionally one above + # what the UI surfaces (2) so we have a backup when the + # first rationale is unhelpful. + if sample: + ex_list: List[str] = existing.setdefault("examples", []) + if len(ex_list) < 3 and not any( + s.strip().lower() == sample.strip().lower() for s in ex_list + ): + ex_list.append(sample) + + return sorted( + by_key.values(), + key=lambda item: (-item["count"], item["key"]), + ) + + +def _get_running_discovered_metrics( + db: Session, + eval_id: UUID, + organization_id: Optional[UUID] = None, + alias_map: Optional[Dict[str, str]] = None, +) -> List[Dict[str, Any]]: + """Slug-deduped view of every discovered top-level metric in this eval. + + Mirrors :func:`_get_running_discovered_labels` but is keyed at the + evaluation level (no ``parent_metric_id``). Walks each completed + row's ``metric_scores[DISCOVERED_METRICS_KEY]`` list, folds entries + that share the same slug (post-alias resolution), and suppresses + slugs that already correspond to a real top-level :class:`Metric` + in the organization. + + Each returned entry is shaped:: + + {"key": "customer_satisfaction", + "name": "Customer Satisfaction", + "description": "...", + "suggested_type": "boolean" | "rating" | "category", + "sample_rationale": "...", + "examples": ["..."], + "count": 12} + """ + + from app.db_sharding.eval_rows import load_evaluation_rows_for_run + + eval_rows = load_evaluation_rows_for_run(db, eval_id) + rows = [ + (row.metric_scores,) + for row in eval_rows + if row.status == CallImportRowStatus.COMPLETED.value + ] + + promoted_slugs: set[str] = set() + if organization_id is not None: + promoted_slugs = _promoted_top_level_metric_slugs( + db, organization_id + ) + aliases = alias_map or {} + + by_key: Dict[str, Dict[str, Any]] = {} + for (scores,) in rows: + if not isinstance(scores, dict): + continue + discovered = scores.get(DISCOVERED_METRICS_KEY) + if not isinstance(discovered, list): + continue + for entry in discovered: + if not isinstance(entry, dict): + continue + raw_key = entry.get("key") or entry.get("name") + key = _slug_label(raw_key) + if not key: + continue + # Apply user merges + deletions first, THEN drop anything + # that ended up on an already-existing top-level metric + # slug. Empty resolved key = tombstoned. + key = _resolve_alias(aliases, key) + if not key or key in promoted_slugs: + continue + name = (entry.get("name") or "").strip() or key.replace( + "_", " " + ) + description = (entry.get("description") or "").strip() or None + sample = (entry.get("rationale") or "").strip() or None + raw_type = str(entry.get("suggested_type") or "").strip().lower() + if raw_type not in _DISCOVERED_METRIC_TYPES: + raw_type = "boolean" + + existing = by_key.get(key) + if existing is None: + examples = [sample] if sample else [] + by_key[key] = { + "key": key, + "name": name, + "description": description, + "suggested_type": raw_type, + "sample_rationale": sample, + "examples": examples, + "count": 1, + } + continue + + existing["count"] += 1 + if not existing["description"] and description: + existing["description"] = description + if not existing["sample_rationale"] and sample: + existing["sample_rationale"] = sample + # Keep the most-frequently-suggested type. We don't track + # per-type frequency yet; defer to the first non-default + # type encountered when the existing entry has the default. + if existing.get("suggested_type") == "boolean" and raw_type != "boolean": + existing["suggested_type"] = raw_type + if sample: + ex_list: List[str] = existing.setdefault("examples", []) + if len(ex_list) < 3 and not any( + s.strip().lower() == sample.strip().lower() for s in ex_list + ): + ex_list.append(sample) + + return sorted( + by_key.values(), + key=lambda item: (-item["count"], item["key"]), + ) + + +def _build_flow_graph( + eval_rows: List[CallImportEvaluationRow], + parent_metric: Metric, + children: List[Metric], + alias_map: Optional[Dict[str, str]] = None, + extra_children: Optional[List[Metric]] = None, +) -> MetricFlowResponse: + """Walk per-row ``sequence`` arrays and produce aggregate nodes/edges. + + A synthetic ``START`` node is prepended to every sequence so the + diagram has a single origin. Children that never appear in any + sequence are still emitted as nodes (count=0) so the UI can render + them in the legend. + + ``alias_map`` lets callers fold merged-out discovered slugs into + their canonical target before building the graph; ``extra_children`` + are children of the parent that aren't in the legend list (e.g. + children promoted *after* the evaluation was created and therefore + missing from ``selected_metric_groups``) but should still resolve in + sequences so the slug doesn't get redrawn as a discovered candidate. + """ + parent_id_str = str(parent_metric.id) + aliases = alias_map or {} + # Build a fast lookup keyed by both the lower_snake child key (what the + # LLM emits in ``sequence``) and the child UUID (what some clients may + # store) so legacy / drifted payloads still resolve. + child_lookup: Dict[str, Metric] = {} + for child in children: + slug = _slug_label(child.name) + child_lookup[slug] = child + child_lookup[str(child.id)] = child + # ``extra_children`` are resolved-only — they shouldn't add legend + # nodes (those come from the explicit ``children`` argument), but + # they need to be in ``child_lookup`` so a sequence step that + # matches a freshly-promoted child resolves to the real child UUID + # instead of falling through to ``discovered_lookup`` and rendering + # as a "discovered" node. + if extra_children: + for child in extra_children: + slug = _slug_label(child.name) + if slug and slug not in child_lookup: + child_lookup[slug] = child + cid = str(child.id) + child_lookup.setdefault(cid, child) + + # Discovered labels: walk every row's discovered_labels first so we + # know which discovered slugs are valid before resolving sequences. + # Discovered nodes get a ``disc:`` prefixed id so they can't collide + # with real child UUIDs in the node/edge graph. We apply + # ``alias_map`` first so merged-out source slugs fold into their + # canonical target — preserving the user's "merge" intent on still- + # in-flight rows whose JSON wasn't rewritten by the merge endpoint. + discovered_lookup: Dict[str, Dict[str, Any]] = {} + for row in eval_rows: + scores = ( + row.metric_scores if isinstance(row.metric_scores, dict) else {} + ) + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + continue + raw_discovered = parent_entry.get("discovered_labels") + if not isinstance(raw_discovered, list): + continue + for entry in raw_discovered: + if not isinstance(entry, dict): + continue + slug = _slug_label(entry.get("key") or entry.get("name")) + slug = _resolve_alias(aliases, slug) + if not slug or slug in child_lookup: + continue + name = (entry.get("name") or "").strip() or slug.replace("_", " ") + existing = discovered_lookup.get(slug) + if existing is None: + discovered_lookup[slug] = { + "id": f"{_DISCOVERED_NODE_PREFIX}{slug}", + "name": name, + } + + node_counts: Dict[str, int] = {} + edge_counts: Dict[tuple[str, str], int] = {} + terminal_counts: Dict[str, int] = {} + + total_rows = len(eval_rows) + rows_with_sequence = 0 + + for row in eval_rows: + scores = ( + row.metric_scores if isinstance(row.metric_scores, dict) else {} + ) + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + continue + raw_sequence = parent_entry.get("sequence") + if not isinstance(raw_sequence, list): + continue + + resolved_ids: List[str] = [] + last_resolved: Optional[str] = None + for item in raw_sequence: + if not isinstance(item, str): + continue + normalized = _resolve_alias(aliases, _slug_label(item)) + child = child_lookup.get(normalized) or child_lookup.get(item) + if child is not None: + cid = str(child.id) + # Adjacent dedupe AFTER alias resolution so two + # different raw slugs that fold to the same target + # don't draw a self-edge through the chart. + if cid == last_resolved: + continue + resolved_ids.append(cid) + last_resolved = cid + continue + disc = discovered_lookup.get(normalized) + if disc is not None: + if disc["id"] == last_resolved: + continue + resolved_ids.append(disc["id"]) + last_resolved = disc["id"] + + if not resolved_ids: + continue + + rows_with_sequence += 1 + for nid in resolved_ids: + node_counts[nid] = node_counts.get(nid, 0) + 1 + + edge_counts[(_FLOW_START_NODE_ID, resolved_ids[0])] = ( + edge_counts.get((_FLOW_START_NODE_ID, resolved_ids[0]), 0) + 1 + ) + for src, tgt in zip(resolved_ids, resolved_ids[1:]): + if src == tgt: + continue + edge_counts[(src, tgt)] = edge_counts.get((src, tgt), 0) + 1 + + terminal_id = resolved_ids[-1] + terminal_counts[terminal_id] = terminal_counts.get(terminal_id, 0) + 1 + + nodes: List[MetricFlowNode] = [] + # Always include a START node so the UI has a stable entry point. + nodes.append( + MetricFlowNode( + id=_FLOW_START_NODE_ID, + label="Start", + count=rows_with_sequence, + is_terminal=False, + ) + ) + + def _emit_child_node(child: Metric) -> None: + cid = str(child.id) + count = node_counts.get(cid, 0) + terminal_count = terminal_counts.get(cid, 0) + is_terminal = False + if rows_with_sequence > 0: + is_terminal = ( + terminal_count / rows_with_sequence + ) >= _FLOW_TERMINAL_THRESHOLD + nodes.append( + MetricFlowNode( + id=cid, + label=child.name, + count=count, + is_terminal=is_terminal, + ) + ) + + emitted_child_ids: set[str] = set() + for child in children: + cid = str(child.id) + if cid in emitted_child_ids: + continue + emitted_child_ids.add(cid) + _emit_child_node(child) + # Extra children (promoted after the eval was created) only get + # legend nodes if they actually appear in the data — otherwise we'd + # pollute the diagram with every standalone promotion the user has + # ever made under this parent. + if extra_children: + for child in extra_children: + cid = str(child.id) + if cid in emitted_child_ids: + continue + if node_counts.get(cid, 0) == 0: + continue + emitted_child_ids.add(cid) + _emit_child_node(child) + # Append discovered nodes after the real children so legend ordering + # keeps user-defined labels first. + for slug, info in discovered_lookup.items(): + nid = info["id"] + count = node_counts.get(nid, 0) + terminal_count = terminal_counts.get(nid, 0) + is_terminal = False + if rows_with_sequence > 0: + is_terminal = ( + terminal_count / rows_with_sequence + ) >= _FLOW_TERMINAL_THRESHOLD + nodes.append( + MetricFlowNode( + id=nid, + label=info["name"], + count=count, + is_terminal=is_terminal, + is_discovered=True, + ) + ) + + edges: List[MetricFlowEdge] = [ + MetricFlowEdge(source=src, target=tgt, count=count) + for (src, tgt), count in sorted( + edge_counts.items(), key=lambda kv: kv[1], reverse=True + ) + ] + + return MetricFlowResponse( + parent_metric_id=parent_id_str, + parent_metric_name=parent_metric.name, + selection_mode=parent_metric.selection_mode, + nodes=nodes, + edges=edges, + total_rows=total_rows, + rows_with_sequence=rows_with_sequence, + ) + + +@router.get( + "/{eval_id}/flow", + response_model=MetricFlowResponse, + operation_id="getCallImportEvaluationFlow", +) +async def get_call_import_evaluation_flow( + call_import_id: UUID, + eval_id: UUID, + parent_metric_id: UUID = Query( + ..., + description=( + "Parent (category) metric whose children's sequences should be " + "aggregated into a flow graph." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> MetricFlowResponse: + """Aggregate the LLM-inferred per-row sequences into one flow graph. + + Returns ``nodes`` (one per child of the parent metric, plus a + synthetic ``START`` node) and ``edges`` (counts of consecutive + label transitions across every row that produced a sequence). The + frontend feeds this directly into a React Flow / xyflow canvas; + edge thickness should scale with ``count / total_rows`` and + ``is_terminal`` nodes should be styled as outcomes. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + parent = ( + db.query(Metric) + .filter( + Metric.id == parent_metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if not parent: + raise HTTPException( + status_code=404, + detail="Parent metric not found in this organization.", + ) + if not parent.selection_mode: + raise HTTPException( + status_code=400, + detail=( + "Flow charts are only meaningful for parent metrics " + "(selection_mode set). This metric is standalone." + ), + ) + + # Children are taken from selected_metric_groups when present so the + # flow chart reflects exactly the subset that ran in this + # evaluation; otherwise fall back to every enabled child of the + # parent. + groups_raw = ( + evaluation.selected_metric_groups + if isinstance(evaluation.selected_metric_groups, dict) + else {} + ) + parent_id_str = str(parent.id) + children: List[Metric] = [] + if parent_id_str in groups_raw and isinstance( + groups_raw[parent_id_str], list + ): + child_ids: List[UUID] = [] + for c in groups_raw[parent_id_str]: + try: + child_ids.append(UUID(str(c))) + except (TypeError, ValueError): + continue + if child_ids: + children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.id.in_(child_ids), + ) + .order_by(Metric.created_at.asc()) + .all() + ) + if not children: + children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id == parent.id, + ) + .order_by(Metric.created_at.asc()) + .all() + ) + + # Children promoted AFTER this evaluation was created aren't in + # ``selected_metric_groups`` but their slugs still appear in already- + # scored rows' sequences. Pass them as ``extra_children`` so those + # sequence entries resolve against the real (now promoted) child + # instead of being redrawn as discovered candidates. + extra_children: List[Metric] = [] + if children: + existing_ids = {child.id for child in children} + all_children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id == parent.id, + ) + .all() + ) + extra_children = [c for c in all_children if c.id not in existing_ids] + + eval_rows = _load_eval_rows(db, eval_id) + + alias_map = _alias_map_for_parent(evaluation, parent.id) + return _build_flow_graph( + eval_rows, + parent, + children, + alias_map=alias_map, + extra_children=extra_children, + ) + + +@router.get( + "/{eval_id}/discovered-labels", + response_model=DiscoveredLabelsResponse, + operation_id="getCallImportEvaluationDiscoveredLabels", +) +async def get_call_import_evaluation_discovered_labels( + call_import_id: UUID, + eval_id: UUID, + parent_metric_id: UUID = Query( + ..., + description=( + "Parent (category) metric whose LLM-discovered candidate " + "sub-labels should be aggregated across rows." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> DiscoveredLabelsResponse: + """Aggregate candidate sub-labels the LLM discovered during this eval. + + Only meaningful for parents with ``allow_discovery=true``; for other + parents we just return an empty ``items`` list rather than 400-ing + so the frontend can call the endpoint unconditionally for every + parent on the Flow tab without branching. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + parent = ( + db.query(Metric) + .filter( + Metric.id == parent_metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if not parent: + raise HTTPException( + status_code=404, + detail="Parent metric not found in this organization.", + ) + + alias_map = _alias_map_for_parent(evaluation, parent_metric_id) + items_raw = _get_running_discovered_labels( + db, + eval_id, + parent_metric_id, + organization_id=organization_id, + alias_map=alias_map, + ) + items = [DiscoveredLabelItem(**item) for item in items_raw] + return DiscoveredLabelsResponse( + parent_metric_id=str(parent.id), items=items + ) + + +@router.post( + "/{eval_id}/discovered-labels/merge", + response_model=DiscoveredLabelsResponse, + operation_id="mergeCallImportEvaluationDiscoveredLabels", +) +async def merge_call_import_evaluation_discovered_labels( + call_import_id: UUID, + eval_id: UUID, + body: DiscoveredLabelMergeRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> DiscoveredLabelsResponse: + """Rewrite every row's ``discovered_labels`` entry from from_key -> to_key. + + Idempotent — re-merging the same pair is a no-op. Discovered slugs + inside per-row ``sequence`` arrays are also rewritten so the flow + chart stays consistent with the panel. When a row already has + ``to_key`` and we're merging ``from_key`` into it, we drop the + ``from_key`` entry instead of producing two entries with the same + slug. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + parent = ( + db.query(Metric) + .filter( + Metric.id == body.parent_metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if not parent: + raise HTTPException( + status_code=404, + detail="Parent metric not found in this organization.", + ) + + from_key = _slug_label(body.from_key) + to_key = _slug_label(body.to_key) + if not from_key or not to_key: + raise HTTPException( + status_code=400, + detail="from_key and to_key must be non-empty slugs.", + ) + if from_key == to_key: + # No-op; just return the current aggregate so the client can + # refresh its view. + alias_map_existing = _alias_map_for_parent(evaluation, parent.id) + items_raw = _get_running_discovered_labels( + db, + eval_id, + body.parent_metric_id, + organization_id=organization_id, + alias_map=alias_map_existing, + ) + return DiscoveredLabelsResponse( + parent_metric_id=str(parent.id), + items=[DiscoveredLabelItem(**item) for item in items_raw], + ) + + parent_id_str = str(parent.id) + from app.db_sharding.eval_rows import foreach_evaluation_row_mutating + + def _merge_discovered_label_row(row: CallImportEvaluationRow) -> bool: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else None + ) + if not scores: + return False + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + return False + + mutated = False + discovered = parent_entry.get("discovered_labels") + if isinstance(discovered, list): + kept: List[Dict[str, Any]] = [] + existing_to = next( + ( + e + for e in discovered + if isinstance(e, dict) + and _slug_label(e.get("key") or e.get("name")) == to_key + ), + None, + ) + for entry in discovered: + if not isinstance(entry, dict): + kept.append(entry) + continue + key = _slug_label(entry.get("key") or entry.get("name")) + if key == from_key: + if existing_to is not None: + mutated = True + continue + new_entry = dict(entry) + new_entry["key"] = to_key + kept.append(new_entry) + mutated = True + else: + kept.append(entry) + if mutated: + parent_entry["discovered_labels"] = kept + + seq = parent_entry.get("sequence") + if isinstance(seq, list): + new_seq: List[str] = [] + seq_changed = False + last_added: Optional[str] = None + for item in seq: + if isinstance(item, str) and _slug_label(item) == from_key: + seq_changed = True + if last_added == to_key: + continue + new_seq.append(to_key) + last_added = to_key + else: + new_seq.append(item) + last_added = ( + _slug_label(item) if isinstance(item, str) else None + ) + if seq_changed: + parent_entry["sequence"] = new_seq + mutated = True + + if mutated: + row.metric_scores = dict(scores) + return mutated + + foreach_evaluation_row_mutating(db, eval_id, _merge_discovered_label_row) + + # Persist the merge at the evaluation level too. This is what makes + # the merge survive future scoring: rows that finish AFTER this + # call (e.g. retries, in-flight workers) will go through the + # alias map in the API surface even if the per-row JSON they + # write still mentions ``from_key``. We chain through any existing + # alias so merging A→B and then B→C resolves A→C in the panel. + raw_aliases = ( + evaluation.discovered_label_aliases + if isinstance(evaluation.discovered_label_aliases, dict) + else {} + ) + aliases_top = dict(raw_aliases) + parent_aliases = dict(aliases_top.get(parent_id_str) or {}) + # Resolve transitively: if to_key itself was previously merged into + # something else, point from_key at the canonical end-of-chain. + canonical_to = _resolve_alias(parent_aliases, to_key) + parent_aliases[from_key] = canonical_to + # Re-target any earlier aliases that pointed AT from_key — without + # this, A→B and then B→C would leave A still pointing to B (now a + # broken pointer because B is gone). Rewriting them keeps the + # alias map self-consistent. + for k, v in list(parent_aliases.items()): + if v == from_key: + parent_aliases[k] = canonical_to + aliases_top[parent_id_str] = parent_aliases + evaluation.discovered_label_aliases = aliases_top + + stamp_evaluation_actor(evaluation, principal) + db.commit() + + alias_map_after = _alias_map_for_parent(evaluation, parent.id) + items_raw = _get_running_discovered_labels( + db, + eval_id, + body.parent_metric_id, + organization_id=organization_id, + alias_map=alias_map_after, + ) + return DiscoveredLabelsResponse( + parent_metric_id=str(parent.id), + items=[DiscoveredLabelItem(**item) for item in items_raw], + ) + + +@router.post( + "/{eval_id}/discovered-labels/delete", + response_model=DiscoveredLabelsResponse, + operation_id="deleteCallImportEvaluationDiscoveredLabel", +) +async def delete_call_import_evaluation_discovered_label( + call_import_id: UUID, + eval_id: UUID, + body: DiscoveredLabelDeleteRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> DiscoveredLabelsResponse: + """Tombstone a single LLM-discovered candidate for this evaluation. + + Symmetric with the merge endpoint, but instead of redirecting the + slug at another candidate we mark it as deleted. After this call: + + * the slug is stripped from every row's + ``metric_scores[parent].discovered_labels`` list, and from + every row's ``sequence`` array (so the flow chart no longer + draws a node for it); + * the slug is recorded in + ``evaluation.discovered_label_aliases[parent][slug] = ""`` + so any worker that finishes a row AFTER this call (e.g. a row + still in flight when the user clicked Delete) silently drops + the slug instead of resurrecting it. + + Idempotent: deleting an already-deleted slug is a no-op. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + parent = ( + db.query(Metric) + .filter( + Metric.id == body.parent_metric_id, + Metric.organization_id == organization_id, + ) + .first() + ) + if not parent: + raise HTTPException( + status_code=404, + detail="Parent metric not found in this organization.", + ) + + target_key = _slug_label(body.key) + if not target_key: + raise HTTPException( + status_code=400, + detail="key must be a non-empty slug.", + ) + + parent_id_str = str(parent.id) + from app.db_sharding.eval_rows import foreach_evaluation_row_mutating + + def _delete_discovered_label_row(row: CallImportEvaluationRow) -> bool: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else None + ) + if not scores: + return False + parent_entry = scores.get(parent_id_str) + if not isinstance(parent_entry, dict): + return False + + mutated = False + discovered = parent_entry.get("discovered_labels") + if isinstance(discovered, list): + kept = [ + e + for e in discovered + if not ( + isinstance(e, dict) + and _slug_label(e.get("key") or e.get("name")) + == target_key + ) + ] + if len(kept) != len(discovered): + parent_entry["discovered_labels"] = kept + mutated = True + + seq = parent_entry.get("sequence") + if isinstance(seq, list): + new_seq: List[str] = [] + seq_changed = False + last_added: Optional[str] = None + for item in seq: + if isinstance(item, str) and _slug_label(item) == target_key: + seq_changed = True + continue + if isinstance(item, str): + norm = _slug_label(item) + if norm == last_added: + seq_changed = True + continue + last_added = norm + new_seq.append(item) + if seq_changed: + parent_entry["sequence"] = new_seq + mutated = True + + if mutated: + row.metric_scores = dict(scores) + return mutated + + foreach_evaluation_row_mutating(db, eval_id, _delete_discovered_label_row) + + # 3. Persist the tombstone on the evaluation so workers that finish + # later don't re-surface the deleted slug. We also retarget any + # existing aliases whose ``to_key`` was the deleted slug — without + # this, a previous merge that pointed at this slug would leave a + # dangling pointer. + raw_aliases = ( + evaluation.discovered_label_aliases + if isinstance(evaluation.discovered_label_aliases, dict) + else {} + ) + aliases_top = dict(raw_aliases) + parent_aliases = dict(aliases_top.get(parent_id_str) or {}) + parent_aliases[target_key] = "" # deletion sentinel + for k, v in list(parent_aliases.items()): + if v == target_key: + parent_aliases[k] = "" + aliases_top[parent_id_str] = parent_aliases + evaluation.discovered_label_aliases = aliases_top + + stamp_evaluation_actor(evaluation, principal) + db.commit() + + alias_map_after = _alias_map_for_parent(evaluation, parent.id) + items_raw = _get_running_discovered_labels( + db, + eval_id, + body.parent_metric_id, + organization_id=organization_id, + alias_map=alias_map_after, + ) + return DiscoveredLabelsResponse( + parent_metric_id=str(parent.id), + items=[DiscoveredLabelItem(**item) for item in items_raw], + ) + + +# --------------------------------------------------------------------------- +# Discovered TOP-LEVEL METRICS (per-evaluation discovery) +# +# These endpoints are the parallel of the discovered-labels trio above but +# scoped to the evaluation as a whole instead of to a parent metric. They +# all live under ``/{eval_id}/discovered-metrics`` and operate on the +# reserved ``DISCOVERED_METRICS_KEY`` slot of each per-row +# ``metric_scores`` plus the flat ``CallImportEvaluation.discovered_metric_aliases`` +# map (no parent-id nesting). +# --------------------------------------------------------------------------- + + +def _flat_metric_aliases( + evaluation: CallImportEvaluation, +) -> Dict[str, str]: + """Pull the flat ``{from_slug: to_slug}`` map for an evaluation.""" + raw = getattr(evaluation, "discovered_metric_aliases", None) + if not isinstance(raw, dict): + return {} + return { + str(k): str(v) + for k, v in raw.items() + if isinstance(k, str) and isinstance(v, str) + } + + +@router.get( + "/{eval_id}/discovered-metrics", + response_model=DiscoveredMetricsResponse, + operation_id="getCallImportEvaluationDiscoveredMetrics", +) +async def get_call_import_evaluation_discovered_metrics( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> DiscoveredMetricsResponse: + """Aggregate top-level metric candidates the LLM discovered during this eval. + + Returns an empty ``items`` list when the evaluation did not opt + into top-level metric discovery; this keeps the frontend able to + call the endpoint unconditionally without branching on the + evaluation's ``discover_new_metrics`` flag. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + if not bool(getattr(evaluation, "discover_new_metrics", False)): + return DiscoveredMetricsResponse(evaluation_id=evaluation.id, items=[]) + + items_raw = _get_running_discovered_metrics( + db, + eval_id, + organization_id=organization_id, + alias_map=_flat_metric_aliases(evaluation), + ) + return DiscoveredMetricsResponse( + evaluation_id=evaluation.id, + items=[DiscoveredMetricItem(**item) for item in items_raw], + ) + + +@router.post( + "/{eval_id}/discovered-metrics/merge", + response_model=DiscoveredMetricsResponse, + operation_id="mergeCallImportEvaluationDiscoveredMetrics", +) +async def merge_call_import_evaluation_discovered_metrics( + call_import_id: UUID, + eval_id: UUID, + body: DiscoveredMetricMergeRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> DiscoveredMetricsResponse: + """Rewrite every row's ``__discovered_metrics__`` entry from→to. + + Mirrors the discovered-labels merge endpoint but operates on the + flat top-level metric list. Idempotent — re-merging is a no-op. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + from_key = _slug_label(body.from_key) + to_key = _slug_label(body.to_key) + if not from_key or not to_key: + raise HTTPException( + status_code=400, + detail="from_key and to_key must be non-empty slugs.", + ) + if from_key == to_key: + items_raw = _get_running_discovered_metrics( + db, + eval_id, + organization_id=organization_id, + alias_map=_flat_metric_aliases(evaluation), + ) + return DiscoveredMetricsResponse( + evaluation_id=evaluation.id, + items=[DiscoveredMetricItem(**item) for item in items_raw], + ) + + from app.db_sharding.eval_rows import foreach_evaluation_row_mutating + + def _merge_discovered_metric_row(row: CallImportEvaluationRow) -> bool: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else None + ) + if not scores: + return False + discovered = scores.get(DISCOVERED_METRICS_KEY) + if not isinstance(discovered, list): + return False + + kept: List[Dict[str, Any]] = [] + mutated = False + existing_to = next( + ( + e + for e in discovered + if isinstance(e, dict) + and _slug_label(e.get("key") or e.get("name")) == to_key + ), + None, + ) + for entry in discovered: + if not isinstance(entry, dict): + kept.append(entry) + continue + key = _slug_label(entry.get("key") or entry.get("name")) + if key == from_key: + if existing_to is not None: + mutated = True + continue + new_entry = dict(entry) + new_entry["key"] = to_key + kept.append(new_entry) + mutated = True + else: + kept.append(entry) + if mutated: + scores[DISCOVERED_METRICS_KEY] = kept + row.metric_scores = dict(scores) + return mutated + + foreach_evaluation_row_mutating(db, eval_id, _merge_discovered_metric_row) + + raw_aliases = ( + evaluation.discovered_metric_aliases + if isinstance(evaluation.discovered_metric_aliases, dict) + else {} + ) + aliases = dict(raw_aliases) + canonical_to = _resolve_alias(aliases, to_key) + aliases[from_key] = canonical_to + for k, v in list(aliases.items()): + if v == from_key: + aliases[k] = canonical_to + evaluation.discovered_metric_aliases = aliases + + stamp_evaluation_actor(evaluation, principal) + db.commit() + + items_raw = _get_running_discovered_metrics( + db, + eval_id, + organization_id=organization_id, + alias_map=_flat_metric_aliases(evaluation), + ) + return DiscoveredMetricsResponse( + evaluation_id=evaluation.id, + items=[DiscoveredMetricItem(**item) for item in items_raw], + ) + + +@router.post( + "/{eval_id}/discovered-metrics/delete", + response_model=DiscoveredMetricsResponse, + operation_id="deleteCallImportEvaluationDiscoveredMetric", +) +async def delete_call_import_evaluation_discovered_metric( + call_import_id: UUID, + eval_id: UUID, + body: DiscoveredMetricDeleteRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> DiscoveredMetricsResponse: + """Tombstone a single LLM-discovered top-level metric candidate.""" + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + target_key = _slug_label(body.key) + if not target_key: + raise HTTPException( + status_code=400, + detail="key must be a non-empty slug.", + ) + + from app.db_sharding.eval_rows import foreach_evaluation_row_mutating + + def _delete_discovered_metric_row(row: CallImportEvaluationRow) -> bool: + scores = ( + row.metric_scores + if isinstance(row.metric_scores, dict) + else None + ) + if not scores: + return False + discovered = scores.get(DISCOVERED_METRICS_KEY) + if not isinstance(discovered, list): + return False + kept = [ + e + for e in discovered + if not ( + isinstance(e, dict) + and _slug_label(e.get("key") or e.get("name")) + == target_key + ) + ] + if len(kept) == len(discovered): + return False + if kept: + scores[DISCOVERED_METRICS_KEY] = kept + else: + scores.pop(DISCOVERED_METRICS_KEY, None) + row.metric_scores = dict(scores) + return True + + foreach_evaluation_row_mutating(db, eval_id, _delete_discovered_metric_row) + + raw_aliases = ( + evaluation.discovered_metric_aliases + if isinstance(evaluation.discovered_metric_aliases, dict) + else {} + ) + aliases = dict(raw_aliases) + aliases[target_key] = "" # tombstone + for k, v in list(aliases.items()): + if v == target_key: + aliases[k] = "" + evaluation.discovered_metric_aliases = aliases + + stamp_evaluation_actor(evaluation, principal) + db.commit() + + items_raw = _get_running_discovered_metrics( + db, + eval_id, + organization_id=organization_id, + alias_map=_flat_metric_aliases(evaluation), + ) + return DiscoveredMetricsResponse( + evaluation_id=evaluation.id, + items=[DiscoveredMetricItem(**item) for item in items_raw], + ) + + +@router.delete( + "/{eval_id}/rows/{eval_row_id}", + status_code=status.HTTP_204_NO_CONTENT, + operation_id="deleteCallImportEvaluationRow", +) +async def delete_call_import_evaluation_row( + call_import_id: UUID, + eval_id: UUID, + eval_row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> Response: + """Delete a single per-row scoring entry within an evaluation run. + + Useful when the user wants to drop a noisy row before re-exporting + the CSV — e.g. a row whose audio was corrupt and skewed the + aggregate. Counters on the parent are recomputed so the rolled-up + status stays accurate. + """ + + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + from app.db_sharding.eval_rows import delete_evaluation_row_on_shards + + if not delete_evaluation_row_on_shards(eval_row_id, eval_id): + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) + _rollup_evaluation_status(evaluation, db) + db.commit() + return Response(status_code=status.HTTP_204_NO_CONTENT) + + eval_row = ( + db.query(CallImportEvaluationRow) + .filter( + CallImportEvaluationRow.id == eval_row_id, + CallImportEvaluationRow.evaluation_id == eval_id, + ) + .first() + ) + if not eval_row: + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) + + # If the row was still in flight, best-effort revoke the worker task + # so it doesn't try to write into a deleted DB row mid-execution. + if eval_row.celery_task_id and eval_row.status in {"pending", "running"}: + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke(eval_row.celery_task_id, terminate=False) + except Exception: + pass + + db.delete(eval_row) + db.flush() + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +# --------------------------------------------------------------------------- +# Retry endpoints +# --------------------------------------------------------------------------- +# +# The create endpoint enqueues every row of a fresh run; these endpoints +# let the user re-enqueue a *subset* of rows in an existing run — most +# commonly the ones that failed. We keep the worker contract identical +# (``evaluate_call_import_row_task(eval_row_id)``), so the retry path +# only has to reset row state and re-fan-out. When a row is missing its +# diarised transcript and the run was configured for diarised +# transcripts, we chain through ``transcribe_call_import_row_task`` the +# same way the create endpoint does — that's what makes "retry" feel +# like "just fix it" instead of "fail again immediately". + + +def _prepare_source_row_for_retry( + source_row: CallImportRow, + *, + transcribe_overwrite: bool, +) -> None: + """Clear stale diarisation markers so retry dispatch can re-run the pipeline.""" + source_row.celery_task_id = None + + # Re-fetch recordings when a prior import failed or stalled without S3 audio. + # Mirrors retry_failed_call_import_rows so eval retry can re-enqueue imports. + if ( + source_row.status + in (CallImportRowStatus.FAILED, CallImportRowStatus.PROCESSING) + and not (source_row.recording_s3_key or "").strip() + ): + source_row.status = CallImportRowStatus.PENDING + source_row.error_message = None + + if transcribe_overwrite and (source_row.diarised_transcript or "").strip(): + source_row.diarised_transcript = None + + has_dia = bool((source_row.diarised_transcript or "").strip()) + dia_status = (source_row.diarised_transcript_status or "").strip().lower() + + if has_dia and not transcribe_overwrite: + source_row.diarised_transcript_status = "completed" + source_row.diarised_transcript_error = None + return + + if dia_status in {"failed", "pending", "running", "idle"}: + source_row.diarised_transcript_status = "idle" + source_row.diarised_transcript_error = None + + +def _reset_eval_row_for_retry( + eval_row: CallImportEvaluationRow, + *, + metric_ids: Optional[List[UUID]] = None, + skip_revoke: bool = False, +) -> None: + """Wipe per-row state so the worker can re-run it cleanly. + + Mirrors the initial state used by ``create_call_import_evaluation`` + when it first inserts a row, with the addition of revoking any + lingering Celery task id. + + When ``metric_ids`` is provided, this is a **metric-subset retry**: + only the scores for those metrics are removed from + ``metric_scores`` (other metrics' previously-computed values are + preserved so the worker's partial-merge write keeps them intact). + Otherwise the entire ``metric_scores`` dict is reset, matching the + legacy behaviour. + """ + if ( + not skip_revoke + and eval_row.celery_task_id + and eval_row.status in {"pending", "running"} + ): + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke(eval_row.celery_task_id, terminate=False) + except Exception: # noqa: BLE001 — revoke is best-effort + pass + eval_row.status = "pending" + eval_row.error_message = None + if metric_ids: + # Strip ONLY the targeted metric keys. Both string and UUID + # forms can appear in ``metric_scores`` depending on which + # code path wrote the dict, so we normalise to lower-case + # strings for the comparison. + existing = ( + eval_row.metric_scores if isinstance(eval_row.metric_scores, dict) else {} + ) + target_keys = {str(mid).lower() for mid in metric_ids} + eval_row.metric_scores = { + key: value + for key, value in existing.items() + if str(key).lower() not in target_keys + } + else: + eval_row.metric_scores = {} + eval_row.started_at = None + eval_row.finished_at = None + eval_row.celery_task_id = None + + +def _enqueue_eval_rows_with_optional_transcribe( + db: Session, + evaluation: CallImportEvaluation, + eval_rows_with_source: List[ + Tuple[CallImportEvaluationRow, CallImportRow] + ], + *, + transcribe_overwrite: bool = False, + restricted_metric_ids: Optional[List[UUID]] = None, +) -> Tuple[int, int]: + """Schedule throttled evaluation dispatch for pending eval rows. + + Returns ``(evaluate_only_count, transcribe_then_evaluate_count)`` for + logging/UI compatibility. Actual Celery fan-out is handled by + :func:`dispatch_evaluation_rows_task` under Redis fair-share limits. + """ + from app.workers.concurrency.eval_dispatch import _needs_transcribe_for_eval + from app.workers.concurrency.fair_dispatch import ( + schedule_fair_dispatch, + store_evaluation_transcribe_overwrite, + store_row_restricted_metrics, + ) + + eval_only_count = 0 + transcribe_count = 0 + if eval_rows_with_source: + for eval_row, source_row in eval_rows_with_source: + if _needs_transcribe_for_eval( + evaluation, + source_row, + transcribe_overwrite=transcribe_overwrite, + ): + transcribe_count += 1 + else: + eval_only_count += 1 + + restricted_metric_ids_str: Optional[List[str]] = ( + [str(mid) for mid in restricted_metric_ids] + if restricted_metric_ids + else None + ) + if restricted_metric_ids_str: + for eval_row, _ in eval_rows_with_source: + store_row_restricted_metrics(eval_row.id, restricted_metric_ids_str) + else: + restricted_metric_ids_str = ( + [str(mid) for mid in restricted_metric_ids] if restricted_metric_ids else None + ) + store_evaluation_transcribe_overwrite( + evaluation.id, + overwrite=transcribe_overwrite, + ) + schedule_fair_dispatch(max_workspace_turns=999) + return eval_only_count, transcribe_count + + +def _apply_telephony_retry_overrides( + db: Session, + *, + call_import: CallImport, + organization_id: UUID, + payload: CallImportEvaluationRetryRequest, +) -> None: + """Pin or clear telephony credentials on the batch for this retry pass.""" + fields_set = payload.model_fields_set + if ( + "provider" not in fields_set + and "telephony_integration_id" not in fields_set + ): + return + + from app.api.v1.routes.call_imports import _resolve_telephony_integration + + if payload.telephony_integration_id is not None: + integration = _resolve_telephony_integration( + db, + organization_id, + payload.telephony_integration_id, + payload.provider or "", + ) + call_import.provider = integration.provider + call_import.telephony_integration_id = integration.id + else: + call_import.provider = None + call_import.telephony_integration_id = None + db.flush() + + +def _apply_retry_overrides( + db: Session, + evaluation: CallImportEvaluation, + organization_id: UUID, + payload: CallImportEvaluationRetryRequest, +) -> None: + """Validate + persist the LLM/STT override fields on the run. + + Mirrors the validation in ``create_call_import_evaluation`` but + only touches the fields the caller actually sent — leaving any + field ``None`` preserves the run's existing value. Raises + ``HTTPException(400)`` on bad input so the route handler can let + FastAPI turn it into a clean 400 response. + """ + # --- LLM provider + model (must be sent together) --- + if payload.llm_provider is not None or payload.llm_model is not None: + if not (payload.llm_provider and payload.llm_model): + raise HTTPException( + status_code=400, + detail=( + "Both llm_provider and llm_model are required when " + "overriding the run LLM on retry." + ), + ) + try: + evaluation.llm_provider = ModelProvider( + payload.llm_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Unknown LLM provider '{payload.llm_provider}'. " + "Valid keys are documented in ModelProvider." + ), + ) + new_model = payload.llm_model.strip() or None + if not new_model: + raise HTTPException( + status_code=400, detail="llm_model cannot be empty." + ) + evaluation.llm_model = new_model + + # --- LLM credential pin --- + if payload.llm_credential_id is not None: + cred = ( + db.query(AIProvider) + .filter( + AIProvider.id == payload.llm_credential_id, + AIProvider.organization_id == organization_id, + ) + .first() + ) + if not cred: + raise HTTPException( + status_code=400, + detail=( + "The provided llm_credential_id does not exist in " + "this organization." + ), + ) + evaluation.llm_credential_id = payload.llm_credential_id + + if payload.llm_config is not None: + evaluation.llm_config = payload.llm_config + + # --- Per-metric LLM overrides --- + # We accept the same dict shape as the create endpoint but + # constrain keys to leaf metrics that are actually in this run. + # Passing an empty dict explicitly clears existing overrides. + if payload.metric_llm_overrides is not None: + valid_leaf_ids = { + str(mid) for mid in (evaluation.selected_metric_ids or []) + } + overrides_payload: Dict[str, Dict[str, Any]] = {} + for metric_id, override in payload.metric_llm_overrides.items(): + if metric_id not in valid_leaf_ids: + raise HTTPException( + status_code=400, + detail=( + "metric_llm_overrides references metric " + f"{metric_id} which is not a leaf metric in " + "this run." + ), + ) + override_dict: Dict[str, Any] = {} + if override.provider is not None: + if not override.model: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} has a " + "provider but no model." + ), + ) + try: + override_dict["provider"] = ModelProvider( + override.provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} uses " + f"unknown provider '{override.provider}'." + ), + ) + override_dict["model"] = override.model.strip() + elif override.model: + raise HTTPException( + status_code=400, + detail=( + f"Override for metric {metric_id} has a model " + "but no provider." + ), + ) + if override.credential_id is not None: + override_dict["credential_id"] = str(override.credential_id) + if override.llm_config is not None: + override_dict["llm_config"] = override.llm_config + if override_dict: + overrides_payload[metric_id] = override_dict + evaluation.metric_llm_overrides = overrides_payload or None + + # --- STT provider + model (must be sent together) --- + if payload.stt_provider is not None or payload.stt_model is not None: + if not (payload.stt_provider and payload.stt_model): + raise HTTPException( + status_code=400, + detail=( + "Both stt_provider and stt_model are required " + "when overriding the run STT on retry." + ), + ) + try: + evaluation.stt_provider = ModelProvider( + payload.stt_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Unknown STT provider '{payload.stt_provider}'.", + ) + new_stt_model = payload.stt_model.strip() or None + if not new_stt_model: + raise HTTPException( + status_code=400, detail="stt_model cannot be empty." + ) + evaluation.stt_model = new_stt_model + + # --- STT credential pin --- + if payload.stt_credential_id is not None: + evaluation.stt_credential_id = payload.stt_credential_id + + # --- LLM diariser provider + model (must be sent together) --- + if ( + payload.diarization_llm_provider is not None + or payload.diarization_llm_model is not None + ): + if not ( + payload.diarization_llm_provider + and payload.diarization_llm_model + ): + raise HTTPException( + status_code=400, + detail=( + "Both diarization_llm_provider and " + "diarization_llm_model are required when overriding " + "the run diariser on retry." + ), + ) + try: + evaluation.diarisation_llm_provider = ModelProvider( + payload.diarization_llm_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + "Unknown diarisation LLM provider " + f"'{payload.diarization_llm_provider}'." + ), + ) + new_diariser_model = ( + payload.diarization_llm_model.strip() or None + ) + if not new_diariser_model: + raise HTTPException( + status_code=400, + detail="diarization_llm_model cannot be empty.", + ) + evaluation.diarisation_llm_model = new_diariser_model + + if payload.diarization_llm_credential_id is not None: + evaluation.diarisation_llm_credential_id = ( + payload.diarization_llm_credential_id + ) + + # ``diarization_prompt`` semantics: None = leave untouched; + # empty string = clear (fall back to the canonical default at + # worker time); anything else = persist verbatim. + if payload.diarization_prompt is not None: + cleaned = payload.diarization_prompt.strip() + evaluation.diarisation_prompt = cleaned or None + + if payload.transcribe_mode is not None: + mode = payload.transcribe_mode.strip().lower() + if mode not in {"stt_llm", "llm_only"}: + raise HTTPException( + status_code=400, + detail=( + f"Unknown transcribe_mode '{payload.transcribe_mode}'. " + "Valid values are 'stt_llm' and 'llm_only'." + ), + ) + evaluation.transcribe_mode = mode + + +def _gather_retry_targets( + db: Session, + evaluation: CallImportEvaluation, + requested_ids: Optional[List[UUID]], + *, + include_completed: bool = False, +) -> Tuple[ + List[Tuple[CallImportEvaluationRow, CallImportRow]], + List[CallImportEvaluationRetrySkippedItem], +]: + """Resolve which rows to retry + reasons for any we refuse. + + When ``requested_ids`` is None we retry every row whose status is + ``failed`` (or every row when ``include_completed`` is also set — + used by the metric-subset retry path which legitimately wants to + recompute a metric on already-successful rows). When the caller + passes ids explicitly we still filter out rows that are currently + in flight; ``include_completed`` controls whether previously- + successful rows are eligible. + """ + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + from app.db_sharding.eval_rows import gather_retry_targets_sharded + + return gather_retry_targets_sharded( + db, + evaluation, + requested_ids, + include_completed=include_completed, + ) + + eval_rows_query = db.query(CallImportEvaluationRow).filter( + CallImportEvaluationRow.evaluation_id == evaluation.id + ) + + targets: List[Tuple[CallImportEvaluationRow, CallImportRow]] = [] + skipped: List[CallImportEvaluationRetrySkippedItem] = [] + + if requested_ids is None: + if include_completed: + # "Retry everything" path used by the metric-subset re-run + # UI. Still skip in-flight rows below so we don't trample + # work the worker is actively doing. + candidate_rows = eval_rows_query.filter( + CallImportEvaluationRow.status.in_(["failed", "completed"]) + ).all() + else: + candidate_rows = eval_rows_query.filter( + CallImportEvaluationRow.status == "failed" + ).all() + else: + requested_set = set(requested_ids) + candidate_rows = eval_rows_query.filter( + CallImportEvaluationRow.id.in_(requested_set) + ).all() + found_ids = {row.id for row in candidate_rows} + for missing in requested_set - found_ids: + skipped.append( + CallImportEvaluationRetrySkippedItem( + eval_row_id=missing, + reason="unknown", + ) + ) + + if not candidate_rows: + return targets, skipped + + source_row_ids = [row.call_import_row_id for row in candidate_rows] + source_rows = ( + db.query(CallImportRow) + .filter(CallImportRow.id.in_(source_row_ids)) + .all() + ) + source_by_id = {row.id: row for row in source_rows} + + for eval_row in candidate_rows: + if eval_row.status in {"pending", "running"}: + skipped.append( + CallImportEvaluationRetrySkippedItem( + eval_row_id=eval_row.id, + reason="in_progress", + ) + ) + continue + if eval_row.status == "completed" and not include_completed: + skipped.append( + CallImportEvaluationRetrySkippedItem( + eval_row_id=eval_row.id, + reason="completed", + ) + ) + continue + source_row = source_by_id.get(eval_row.call_import_row_id) + if source_row is None: + skipped.append( + CallImportEvaluationRetrySkippedItem( + eval_row_id=eval_row.id, + reason="source_row_missing", + ) + ) + continue + targets.append((eval_row, source_row)) + + return targets, skipped + + +@router.post( + "/{eval_id}/retry", + response_model=CallImportEvaluationRetryResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="retryCallImportEvaluation", +) +async def retry_call_import_evaluation( + call_import_id: UUID, + eval_id: UUID, + payload: Optional[CallImportEvaluationRetryRequest] = Body(default=None), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationRetryResponse: + """Re-enqueue failed rows in an evaluation run. + + Default behavior (no body) is "retry every row that failed". Pass + ``eval_row_ids`` to scope the retry to a specific subset (e.g. the + single row a user clicked in the UI). Rows that are still + in-flight or already completed are returned in ``skipped`` rather + than re-enqueued, so this endpoint is always safe to call. + + When ``metric_ids`` is set in the payload, this is a **metric- + subset retry**: only the listed metrics are recomputed (and merged + into the row's existing ``metric_scores`` — other metrics' values + are preserved). The route auto-flips ``include_completed=True`` in + that case so previously-successful rows are eligible for re- + scoring; without it the call would no-op because every row would + be skipped as ``completed``. + + The worker contract is the same as the create endpoint: + ``evaluate_call_import_row_task(eval_row_id, [restricted_metric_ids])``. + When the run is configured for diarised transcripts and the row's + diarised transcript is missing, we chain through + ``transcribe_call_import_row_task`` first — matching the + auto-transcribe behavior of POST ``/evaluations``. + """ + del api_key + call_import = _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + requested_ids = payload.eval_row_ids if payload else None + # Metric-subset retry: validate that every metric is something this + # run actually scored. Empty list is rejected too — callers that + # want a full re-run should omit the field entirely. + # + # ``selected_metric_ids`` holds the LEAVES only (children for + # hierarchical / category metrics, standalone metrics otherwise) — + # see ``leaf_metric_ids`` in :func:`create_call_import_evaluation`. + # Parent IDs for hierarchical metrics live separately in + # ``selected_metric_groups`` (``{parent_id: [child_ids]}``) so the + # UI can reconstruct the tree without round-tripping through the + # metric table. + # + # The Re-run-metrics modal surfaces PARENTS for hierarchical + # metrics (it suppresses individual children via + # ``childrenInGroups`` in ``CallImportEvaluationDetail.tsx``), so a + # naive ``metric_ids ⊆ selected_metric_ids`` check rejects every + # parent-ID request with a misleading "unknown ids" 400. We accept + # both shapes here and then EXPAND any parent IDs into + # ``{parent_id, *child_ids}`` so the downstream helpers see the + # full set of keys that need clearing + the full set of leaves + # that need re-scoring. + metric_ids: Optional[List[UUID]] = ( + payload.metric_ids if payload else None + ) + if metric_ids is not None: + if not metric_ids: + raise HTTPException( + status_code=400, + detail=( + "metric_ids must be a non-empty list. Omit the " + "field to re-run all metrics." + ), + ) + + leaf_set: Set[str] = { + str(item).lower() + for item in (evaluation.selected_metric_ids or []) + } + # ``selected_metric_groups`` is a dict ``{parent_id_str: + # [child_id_str, ...]}`` (see line ~487 in + # ``create_call_import_evaluation``). We tolerate stale data + # (string / UUID / non-dict) without crashing the retry path — + # if it's malformed we just treat it as "no parents" and fall + # back to the leaf-only check. + groups_raw = ( + evaluation.selected_metric_groups + if isinstance(evaluation.selected_metric_groups, dict) + else {} + ) + parent_to_children_str: Dict[str, List[str]] = {} + for parent_key, children_raw in groups_raw.items(): + if not isinstance(children_raw, (list, tuple)): + continue + children_norm = [ + str(c).lower() for c in children_raw if c is not None + ] + parent_to_children_str[str(parent_key).lower()] = children_norm + parent_set = set(parent_to_children_str.keys()) + + unknown = [ + mid for mid in metric_ids + if str(mid).lower() not in leaf_set + and str(mid).lower() not in parent_set + ] + if unknown: + raise HTTPException( + status_code=400, + detail=( + "metric_ids must be a subset of this evaluation's " + f"selected metrics; unknown ids: {[str(u) for u in unknown]}." + ), + ) + + # Expand parent IDs into ``{parent, *children}`` so: + # * ``_reset_eval_row_for_retry`` strips BOTH the parent + # entry (with ``chosen_child_id`` / rationale) AND every + # per-child boolean entry that the LLM evaluator wrote + # under each child's ID (see + # ``app/workers/tasks/helpers/llm_evaluation.py`` lines + # 1584 and 1649). + # * ``_enqueue_eval_rows_with_optional_transcribe`` → + # ``evaluate_call_import_row_task`` filters the work-list + # off ``selected_metric_ids`` (leaves), so we MUST hand it + # the child IDs for the parent to actually get re-scored. + # Leaves pass through unchanged. + expanded: List[UUID] = [] + seen: Set[str] = set() + for mid in metric_ids: + mid_norm = str(mid).lower() + children_str = parent_to_children_str.get(mid_norm) + if children_str is not None: + # Parent: include the parent ID itself (so the parent + # entry in ``metric_scores`` is also cleared) and all + # of its children. + candidates = [mid_norm, *children_str] + else: + candidates = [mid_norm] + for candidate in candidates: + if candidate in seen: + continue + try: + expanded.append(UUID(candidate)) + except (TypeError, ValueError): + # Defensive: skip junk values rather than 500. + continue + seen.add(candidate) + metric_ids = expanded + + # ``include_completed`` is auto-enabled when the caller asked for a + # metric subset (otherwise the metric-subset retry would always + # no-op on a green run, which is the whole reason this feature + # exists). The explicit payload flag wins for full-row retries. + include_completed = bool( + (payload.include_completed if payload else False) + or (metric_ids is not None) + ) + + transcribe_overwrite = bool( + payload.transcribe_overwrite if payload else False + ) + + skipped: List[CallImportEvaluationRetrySkippedItem] = [] + if requested_ids is None: + from app.db_sharding.eval_rows import count_evaluation_rows_for_run + from app.db_sharding.sessions import is_sharding_enabled + + if is_sharding_enabled(): + statuses = ( + ["failed", "completed"] if include_completed else ["failed"] + ) + target_count = count_evaluation_rows_for_run( + db, eval_id, statuses=statuses + ) + else: + from sqlalchemy import func + + count_query = db.query(func.count(CallImportEvaluationRow.id)).filter( + CallImportEvaluationRow.evaluation_id == eval_id + ) + if include_completed: + count_query = count_query.filter( + CallImportEvaluationRow.status.in_(["failed", "completed"]) + ) + else: + count_query = count_query.filter( + CallImportEvaluationRow.status == "failed" + ) + target_count = int(count_query.scalar() or 0) + if target_count == 0: + return CallImportEvaluationRetryResponse( + requeued=0, + transcribe_requeued=0, + skipped=skipped, + ) + else: + targets, skipped = _gather_retry_targets( + db, + evaluation, + requested_ids, + include_completed=include_completed, + ) + if not targets: + return CallImportEvaluationRetryResponse( + requeued=0, + transcribe_requeued=0, + skipped=skipped, + ) + target_count = len(targets) + + # Apply LLM / STT overrides BEFORE enqueueing so the persisted run + # config is correct by the time the worker reads it. + if payload is not None: + _apply_retry_overrides(db, evaluation, organization_id, payload) + _apply_telephony_retry_overrides( + db, + call_import=call_import, + organization_id=organization_id, + payload=payload, + ) + + evaluation.error_message = None + evaluation.finished_at = None + evaluation.status = "running" + if not evaluation.started_at: + from datetime import datetime, timezone + + evaluation.started_at = datetime.now(timezone.utc) + + _claim_evaluation_bulk_operation(eval_id, "retry") + stamp_evaluation_actor(evaluation, principal) + db.commit() + + from app.workers.tasks.call_import_bulk_ops import ( + retry_call_import_evaluation_task, + ) + + retry_call_import_evaluation_task.delay( + str(eval_id), + { + "eval_row_ids": [str(rid) for rid in requested_ids] + if requested_ids + else None, + "metric_ids": [str(mid) for mid in metric_ids] if metric_ids else None, + "include_completed": include_completed, + "transcribe_overwrite": transcribe_overwrite, + }, + ) + + return CallImportEvaluationRetryResponse( + requeued=target_count, + transcribe_requeued=0, + skipped=skipped, + ) + + +@router.post( + "/{eval_id}/rows/{eval_row_id}/retry", + response_model=CallImportEvaluationRowResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="retryCallImportEvaluationRow", +) +async def retry_call_import_evaluation_row( + call_import_id: UUID, + eval_id: UUID, + eval_row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportEvaluationRowResponse: + """Re-enqueue a single failed evaluation row. + + Convenience wrapper around ``retry_call_import_evaluation`` for the + "Retry this row" affordance in the row table. Returns the + refreshed row so the UI can update its badge immediately, without + waiting for the next polling tick. + """ + del api_key + _require_import(db, call_import_id, organization_id) + + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException( + status_code=404, detail="Call import evaluation not found" + ) + + _require_no_evaluation_bulk_operation(eval_id) + + from app.db_sharding.eval_rows import ( + evaluation_row_session, + find_evaluation_row_in_run, + ) + from app.db_sharding.sessions import is_sharding_enabled + + eval_row, _source_stub = find_evaluation_row_in_run(db, eval_id, eval_row_id) + if eval_row is None: + raise HTTPException( + status_code=404, detail="Evaluation row not found in this run" + ) + + if eval_row.status in {"pending", "running"}: + raise HTTPException( + status_code=409, + detail=( + "This row is still in progress — wait for it to finish " + "before retrying." + ), + ) + + targets, _ = _gather_retry_targets(db, evaluation, [eval_row.id]) + if not targets: + raise HTTPException( + status_code=409, + detail=( + "This row cannot be retried in its current state " + f"(status={eval_row.status})." + ), + ) + + if is_sharding_enabled(): + with evaluation_row_session(eval_row_id) as ( + row_db, + _catalog_db, + eval_row, + source_row, + _shard_id, + ): + _prepare_source_row_for_retry(source_row, transcribe_overwrite=False) + _reset_eval_row_for_retry(eval_row) + row_db.commit() + targets = [(eval_row, source_row)] + else: + for er, source_row in targets: + _prepare_source_row_for_retry(source_row, transcribe_overwrite=False) + _reset_eval_row_for_retry(er) + + evaluation.error_message = None + evaluation.finished_at = None + evaluation.status = "running" + if not evaluation.started_at: + from datetime import datetime, timezone + + evaluation.started_at = datetime.now(timezone.utc) + db.flush() + _rollup_evaluation_status(evaluation, db) + stamp_evaluation_actor(evaluation, principal) + db.commit() + + try: + _enqueue_eval_rows_with_optional_transcribe(db, evaluation, targets) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to re-enqueue retry for evaluation row {}", eval_row_id + ) + if is_sharding_enabled(): + with evaluation_row_session(eval_row_id) as ( + row_db, + _catalog_db, + eval_row, + _source_row, + _shard_id, + ): + eval_row.status = "failed" + eval_row.error_message = f"Failed to re-enqueue retry: {exc}" + row_db.commit() + else: + eval_row.status = "failed" + eval_row.error_message = f"Failed to re-enqueue retry: {exc}" + _rollup_evaluation_status(evaluation, db) + db.commit() + raise HTTPException( + status_code=500, + detail=f"Failed to re-enqueue retry: {exc}", + ) + + if is_sharding_enabled(): + with evaluation_row_session(eval_row_id) as ( + _row_db, + _catalog_db, + eval_row, + source_row, + _shard_id, + ): + return _to_evaluation_row_response(eval_row, source_row, evaluation) + + db.refresh(eval_row) + source_row = targets[0][1] + return _to_evaluation_row_response(eval_row, source_row, evaluation) + + +from app.core.auth.capabilities import EVALS_RUN, EVALS_VIEW, REPORTS_GENERATE +from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities + +apply_workspace_route_capabilities( + router, + view_capability=EVALS_VIEW, + manage_capability=EVALS_RUN, + run_capability=EVALS_RUN, + report_capability=REPORTS_GENERATE, +) diff --git a/app/api/v1/routes/call_imports.py b/app/api/v1/routes/call_imports.py index e7461504..3b7bab55 100644 --- a/app/api/v1/routes/call_imports.py +++ b/app/api/v1/routes/call_imports.py @@ -1,3947 +1,4016 @@ -"""CSV-driven call import routes. - -Users upload a CSV plus a per-batch column mapping (CSV header -> system -field). The backend persists a CallImport batch + one CallImportRow per -line, then fans the rows out to the Celery ``imports`` queue where each -row is downloaded using the telephony credential pinned on the batch. -Exotel credentialed imports require a ``recording_url`` on every row; -direct-URL imports (no credential) also require a mapped recording URL. -""" - -from __future__ import annotations - -import csv -import io -import json -import re -from dataclasses import dataclass, field -from datetime import date, datetime, time, timedelta -from typing import Any, Dict, Iterable, List, Optional, Tuple -from uuid import UUID, uuid4 - -from fastapi import APIRouter, Body, BackgroundTasks, Depends, File, Form, HTTPException, Query, Response, UploadFile, status -from loguru import logger -from sqlalchemy import desc, func, or_ -from sqlalchemy.orm import Session - -from app.config import settings -from app.core.auth.rbac import require_admin -from app.database import get_db -from app.db_sharding.sessions import is_sharding_enabled -from app.dependencies import ( - get_api_key, - get_organization_id, - get_workspace_id, - require_enterprise_feature, -) -from app.services.billing.flexprice_service import record_call_import_batch_created -from app.services.call_imports.dispatch_diagnostics import ( - build_call_import_dispatch_diagnostics, -) -from app.models.database import ( - CallImport, - CallImportRow, - CallImportSchema, - CallImportSchemaParameter, - CallImportTag, - TelephonyIntegration, -) -from app.models.enums import ( - CallImportParameterType, - CallImportRowStatus, - CallImportStatus, -) -from app.models.schemas import ( - CallImportCancelDiarisationRequest, - CallImportCancelDiarisationResponse, - CallImportDetailResponse, - CallImportDeleteResponse, - CallImportDiarisationPromptDefaultResponse, - CallImportDispatchDiagnosticsResponse, - CallImportInsightsMetric, - CallImportInsightsResponse, - CallImportInsightsRunPoint, - CallImportListResponse, - CallImportMappingUpdate, - CallImportMetricAggregate, - CallImportPreviewResponse, - CallImportPreviewSheet, - CallImportRetryFailedRowsRequest, - CallImportRetryFailedRowsResponse, - CallImportResponse, - CallImportRowIdsResponse, - CallImportRowBulkDelete, - CallImportRowBulkDeleteResponse, - CallImportRowResponse, - CallImportStartRequest, - CallImportTranscribeRequest, - CallImportTranscribeResponse, - CallImportUpdate, - CallImportUploadResponse, -) - - -router = APIRouter( - prefix="/call-imports", - tags=["Call Imports"], - dependencies=[Depends(require_enterprise_feature("call_imports"))], -) - - -@dataclass(frozen=True) -class CallImportParseSkip: - """One source row excluded during CSV/Excel parse (identity / recording URL).""" - - source_row: int - reason: str - message: str - - -@dataclass -class CallImportParseResult: - rows: List[Dict[str, Any]] = field(default_factory=list) - skipped: List[CallImportParseSkip] = field(default_factory=list) - - -def parse_skips_to_json(skips: List[CallImportParseSkip]) -> List[Dict[str, Any]]: - """Persistable JSON shape for ``CallImport.source_row_skips``.""" - return [ - { - "source_row": item.source_row, - "reason": item.reason, - "message": item.message, - } - for item in skips - ] - - -def _normalize_dataset(raw: Optional[str]) -> Optional[str]: - """Trim and treat empty strings as 'no dataset' (NULL).""" - if raw is None: - return None - cleaned = raw.strip() - return cleaned or None - - -def _serialize_call_import(db: Session, call_import: CallImport) -> CallImportResponse: - """Catalog parent fields; counters come from SQL rollup (not Redis merge).""" - from app.services.call_imports.bulk_ops import rollup_call_import_batch_status - from app.services.call_imports.progress_counters import ( - clear_import_progress_redis, - read_import_progress, - ) - - redis_completed, redis_failed = read_import_progress(call_import.id) - if ( - redis_completed - or redis_failed - or int(call_import.completed_rows or 0) > int(call_import.total_rows or 0) - or int(call_import.failed_rows or 0) > int(call_import.total_rows or 0) - ): - rollup_call_import_batch_status(db, call_import) - db.flush() - - clear_import_progress_redis(call_import.id) - db.refresh(call_import) - total = int(call_import.total_rows or 0) - completed = min(int(call_import.completed_rows or 0), total) if total else int( - call_import.completed_rows or 0 - ) - failed = min(int(call_import.failed_rows or 0), total) if total else int( - call_import.failed_rows or 0 - ) - base = CallImportResponse.model_validate(call_import) - return base.model_copy(update={"completed_rows": completed, "failed_rows": failed}) - - -def _resolve_tags( - db: Session, organization_id: UUID, tag_ids: Optional[List[UUID]] -) -> List[CallImportTag]: - """Look up tag rows by id, scoped to the organization. - - Raises HTTPException(400) if any id is unknown for the org. - """ - if not tag_ids: - return [] - rows = ( - db.query(CallImportTag) - .filter( - CallImportTag.organization_id == organization_id, - CallImportTag.id.in_(tag_ids), - ) - .all() - ) - found_ids = {row.id for row in rows} - missing = [str(tag_id) for tag_id in tag_ids if tag_id not in found_ids] - if missing: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Unknown call_import_tag id(s): {missing}", - ) - return rows - - -MAX_UPLOAD_BYTES = 15 * 1024 * 1024 # 15 MB upload cap (CSV or Excel) - -# File extensions accepted by the upload + preview endpoints. Keep in -# lockstep with the frontend ``accept`` attribute on the file picker. -CSV_EXTENSIONS = (".csv",) -XLSX_EXTENSIONS = (".xlsx", ".xlsm") -ALLOWED_EXTENSIONS = CSV_EXTENSIONS + XLSX_EXTENSIONS - -AUDIO_CONTENT_TYPES = { - "wav": "audio/wav", - "mp3": "audio/mpeg", - "flac": "audio/flac", - "m4a": "audio/mp4", -} - - -def _file_format(filename: Optional[str]) -> Optional[str]: - """Classify ``filename`` as ``'csv'`` / ``'xlsx'`` or ``None`` if unsupported.""" - if not filename: - return None - name = filename.lower() - if name.endswith(CSV_EXTENSIONS): - return "csv" - if name.endswith(XLSX_EXTENSIONS): - return "xlsx" - return None - - -def _audio_extension(filename: Optional[str]) -> Optional[str]: - """Return the validated lower-case extension for a manual recording.""" - if not filename or "." not in filename: - return None - ext = filename.rsplit(".", 1)[-1].lower().strip() - allowed = {fmt.lower().lstrip(".") for fmt in settings.ALLOWED_AUDIO_FORMATS} - return ext if ext in allowed else None - - -def _audio_content_type(ext: str, upload_content_type: Optional[str]) -> str: - """Prefer the browser-supplied audio content type, with a safe fallback.""" - supplied = (upload_content_type or "").strip() - if supplied and supplied != "application/octet-stream": - return supplied - return AUDIO_CONTENT_TYPES.get(ext.lower(), "application/octet-stream") - - -def _audio_s3_key( - organization_id: UUID, call_import_id: UUID, row_id: UUID, ext: str -) -> str: - """Build the canonical S3 key for a manually uploaded recording.""" - from app.services.storage.s3_service import s3_service - - return ( - f"{s3_service.prefix}organizations/{organization_id}/call_imports/" - f"{call_import_id}/{row_id}.{ext}" - ) - - -def _filename_stem(filename: Optional[str]) -> str: - """Extract a cross-platform filename stem from an UploadFile name.""" - raw = (filename or "").strip() - basename = re.split(r"[\\/]", raw)[-1] if raw else "" - if "." in basename: - basename = basename.rsplit(".", 1)[0] - return basename.strip() - - -def _sanitize_conversation_id(raw: str) -> str: - """Turn a filename stem into a stable conversation_id.""" - cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", raw.strip()) - cleaned = re.sub(r"_+", "_", cleaned).strip("._-") - return (cleaned or "recording")[:255] - - -def _dedupe_conversation_id( - base: str, counts: Dict[str, int] -) -> str: - """Make conversation ids unique within one manual upload batch.""" - count = counts.get(base, 0) + 1 - counts[base] = count - if count == 1: - return base - suffix = f"-{count}" - return f"{base[: 255 - len(suffix)]}{suffix}" - - -def _normalize_header(name: str) -> str: - return (name or "").strip().lower() - - -def _header_lookup(fieldnames: List[str]) -> Dict[str, str]: - """Map normalized header -> original header for case-insensitive lookup.""" - return {_normalize_header(h): h for h in fieldnames or []} - - -def _resolve_mapped_header( - mapping_value: Optional[str], header_lookup: Dict[str, str] -) -> Optional[str]: - """Translate a user-supplied CSV header into the actual column key. - - The frontend sends headers exactly as they appear in the source file, - but we still normalize on the server so trailing whitespace / casing - doesn't break matching. Returns the canonical fieldname or ``None`` - if not present in the file. - """ - if not mapping_value: - return None - return header_lookup.get(_normalize_header(mapping_value)) - - -def _xlsx_cell_to_str(value: Any) -> str: - """Coerce an openpyxl cell value to the string the rest of the - pipeline expects. - - openpyxl returns native Python types (int, float, datetime, bool, - None). The CSV path always works with strings, so we mirror that: - integers stringify cleanly (no ``.0`` suffix on whole-number floats), - datetimes use ISO-8601, booleans use SQL-style ``TRUE`` / ``FALSE``. - """ - if value is None: - return "" - if isinstance(value, bool): - return "TRUE" if value else "FALSE" - if isinstance(value, int): - return str(value) - if isinstance(value, float): - if value.is_integer(): - return str(int(value)) - return str(value) - if isinstance(value, datetime): - return value.isoformat() - if isinstance(value, date): - return value.isoformat() - if isinstance(value, time): - return value.isoformat() - if isinstance(value, timedelta): - return str(value) - return str(value) - - -def _parse_recording_date_cell(cell: str) -> date: - """Parse day-first dates with one/two digit day-month parts.""" - match = re.fullmatch(r"\s*(\d{1,2})[/-](\d{1,2})[/-](\d{4})\s*", cell) - if match: - day, month, year = (int(part) for part in match.groups()) - return date(year, month, day) - - # Native Excel date cells arrive from ``_xlsx_cell_to_str`` as ISO - # datetimes (e.g. ``2026-01-04T00:00:00``). Accept that resolved date, - # while keeping plain ISO dates rejected for hand-entered text/CSV cells. - if "T" in cell: - return datetime.fromisoformat(cell.replace("Z", "+00:00")).date() - - raise ValueError("expected D/M/YYYY or D-M-YYYY") - - -def _coerce_parameter_value( - raw: str, - param_type: CallImportParameterType, - *, - row_idx: int, - param_name: str, -) -> Any: - """Validate + coerce a single CSV cell against its declared type. - - Returns the typed Python value to surface in ``raw_columns``. Empty - strings are returned as ``None`` regardless of the parameter type so - optional cells stay null end-to-end. Coercion failures raise a - 400 with a row-anchored message. - """ - cell = (raw or "").strip() - if not cell: - return None - - if param_type == CallImportParameterType.CONVERSATION_ID: - return cell - if param_type == CallImportParameterType.RECORDING_URL: - # Recording URLs are exercised by the worker (which downloads - # them); we only do a light "starts with http" check here so a - # paste-error surfaces immediately at upload time. - lower = cell.lower() - if not (lower.startswith("http://") or lower.startswith("https://")): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - "valid recording URL (must start with http:// or https://)." - ), - ) - return cell - if param_type == CallImportParameterType.RECORDING_DATE: - try: - parsed_date = _parse_recording_date_cell(cell) - except ValueError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - f"valid recording date ({cell!r}); expected day-first " - "D/M/YYYY or D-M-YYYY." - ), - ) - return parsed_date.strftime("%d/%m/%Y") - if param_type == CallImportParameterType.TRANSCRIPT: - return cell - if param_type == CallImportParameterType.TEXT: - return cell - if param_type == CallImportParameterType.NUMBER: - try: - value = float(cell) - except ValueError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - f"valid number ({cell!r})." - ), - ) - if value.is_integer(): - return int(value) - return value - if param_type == CallImportParameterType.BOOLEAN: - truthy = {"true", "yes", "y", "1", "t"} - falsy = {"false", "no", "n", "0", "f"} - norm = cell.lower() - if norm in truthy: - return True - if norm in falsy: - return False - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - f"valid boolean ({cell!r})." - ), - ) - if param_type == CallImportParameterType.DATETIME: - try: - parsed = datetime.fromisoformat(cell.replace("Z", "+00:00")) - except ValueError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - f"valid ISO-8601 date/time ({cell!r})." - ), - ) - return parsed.isoformat() - if param_type == CallImportParameterType.URL: - lower = cell.lower() - if not (lower.startswith("http://") or lower.startswith("https://")): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {row_idx + 1}: value for '{param_name}' is not a " - "valid URL (must start with http:// or https://)." - ), - ) - return cell - # Unknown types: store as text and let the next migration catch up. - return cell - - -def _recording_url_cell_is_valid_http(raw: str) -> bool: - cell = (raw or "").strip() - if not cell: - return False - lower = cell.lower() - return lower.startswith("http://") or lower.startswith("https://") - - -def _parameter_is_required(param: CallImportSchemaParameter) -> bool: - """Return whether a schema parameter must be mapped on every upload.""" - if param.is_required: - return True - try: - param_type = CallImportParameterType(param.type) - except ValueError: - return False - return param_type in ( - CallImportParameterType.CONVERSATION_ID, - CallImportParameterType.RECORDING_URL, - ) - - -def _apply_schema_mapping( - fieldnames: List[str], - rows_iter: Iterable[Dict[str, str]], - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, str], - skipped_columns: List[str], - *, - source_label: str = "CSV", - validate_only: bool = False, -) -> CallImportParseResult: - """Schema-driven row projection: parameter -> CSV header -> typed value. - - Validates that every required schema parameter is mapped to a CSV - header that actually exists in the file, and that every CSV header - is either mapped to a parameter or explicitly listed in - ``skipped_columns``. Returns one dict per non-empty data row with: - - * ``conversation_id`` (str, mandatory) - * ``recording_date`` (Optional[str], DD/MM/YYYY date) - * ``recording_url`` (Optional[str]) - * ``transcript`` (Optional[str]) - * ``parameter_values`` (Dict[str, Any]) of typed values keyed by - parameter name (drives ``raw_columns`` so the export can - reproduce the source). - - ``validate_only=True`` runs the header / mapping / skipped-column - checks (every check that doesn't need to read row data) and then - returns an empty list — used by the MAP stage to validate a - mapping payload against the cached sheet snapshot without - re-fetching the source bytes from S3. - """ - header_lookup = _header_lookup(list(fieldnames)) - - # 1. Look up the conversation_id parameter so we can address it - # directly while building each row. - conv_param = next( - (p for p in parameters if p.type == CallImportParameterType.CONVERSATION_ID), - None, - ) - if conv_param is None: - # The schema invariant should have caught this on create/update, - # but a defensive 400 here keeps us safe against hand-rolled - # API callers that bypassed validation. - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Selected schema is missing the mandatory conversation_id parameter.", - ) - # 2. Resolve every mapped parameter to a canonical fieldname. - # Required parameters MUST resolve; optional ones may resolve to - # None if the user left them blank (no mapping). - canonical_by_param: Dict[str, Optional[str]] = {} - recording_date_param_name: Optional[str] = None - rec_url_param_name: Optional[str] = None - transcript_param_name: Optional[str] = None - for param in parameters: - mapped_header = parameter_mapping.get(param.name) - canonical = ( - _resolve_mapped_header(mapped_header, header_lookup) - if mapped_header - else None - ) - if _parameter_is_required(param) and canonical is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"{source_label} does not contain the column " - f"'{mapped_header or ''}' mapped to required parameter " - f"'{param.name}'." - ), - ) - canonical_by_param[param.name] = canonical - if param.type == CallImportParameterType.RECORDING_DATE: - recording_date_param_name = param.name - elif param.type == CallImportParameterType.RECORDING_URL: - rec_url_param_name = param.name - elif param.type == CallImportParameterType.TRANSCRIPT: - transcript_param_name = param.name - - # 3. Every CSV column must either be mapped to a parameter or - # explicitly skipped. Catches "I forgot to skip the email - # column" gracefully instead of dropping data silently. - mapped_canonicals = {c for c in canonical_by_param.values() if c} - skipped_canonicals = { - _resolve_mapped_header(h, header_lookup) - for h in skipped_columns - } - skipped_canonicals.discard(None) - unhandled = [ - h - for h in fieldnames - if h not in mapped_canonicals and h not in skipped_canonicals - ] - if unhandled: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"{source_label} columns must either be mapped to a schema " - f"parameter or explicitly skipped. Unhandled: {unhandled}." - ), - ) - - conv_canonical = canonical_by_param[conv_param.name] - rec_canonical = ( - canonical_by_param.get(rec_url_param_name) - if rec_url_param_name - else None - ) - recording_date_canonical = ( - canonical_by_param.get(recording_date_param_name) - if recording_date_param_name - else None - ) - transcript_canonical = ( - canonical_by_param.get(transcript_param_name) - if transcript_param_name - else None - ) - - if validate_only: - # MAP-stage validation: every header check above has already - # run; the row loop only matters at IMPORT time. Skip it (and - # the "no data rows" guard at the bottom of the function) so - # the caller gets a clean pass when the mapping is shaped right. - return CallImportParseResult() - - parsed: List[Dict[str, Any]] = [] - skipped: List[CallImportParseSkip] = [] - for idx, row in enumerate(rows_iter): - # Drop fully-blank lines - matches the legacy parser behavior so - # trailing-newline edge cases don't fail an otherwise-good upload. - non_blank = any( - (row.get(c) or "").strip() - for c in mapped_canonicals - if c - ) - if not non_blank: - continue - - source_row = idx + 1 - conv_value = (row.get(conv_canonical) or "").strip() if conv_canonical else "" - if not conv_value: - skipped.append( - CallImportParseSkip( - source_row=source_row, - reason="missing_conversation_id", - message=( - f"Row {source_row} is missing the '{conv_param.name}' " - "(conversation_id) value." - ), - ) - ) - continue - - if rec_canonical and rec_url_param_name: - rec_param = next( - (p for p in parameters if p.name == rec_url_param_name), - None, - ) - if rec_param is not None and _parameter_is_required(rec_param): - rec_raw = (row.get(rec_canonical) or "").strip() - if not rec_raw: - skipped.append( - CallImportParseSkip( - source_row=source_row, - reason="missing_recording_url", - message=( - f"Row {source_row} is missing the required " - f"'{rec_url_param_name}' value." - ), - ) - ) - continue - if not _recording_url_cell_is_valid_http(rec_raw): - skipped.append( - CallImportParseSkip( - source_row=source_row, - reason="invalid_recording_url", - message=( - f"Row {source_row}: value for " - f"'{rec_url_param_name}' is not a valid recording " - "URL (must start with http:// or https://)." - ), - ) - ) - continue - - # Materialize every mapped parameter into the per-row snapshot, - # running per-type coercion so a bad cell aborts the upload - # rather than silently storing garbage. - parameter_values: Dict[str, Any] = {} - row_skipped = False - for param in parameters: - canonical = canonical_by_param[param.name] - if canonical is None: - continue - try: - param_type = CallImportParameterType(param.type) - except ValueError: - param_type = CallImportParameterType.TEXT - coerced = _coerce_parameter_value( - row.get(canonical) or "", - param_type, - row_idx=idx, - param_name=param.name, - ) - if _parameter_is_required(param) and coerced is None: - if param_type == CallImportParameterType.RECORDING_URL: - skipped.append( - CallImportParseSkip( - source_row=source_row, - reason="missing_recording_url", - message=( - f"Row {source_row} is missing the required " - f"'{param.name}' value." - ), - ) - ) - row_skipped = True - break - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Row {source_row} is missing the required " - f"'{param.name}' value." - ), - ) - parameter_values[param.name] = coerced - if row_skipped: - continue - - rec_value = ( - (row.get(rec_canonical) or "").strip() if rec_canonical else "" - ) - transcript_value = ( - (row.get(transcript_canonical) or "").strip() - if transcript_canonical - else "" - ) - recording_date_value = ( - parameter_values.get(recording_date_param_name) - if recording_date_param_name - else None - ) - - parsed.append( - { - "conversation_id": conv_value, - "recording_date": recording_date_value, - "recording_url": rec_value or None, - "transcript": transcript_value or None, - "parameter_values": parameter_values, - } - ) - - if not parsed and not skipped: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"{source_label} did not contain any data rows.", - ) - - return CallImportParseResult(rows=parsed, skipped=skipped) - - -def _raise_if_no_importable_rows( - result: CallImportParseResult, *, source_label: str = "CSV" -) -> None: - """Sync upload / API callers fail fast when every data row was skipped.""" - if result.rows: - return - if result.skipped: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"No importable rows. {len(result.skipped)} row(s) skipped due " - "to missing or invalid conversation ID or recording URL." - ), - ) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"{source_label} did not contain any data rows.", - ) - - -def _parse_csv( - file_bytes: bytes, - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, str], - skipped_columns: List[str], -) -> CallImportParseResult: - """Parse a CSV file using the resolved schema parameters.""" - if not file_bytes: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Uploaded CSV is empty.", - ) - - try: - text_stream = io.StringIO(file_bytes.decode("utf-8-sig")) - except UnicodeDecodeError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="CSV must be UTF-8 encoded.", - ) - - reader = csv.DictReader(text_stream) - if not reader.fieldnames: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="CSV is missing a header row.", - ) - - return _apply_schema_mapping( - list(reader.fieldnames), - reader, - parameters, - parameter_mapping, - skipped_columns, - source_label="CSV", - ) - - -def _open_xlsx_workbook(file_bytes: bytes): - """Open an xlsx/xlsm workbook from in-memory bytes (read-only stream). - - Imports openpyxl lazily so the module loads even in environments that - haven't installed the optional dep yet (e.g. lightweight tooling - images). Surfaces a clean 400 if openpyxl is missing or the file is - not a valid Office Open XML workbook. - """ - if not file_bytes: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Uploaded Excel file is empty.", - ) - try: - from openpyxl import load_workbook # type: ignore - from openpyxl.utils.exceptions import InvalidFileException # type: ignore - except ImportError as exc: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=( - "Excel uploads require the 'openpyxl' package which is " - "not installed in this environment." - ), - ) from exc - - try: - return load_workbook( - io.BytesIO(file_bytes), - read_only=True, - data_only=True, - ) - except InvalidFileException as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"File is not a valid .xlsx workbook: {exc}", - ) from exc - except Exception as exc: # zipfile.BadZipFile etc. - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Could not open Excel workbook: {exc}", - ) from exc - - -def _xlsx_sheet_headers_and_rows( - worksheet, -) -> Tuple[List[str], List[Dict[str, str]]]: - """Read row 1 as headers and the rest as dicts of stringified cells. - - Empty trailing header cells are dropped. Duplicate headers preserve - the first occurrence (matches ``csv.DictReader`` behavior, which - silently drops duplicates). - """ - iterator = worksheet.iter_rows(values_only=True) - try: - header_row = next(iterator) - except StopIteration: - return [], [] - - headers: List[str] = [] - seen: set[str] = set() - for cell in header_row: - name = _xlsx_cell_to_str(cell).strip() - if not name: - # Stop at the first blank header — treats trailing empty - # columns as not part of the table (matches typical Excel - # workbook conventions). - break - norm = name.lower() - if norm in seen: - continue - seen.add(norm) - headers.append(name) - - rows: List[Dict[str, str]] = [] - for row in iterator: - if row is None: - continue - # Pad / truncate to the header length so dict construction is - # stable even when a row has fewer / extra cells than the header. - cells = list(row[: len(headers)]) - if len(cells) < len(headers): - cells.extend([None] * (len(headers) - len(cells))) - if not any(_xlsx_cell_to_str(c).strip() for c in cells): - # Skip fully-blank rows (openpyxl read_only routinely yields - # trailing empties when the worksheet's used range exceeds - # the actual data). - continue - rows.append( - { - header: _xlsx_cell_to_str(value) - for header, value in zip(headers, cells) - } - ) - - return headers, rows - - -def _parse_xlsx( - file_bytes: bytes, - sheet_name: Optional[str], - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, str], - skipped_columns: List[str], -) -> CallImportParseResult: - """Parse a single worksheet from an xlsx/xlsm workbook. - - ``sheet_name`` must match one of the workbook's sheets (case - insensitive whitespace-trimmed match). Returns the same shape as - :func:`_parse_csv` so the upload handler can persist either format - through the same code path. - """ - if not sheet_name or not sheet_name.strip(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is required when uploading an Excel workbook.", - ) - - workbook = _open_xlsx_workbook(file_bytes) - try: - sheet_names = list(workbook.sheetnames) - target_norm = sheet_name.strip().lower() - match = next( - (s for s in sheet_names if s.strip().lower() == target_norm), - None, - ) - if match is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Sheet '{sheet_name}' not found in workbook. " - f"Available sheets: {sheet_names}" - ), - ) - worksheet = workbook[match] - headers, rows = _xlsx_sheet_headers_and_rows(worksheet) - finally: - workbook.close() - - if not headers: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Sheet '{sheet_name}' is missing a header row.", - ) - - return _apply_schema_mapping( - headers, - rows, - parameters, - parameter_mapping, - skipped_columns, - source_label=f"Sheet '{sheet_name}'", - ) - - -def _csv_preview_sheets( - file_bytes: bytes, filename: Optional[str] -) -> List[CallImportPreviewSheet]: - """Build the synthetic single-sheet preview entry for a CSV upload.""" - if not file_bytes: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Uploaded CSV is empty.", - ) - try: - text_stream = io.StringIO(file_bytes.decode("utf-8-sig")) - except UnicodeDecodeError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="CSV must be UTF-8 encoded.", - ) - reader = csv.DictReader(text_stream) - if not reader.fieldnames: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="CSV is missing a header row.", - ) - headers = list(reader.fieldnames) - row_count = 0 - for row in reader: - # Match the parse-time skip: ignore fully blank rows so the - # count the user sees lines up with what /upload will ingest. - if any((v or "").strip() for v in row.values()): - row_count += 1 - - sheet_label = (filename or "sheet1").rsplit("/", 1)[-1] or "sheet1" - return [ - CallImportPreviewSheet( - name=sheet_label, - headers=headers, - row_count=row_count, - ) - ] - - -def _xlsx_preview_sheets(file_bytes: bytes) -> List[CallImportPreviewSheet]: - """List every worksheet in the workbook with its headers and row count.""" - workbook = _open_xlsx_workbook(file_bytes) - sheets: List[CallImportPreviewSheet] = [] - try: - for name in workbook.sheetnames: - worksheet = workbook[name] - headers, rows = _xlsx_sheet_headers_and_rows(worksheet) - sheets.append( - CallImportPreviewSheet( - name=name, - headers=headers, - row_count=len(rows), - ) - ) - finally: - workbook.close() - return sheets - - -def _parse_json_form_field(name: str, raw: Optional[str], default): - """Decode a JSON-encoded form field with a friendly 400 on bad JSON.""" - if raw is None or raw == "": - return default - try: - return json.loads(raw) - except json.JSONDecodeError as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"{name} must be valid JSON: {exc}", - ) - - -# --------------------------------------------------------------------------- -# Shared helpers used by the staged endpoints (UPLOAD / MAP / IMPORT) and the -# legacy one-shot ``POST /upload`` shim. Extracted here so each stage and the -# back-compat path operate on the exact same validation + persistence code. -# --------------------------------------------------------------------------- - - -def _source_content_type(fmt: str) -> str: - """Return the canonical ``Content-Type`` for a parsed file format.""" - if fmt == "csv": - return "text/csv" - if fmt == "xlsx": - return ( - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ) - return "application/octet-stream" - - -def _source_s3_key( - organization_id: UUID, call_import_id: UUID, fmt: str -) -> str: - """Build the canonical S3 key for an upload's source file. - - Mirrors the per-row recording key convention used by - ``process_call_import_row`` so a single prefix sweep on delete still - cleans up both the source artefact and every fetched recording. - """ - from app.services.storage.s3_service import s3_service - - ext = "xlsx" if fmt == "xlsx" else "csv" - return ( - f"{s3_service.prefix}organizations/{organization_id}/call_imports/" - f"{call_import_id}/source.{ext}" - ) - - -def _build_available_sheets( - file_bytes: bytes, fmt: str, filename: Optional[str] -) -> List[CallImportPreviewSheet]: - """Snapshot of sheets + headers cached on the batch at UPLOAD time.""" - if fmt == "csv": - return _csv_preview_sheets(file_bytes, filename) - return _xlsx_preview_sheets(file_bytes) - - -def _resolve_schema( - db: Session, - organization_id: UUID, - workspace_id: UUID, - schema_id: UUID, -) -> CallImportSchema: - """Fetch + validate a schema row in the active workspace. - - Eager-loads ``parameters`` so callers can iterate without re-querying. - """ - from sqlalchemy.orm import selectinload as _selectinload - - schema = ( - db.query(CallImportSchema) - .options(_selectinload(CallImportSchema.parameters)) - .filter( - CallImportSchema.id == schema_id, - CallImportSchema.organization_id == organization_id, - CallImportSchema.workspace_id == workspace_id, - ) - .first() - ) - if not schema: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Call import schema not found in the active workspace.", - ) - if not list(schema.parameters): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Selected schema has no parameters defined.", - ) - return schema - - -def _validate_direct_url_import_ready( - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, Any], -) -> None: - """Ensure direct-URL import has a mapped recording_url column.""" - rec_url_param = next( - ( - p - for p in parameters - if p.type == CallImportParameterType.RECORDING_URL.value - ), - None, - ) - if rec_url_param is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Direct URL import requires a schema parameter of type " - "'recording_url'." - ), - ) - mapped_header = (parameter_mapping or {}).get(rec_url_param.name) - if not (mapped_header or "").strip(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Direct URL import requires the 'recording_url' parameter to " - "be mapped to a source column." - ), - ) - - -def _validate_exotel_import_ready( - parameters: List[CallImportSchemaParameter], - parameter_mapping: Dict[str, Any], -) -> None: - """Ensure Exotel credentialed import has a mapped recording_url column.""" - rec_url_param = next( - ( - p - for p in parameters - if p.type == CallImportParameterType.RECORDING_URL.value - ), - None, - ) - if rec_url_param is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Exotel import requires a schema parameter of type " - "'recording_url'." - ), - ) - mapped_header = (parameter_mapping or {}).get(rec_url_param.name) - if not (mapped_header or "").strip(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Exotel import requires the 'recording_url' parameter to " - "be mapped to a source column." - ), - ) - - -def _resolve_telephony_integration( - db: Session, - organization_id: UUID, - telephony_integration_id: UUID, - provider: str, -) -> TelephonyIntegration: - """Fetch + validate a telephony credential against the requested provider.""" - integration = ( - db.query(TelephonyIntegration) - .filter( - TelephonyIntegration.id == telephony_integration_id, - TelephonyIntegration.organization_id == organization_id, - ) - .first() - ) - if not integration: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Telephony credential not found for this organization.", - ) - if (integration.provider or "").lower() != provider.lower(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Selected credential is for provider '{integration.provider}', " - f"but request specified '{provider}'." - ), - ) - if not integration.is_active: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Selected telephony credential is inactive.", - ) - return integration - - -def _clean_parameter_mapping( - mapping_payload: Any, - parameters: List[CallImportSchemaParameter], - schema_name: str, -) -> Dict[str, str]: - """Trim values and drop empties; reject unknown parameter names. - - Accepts an already-decoded value (dict-shaped) so the same helper - works for the JSON-form upload path and the JSON-body PATCH path. - """ - if not isinstance(mapping_payload, dict) or not all( - isinstance(k, str) and (v is None or isinstance(v, str)) - for k, v in mapping_payload.items() - ): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "parameter_mapping must be an object of " - "{parameter_name: csv_header}." - ), - ) - - valid_param_names = {p.name for p in parameters} - cleaned: Dict[str, str] = {} - for raw_name, raw_header in mapping_payload.items(): - if raw_name not in valid_param_names: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"parameter_mapping references unknown parameter " - f"'{raw_name}' on schema '{schema_name}'." - ), - ) - header = (raw_header or "").strip() - if header: - cleaned[raw_name] = header - return cleaned - - -def _clean_skipped_columns(skipped_payload: Any) -> List[str]: - """Dedupe (case-insensitively) and drop blanks; preserve original casing.""" - if not isinstance(skipped_payload, list) or not all( - isinstance(item, str) for item in skipped_payload - ): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="skipped_columns must be a list of header strings.", - ) - cleaned: List[str] = [] - seen: set[str] = set() - for item in skipped_payload: - norm = _normalize_header(item) - if not norm or norm in seen: - continue - seen.add(norm) - cleaned.append(item) - return cleaned - - -def _parse_source_file( - file_bytes: bytes, - fmt: str, - sheet_name: Optional[str], - parameters: List[CallImportSchemaParameter], - cleaned_mapping: Dict[str, str], - cleaned_skipped: List[str], -) -> CallImportParseResult: - """Run the format-appropriate parser against a buffer of file bytes.""" - if fmt == "csv": - return _parse_csv(file_bytes, parameters, cleaned_mapping, cleaned_skipped) - return _parse_xlsx( - file_bytes, sheet_name, parameters, cleaned_mapping, cleaned_skipped - ) - - -def _materialize_rows( - db: Session, - call_import: CallImport, - parsed_rows: List[Dict[str, Any]], - organization_id: UUID, -) -> List[CallImportRow]: - """Insert one ``CallImportRow`` per parsed row, returning the new models.""" - row_models: List[CallImportRow] = [] - for idx, row in enumerate(parsed_rows): - # Stamp ``transcript_source='csv'`` when the upload actually - # provided a transcript so the UI badge ("From CSV") works from - # day one. Blank cells stay NULL so the row reads as "no - # production transcript yet". - csv_transcript = row["transcript"] - row_model = CallImportRow( - call_import_id=call_import.id, - organization_id=organization_id, - workspace_id=call_import.workspace_id, - row_index=idx, - conversation_id=row["conversation_id"], - recording_date=( - _parse_recording_date_cell(row["recording_date"]) - if row.get("recording_date") - else None - ), - recording_url=row["recording_url"], - transcript=csv_transcript, - transcript_source=( - "csv" if csv_transcript and csv_transcript.strip() else None - ), - raw_columns=row["parameter_values"] or None, - status=CallImportRowStatus.PENDING, - ) - db.add(row_model) - row_models.append(row_model) - return row_models - - -def _enqueue_row_tasks( - db: Session, - call_import: CallImport, - row_models: List[CallImportRow], -) -> None: - """Schedule fair round-robin dispatch for pending import rows.""" - del db, call_import, row_models - from app.workers.concurrency.fair_import_dispatch import ( - schedule_fair_import_dispatch, - ) - - schedule_fair_import_dispatch(max_workspace_turns=999) - - -def _ensure_blob_storage_enabled() -> None: - """Hard-fail UPLOAD if cloud blob storage isn't configured (no local fallback).""" - from app.services.storage.s3_service import s3_service - - if not s3_service.is_enabled(): - err = ( - s3_service.get_status_message() - or "Cloud blob storage is not enabled or not configured" - ) - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail=( - "Call uploads require cloud blob storage so the file can be " - f"persisted between stages: {err}" - ), - ) - - -def _validate_sheet_choice( - fmt: str, - sheet_name: Optional[str], - available_sheets: Optional[List[Dict[str, Any]]], -) -> Optional[str]: - """Normalize / validate ``sheet_name`` against the persisted snapshot. - - Returns the canonical sheet name (matching the workbook's casing) - so downstream parsing addresses the right worksheet. - """ - if fmt == "csv": - if sheet_name and sheet_name.strip(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is not applicable to CSV uploads.", - ) - return None - - cleaned = (sheet_name or "").strip() or None - if cleaned is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is required when the source is an Excel workbook.", - ) - - if not available_sheets: - # Nothing to validate against (e.g. legacy batch without snapshot); - # let downstream parsing error out instead of silently importing. - return cleaned - - target = cleaned.strip().lower() - for entry in available_sheets: - name = entry.get("name") if isinstance(entry, dict) else None - if isinstance(name, str) and name.strip().lower() == target: - return name - sheet_names = [ - entry.get("name") for entry in available_sheets if isinstance(entry, dict) - ] - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Sheet '{cleaned}' not found in the staged file. " - f"Available sheets: {sheet_names}" - ), - ) - - -def _tag_response_payload(tags: Optional[List[CallImportTag]]) -> List[Dict[str, Any]]: - """Shape a CallImport's tag relationship for the upload response.""" - return [ - { - "id": tag.id, - "name": tag.name, - "color": tag.color, - "created_at": tag.created_at, - "updated_at": tag.updated_at, - } - for tag in (tags or []) - ] - - -@router.post( - "/preview", - response_model=CallImportPreviewResponse, - operation_id="previewCallImportFile", -) -async def preview_call_import_file( - file: UploadFile = File(...), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportPreviewResponse: - """Inspect an uploaded CSV / Excel file and return its sheets + headers. - - Drives the column-mapping UI without forcing the frontend to parse - CSV / xlsx itself — keeps client and server in lockstep on quoted - fields, encodings, and Excel cell coercion. CSVs return a single - synthetic sheet named after the filename; Excel workbooks return one - entry per worksheet (in workbook order). - """ - del api_key, organization_id, workspace_id, db # auth only - - fmt = _file_format(file.filename) - if fmt is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Unsupported file format. Allowed extensions: " - f"{', '.join(ALLOWED_EXTENSIONS)}." - ), - ) - - file_bytes = await file.read() - if len(file_bytes) > MAX_UPLOAD_BYTES: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", - ) - - if fmt == "csv": - sheets = _csv_preview_sheets(file_bytes, file.filename) - else: - sheets = _xlsx_preview_sheets(file_bytes) - - return CallImportPreviewResponse(format=fmt, sheets=sheets) - - -@router.post( - "", - response_model=CallImportResponse, - status_code=status.HTTP_201_CREATED, - operation_id="createCallImport", -) -async def create_call_import( - file: UploadFile = File( - ..., - description="CSV / Excel file to stage. Persisted to S3 between stages.", - ), - dataset: str = Form( - ..., - description=( - "Required free-text dataset label. Collected up-front so the " - "batch is filterable from the moment it lands." - ), - ), - tag_ids: Optional[List[UUID]] = Form( - None, - description="Optional list of CallImportTag ids to attach to the new batch.", - ), - schema_id: Optional[UUID] = Form( - None, - description=( - "Optional schema pre-pick. The user can still change it during " - "the MAP stage; provided here only so the detail page can pre-" - "select the schema dropdown." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportResponse: - """UPLOAD stage of the staged call-import flow. - - Persists the source file to S3 and creates a ``CallImport`` row with - ``status='uploaded'``. No mapping, no provider, no rows yet — the - user moves through MAP and IMPORT as separate idempotent steps. - - Dataset is collected here (rather than at IMPORT) so the batch is - filterable from the moment it appears in the list view. - """ - del api_key - - fmt = _file_format(file.filename) - if fmt is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Unsupported file format. Allowed extensions: " - f"{', '.join(ALLOWED_EXTENSIONS)}." - ), - ) - - normalized_dataset = _normalize_dataset(dataset) - if not normalized_dataset: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="dataset is required and must be a non-empty string.", - ) - - file_bytes = await file.read() - if len(file_bytes) > MAX_UPLOAD_BYTES: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", - ) - - # Parse-now so we (a) reject garbage uploads up-front instead of - # later in the MAP step, and (b) capture the sheets snapshot the - # MAP UI needs without having to re-fetch the file from S3. - sheets = _build_available_sheets(file_bytes, fmt, file.filename) - - # Optional schema pre-pick: validated only if supplied (the user is - # allowed to set it for the first time during MAP). - if schema_id is not None: - _resolve_schema(db, organization_id, workspace_id, schema_id) - - tag_rows = _resolve_tags(db, organization_id, tag_ids) - - _ensure_blob_storage_enabled() - - # Pre-generate the id so we can compute a deterministic S3 key - # before the row is persisted, keeping ``source_s3_key`` consistent - # with the prefix sweep used at delete-time. - import uuid as _uuid - - call_import_id = _uuid.uuid4() - s3_key = _source_s3_key(organization_id, call_import_id, fmt) - content_type = _source_content_type(fmt) - - from app.services.storage.s3_service import s3_service, StorageError - - try: - s3_service.upload_file_by_key(file_bytes, s3_key, content_type=content_type) - except StorageError as exc: - logger.exception( - "Failed to upload source file to S3 for new call import {}", - call_import_id, - ) - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail=f"Failed to persist upload to S3: {exc}", - ) - - call_import = CallImport( - id=call_import_id, - organization_id=organization_id, - workspace_id=workspace_id, - # Provider + credential aren't known until the IMPORT stage; leave - # them NULL so the staged-vs-legacy distinction is visible at a - # glance from the DB. - provider=None, - telephony_integration_id=None, - original_filename=file.filename, - sheet_name=None, - dataset=normalized_dataset, - schema_id=schema_id, - parameter_mapping={}, - skipped_columns=[], - column_mapping={}, - extra_columns=[], - custom_column_mapping={}, - source_s3_key=s3_key, - source_format=fmt, - source_size_bytes=len(file_bytes), - source_content_type=content_type, - available_sheets=[sheet.model_dump() for sheet in sheets], - total_rows=0, - completed_rows=0, - failed_rows=0, - status=CallImportStatus.UPLOADED, - ) - if tag_rows: - call_import.tags = tag_rows - - db.add(call_import) - try: - db.commit() - except Exception: - db.rollback() - # Best-effort cleanup of the uploaded S3 object so a failed - # commit doesn't leak storage. - try: - s3_service.delete_file_by_key(s3_key) - except Exception as cleanup_exc: # noqa: BLE001 - logger.warning( - "Failed to clean up orphaned S3 object {} after DB rollback: {}", - s3_key, - cleanup_exc, - ) - raise - - db.refresh(call_import) - return _serialize_call_import(db, call_import) - - -@router.patch( - "/{call_import_id}/mapping", - response_model=CallImportResponse, - operation_id="updateCallImportMapping", -) -async def update_call_import_mapping( - call_import_id: UUID, - payload: CallImportMappingUpdate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportResponse: - """MAP stage of the staged call-import flow. - - Validates ``parameter_mapping`` + ``skipped_columns`` against the - sheet headers captured at UPLOAD time and persists them on the - batch. Idempotent: callers may submit this multiple times while - the batch is in ``uploaded`` or ``mapped`` state. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if call_import.status not in ( - CallImportStatus.UPLOADED, - CallImportStatus.MAPPED, - ): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - f"Cannot edit mapping on a batch in status " - f"'{call_import.status.value}'. Mapping can only be edited " - "before the IMPORT stage." - ), - ) - - if not call_import.source_s3_key or not call_import.source_format: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This batch was not uploaded through the staged flow and " - "cannot have its mapping edited." - ), - ) - - schema = _resolve_schema( - db, organization_id, workspace_id, payload.schema_id - ) - parameters = list(schema.parameters) - - canonical_sheet = _validate_sheet_choice( - call_import.source_format, - payload.sheet_name, - call_import.available_sheets, - ) - - # Pull the headers for the selected sheet straight out of the - # snapshot so we don't have to re-download the file from S3 just to - # validate the mapping. - headers: List[str] = [] - if call_import.available_sheets: - if canonical_sheet is None: - # CSV: single synthetic sheet. - entry = call_import.available_sheets[0] - headers = list(entry.get("headers") or []) - else: - for entry in call_import.available_sheets: - if not isinstance(entry, dict): - continue - name = entry.get("name") - if isinstance(name, str) and name == canonical_sheet: - headers = list(entry.get("headers") or []) - break - - cleaned_mapping = _clean_parameter_mapping( - payload.parameter_mapping, parameters, schema.name - ) - cleaned_skipped = _clean_skipped_columns(payload.skipped_columns) - - # Run the same per-column validation as the parse path so the user - # gets an immediate 400 if a required parameter is left unmapped or - # a header is neither mapped nor skipped — without needing to read - # the file. ``validate_only`` skips the row loop (and the empty-rows - # guard) since the row data lives in S3, not in this request. - if headers: - _apply_schema_mapping( - headers, - iter(()), - parameters, - cleaned_mapping, - cleaned_skipped, - source_label=( - f"Sheet '{canonical_sheet}'" - if canonical_sheet is not None - else "CSV" - ), - validate_only=True, - ) - - call_import.schema_id = schema.id - call_import.parameter_mapping = dict(cleaned_mapping) - call_import.skipped_columns = list(cleaned_skipped) - call_import.sheet_name = canonical_sheet - call_import.status = CallImportStatus.MAPPED - db.commit() - db.refresh(call_import) - return _serialize_call_import(db, call_import) - - -@router.post( - "/{call_import_id}/import", - response_model=CallImportUploadResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="startCallImport", -) -async def start_call_import( - call_import_id: UUID, - payload: CallImportStartRequest, - background_tasks: BackgroundTasks, - legacy: bool = Query( - False, - description=( - "Deprecated escape hatch for import-only processing. " - "New batches should use Run Evaluation instead." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportUploadResponse: - """Deprecated IMPORT stage — use Run Evaluation for new batches. - - Recording fetch is part of the unified evaluation pipeline. This - endpoint remains available only with ``?legacy=true`` for backward - compatibility. - """ - del api_key, background_tasks - - if not legacy: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "Standalone import is deprecated. Use Run Evaluation — " - "recording fetch is part of the evaluation pipeline. " - "Append ?legacy=true to use the import-only path." - ), - ) - - from sqlalchemy.orm import selectinload as _selectinload - - call_import = ( - db.query(CallImport) - .options(_selectinload(CallImport.tags)) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if call_import.status != CallImportStatus.MAPPED: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - f"Cannot start import for a batch in status " - f"'{call_import.status.value}'. Map the columns first." - ), - ) - - if not call_import.source_s3_key or not call_import.source_format: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This batch has no staged source file and cannot be imported " - "through the staged flow." - ), - ) - - if not call_import.schema_id: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="Cannot start import without a mapped schema.", - ) - - schema = _resolve_schema( - db, organization_id, workspace_id, call_import.schema_id - ) - parameters = list(schema.parameters) - - if payload.telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, - organization_id, - payload.telephony_integration_id, - payload.provider or "", - ) - if (integration.provider or "").lower() == "exotel": - _validate_exotel_import_ready( - parameters, dict(call_import.parameter_mapping or {}) - ) - else: - _validate_direct_url_import_ready( - parameters, dict(call_import.parameter_mapping or {}) - ) - integration = None - - _ensure_blob_storage_enabled() - - if integration is not None: - call_import.provider = integration.provider - call_import.telephony_integration_id = integration.id - else: - call_import.provider = None - call_import.telephony_integration_id = None - - call_import.total_rows = 0 - call_import.completed_rows = 0 - call_import.failed_rows = 0 - call_import.error_message = None - call_import.status = CallImportStatus.PROCESSING - db.commit() - db.refresh(call_import) - - from app.workers.tasks.call_import_bulk_ops import ( - materialize_call_import_rows_task, - ) - - materialize_call_import_rows_task.delay( - str(call_import_id), - str(organization_id), - str(workspace_id), - schedule_import_dispatch=True, - ) - - return CallImportUploadResponse( - id=call_import.id, - total_rows=0, - status=call_import.status, - dataset=call_import.dataset, - tags=_tag_response_payload(call_import.tags), - message=( - "Import accepted. Rows are being materialized in the background; " - "recordings will be fetched asynchronously." - ), - ) - - -@router.post( - "/upload", - response_model=CallImportUploadResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="uploadCallImportCsv", - deprecated=True, -) -async def upload_call_import_csv( - background_tasks: BackgroundTasks, - file: UploadFile = File(...), - provider: Optional[str] = Form( - None, - description=( - "Telephony provider key (e.g. 'exotel', 'plivo'). Must match the " - "selected telephony_integration_id's provider. Omit together " - "with telephony_integration_id for direct-URL import." - ), - ), - telephony_integration_id: Optional[UUID] = Form( - None, - description=( - "Specific TelephonyIntegration credential row to use when " - "downloading recordings for this batch. Omit together with " - "provider for direct-URL import." - ), - ), - schema_id: UUID = Form( - ..., - description=( - "Reusable Input Parameter schema this upload is mapped against. " - "Must belong to the active workspace." - ), - ), - parameter_mapping: str = Form( - ..., - description=( - "JSON-encoded ``{schema_parameter_name: source_header}`` map " - "covering every required schema parameter. Optional parameters " - "may be omitted or set to an empty string." - ), - ), - skipped_columns: Optional[str] = Form( - None, - description=( - "JSON-encoded list of source header strings the uploader has " - "explicitly skipped. Every header in the file must either be " - "mapped or appear here; otherwise the upload is rejected so a " - "forgotten column never silently drops." - ), - ), - dataset: Optional[str] = Form( - None, - description=( - "Optional free-text dataset label for high-level segregation. " - "Empty strings are stored as NULL." - ), - ), - tag_ids: Optional[List[UUID]] = Form( - None, - description="Optional list of CallImportTag ids to attach to the new batch.", - ), - sheet_name: Optional[str] = Form( - None, - description=( - "Worksheet to import when the file is an Excel workbook " - "(.xlsx / .xlsm). REQUIRED for Excel uploads. Ignored for CSV " - "uploads (rejected with 400 if non-empty so typos surface " - "instead of silently importing the wrong source)." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportUploadResponse: - """Legacy one-shot upload kept for backward compatibility. - - DEPRECATED: prefer the staged flow - (``POST /`` → ``PATCH /{id}/mapping`` → ``POST /{id}/import``) so - each step is idempotent and resumable. This endpoint runs all three - stages inline in a single transaction so existing scripts / - integrations keep working unchanged. - """ - del api_key - - fmt = _file_format(file.filename) - if fmt is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Unsupported file format. Allowed extensions: " - f"{', '.join(ALLOWED_EXTENSIONS)}." - ), - ) - - sheet_name_clean = (sheet_name or "").strip() or None - if fmt == "csv" and sheet_name_clean is not None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is not applicable to CSV uploads.", - ) - if fmt == "xlsx" and sheet_name_clean is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="sheet_name is required when uploading an Excel workbook.", - ) - - file_bytes = await file.read() - if len(file_bytes) > MAX_UPLOAD_BYTES: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", - ) - - schema = _resolve_schema(db, organization_id, workspace_id, schema_id) - parameters = list(schema.parameters) - - mapping_payload = _parse_json_form_field( - "parameter_mapping", parameter_mapping, {} - ) - cleaned_mapping = _clean_parameter_mapping( - mapping_payload, parameters, schema.name - ) - - skipped_payload = _parse_json_form_field("skipped_columns", skipped_columns, []) - cleaned_skipped = _clean_skipped_columns(skipped_payload) - - has_provider = bool((provider or "").strip()) - has_integration = telephony_integration_id is not None - if has_provider != has_integration: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "provider and telephony_integration_id must both be provided " - "or both omitted for direct-URL import." - ), - ) - - if telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, organization_id, telephony_integration_id, provider or "" - ) - if (integration.provider or "").lower() == "exotel": - _validate_exotel_import_ready(parameters, cleaned_mapping) - else: - _validate_direct_url_import_ready(parameters, cleaned_mapping) - integration = None - - parsed_rows = _parse_source_file( - file_bytes, fmt, sheet_name_clean, parameters, cleaned_mapping, cleaned_skipped - ) - _raise_if_no_importable_rows(parsed_rows, source_label=fmt) - - tag_rows = _resolve_tags(db, organization_id, tag_ids) - - call_import = CallImport( - organization_id=organization_id, - workspace_id=workspace_id, - provider=integration.provider if integration is not None else None, - telephony_integration_id=integration.id if integration is not None else None, - original_filename=file.filename, - sheet_name=sheet_name_clean, - dataset=_normalize_dataset(dataset), - schema_id=schema.id, - parameter_mapping=dict(cleaned_mapping), - skipped_columns=list(cleaned_skipped), - # Legacy columns are left empty on new uploads; the detail page - # falls back to ``parameter_mapping`` when ``schema_id`` is set. - column_mapping={}, - extra_columns=[], - custom_column_mapping={}, - total_rows=len(parsed_rows.rows), - completed_rows=0, - failed_rows=0, - status=CallImportStatus.PENDING, - source_row_skips=parse_skips_to_json(parsed_rows.skipped), - ) - if tag_rows: - call_import.tags = tag_rows - db.add(call_import) - db.flush() # populate call_import.id - if integration is None: - # The model's historical Python default is "exotel"; direct-URL - # imports intentionally have no telephony provider. - call_import.provider = None - - row_models = _materialize_rows( - db, call_import, parsed_rows.rows, organization_id - ) - - call_import.status = CallImportStatus.PROCESSING - db.commit() - db.refresh(call_import) - - background_tasks.add_task( - record_call_import_batch_created, - organization_id, - call_import.id, - workspace_id=workspace_id, - total_rows=call_import.total_rows, - source="csv", - provider=call_import.provider, - ) - - _enqueue_row_tasks(db, call_import, row_models) - - return CallImportUploadResponse( - id=call_import.id, - total_rows=call_import.total_rows, - status=call_import.status, - dataset=call_import.dataset, - tags=_tag_response_payload(call_import.tags), - message=( - f"Accepted {call_import.total_rows} rows for import. " - "Recordings will be fetched asynchronously." - ), - ) - - -@router.post( - "/audio-upload", - response_model=CallImportUploadResponse, - status_code=status.HTTP_201_CREATED, - operation_id="uploadCallImportAudio", -) -async def upload_call_import_audio( - background_tasks: BackgroundTasks, - files: List[UploadFile] = File( - ..., - description="One or more manual call recording audio files.", - ), - dataset: str = Form( - ..., - description="Required free-text dataset label for the manual upload batch.", - ), - tag_ids: Optional[List[UUID]] = Form( - None, - description="Optional list of CallImportTag ids to attach to the new batch.", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportUploadResponse: - """Persist manually uploaded recordings as completed CallImport rows. - - The rows skip the provider-download worker entirely because the audio - bytes are already in hand. From this point onward they behave exactly - like completed CSV-import rows: playback reads ``recording_s3_key`` and - the existing diarisation/evaluation endpoints can operate on them. - """ - - normalized_dataset = _normalize_dataset(dataset) - if not normalized_dataset: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="dataset is required and must be a non-empty string.", - ) - if not files: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="At least one audio file is required.", - ) - - _ensure_blob_storage_enabled() - tag_rows = _resolve_tags(db, organization_id, tag_ids) - - max_bytes = int(settings.MAX_FILE_SIZE_MB) * 1024 * 1024 - prepared: List[Dict[str, Any]] = [] - conversation_counts: Dict[str, int] = {} - - for idx, upload in enumerate(files): - filename = upload.filename or f"recording-{idx + 1}" - ext = _audio_extension(filename) - if not ext: - allowed = ", ".join(settings.ALLOWED_AUDIO_FORMATS) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Unsupported audio file '{filename}'. Allowed formats: {allowed}.", - ) - - contents = await upload.read() - if not contents: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Audio file '{filename}' is empty.", - ) - if len(contents) > max_bytes: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=( - f"Audio file '{filename}' exceeds " - f"{settings.MAX_FILE_SIZE_MB} MB." - ), - ) - - base_conversation_id = _sanitize_conversation_id(_filename_stem(filename)) - conversation_id = _dedupe_conversation_id( - base_conversation_id, - conversation_counts, - ) - prepared.append( - { - "filename": filename, - "extension": ext, - "content_type": _audio_content_type(ext, upload.content_type), - "contents": contents, - "conversation_id": conversation_id, - } - ) - - original_filename = ( - prepared[0]["filename"] - if len(prepared) == 1 - else f"{len(prepared)} manual recordings" - ) - total_size = sum(len(item["contents"]) for item in prepared) - uploaded_keys: List[str] = [] - - from app.services.storage.s3_service import s3_service - - call_import = CallImport( - organization_id=organization_id, - workspace_id=workspace_id, - provider=None, - telephony_integration_id=None, - original_filename=original_filename, - source_format="audio", - source_size_bytes=total_size, - source_content_type="audio/*", - dataset=normalized_dataset, - total_rows=len(prepared), - completed_rows=len(prepared), - failed_rows=0, - status=CallImportStatus.COMPLETED, - ) - if tag_rows: - call_import.tags = tag_rows - - try: - db.add(call_import) - db.flush() - # The model's historical Python default is "exotel"; manual uploads - # intentionally have no telephony provider. - call_import.provider = None - - row_mappings: List[Dict[str, Any]] = [] - for idx, item in enumerate(prepared): - row_id = uuid4() - key = _audio_s3_key( - organization_id, - call_import.id, - row_id, - item["extension"], - ) - s3_service.upload_file_by_key( - item["contents"], - key, - content_type=item["content_type"], - ) - uploaded_keys.append(key) - - row_mappings.append( - { - "id": row_id, - "call_import_id": call_import.id, - "organization_id": organization_id, - "workspace_id": workspace_id, - "row_index": idx, - "conversation_id": item["conversation_id"], - "recording_url": None, - "transcript": None, - "transcript_source": None, - "raw_columns": {"conversation_id": item["conversation_id"]}, - "status": CallImportRowStatus.COMPLETED, - "recording_s3_key": key, - "recording_content_type": item["content_type"], - "recording_size_bytes": len(item["contents"]), - } - ) - - if is_sharding_enabled(): - from app.db_sharding.row_ops import ( - bulk_insert_mappings_on_shards, - register_shard_slices, - ) - - bulk_insert_mappings_on_shards(db, call_import.id, row_mappings) - register_shard_slices(db, call_import.id, len(row_mappings)) - else: - for mapping in row_mappings: - db.add(CallImportRow(**mapping)) - - db.commit() - except Exception as exc: - db.rollback() - if uploaded_keys and s3_service.is_enabled(): - try: - s3_service.delete_keys(uploaded_keys) - except Exception: - logger.exception( - "Failed to clean up manual audio upload keys after error" - ) - logger.exception("Failed to persist manual call recording upload") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to upload manual recordings: {exc}", - ) from exc - - db.refresh(call_import) - background_tasks.add_task( - record_call_import_batch_created, - organization_id, - call_import.id, - workspace_id=workspace_id, - total_rows=call_import.total_rows, - source="audio", - provider=None, - ) - return CallImportUploadResponse( - id=call_import.id, - total_rows=call_import.total_rows, - status=call_import.status, - dataset=call_import.dataset, - tags=_tag_response_payload(call_import.tags), - message=( - f"Uploaded {call_import.total_rows} manual recording" - f"{'' if call_import.total_rows == 1 else 's'}." - ), - ) - - -@router.get( - "", - response_model=CallImportListResponse, - operation_id="listCallImports", -) -async def list_call_imports( - page: int = Query(1, ge=1), - page_size: int = Query(20, ge=1, le=100), - status_filter: Optional[CallImportStatus] = Query(None, alias="status"), - dataset: Optional[str] = Query( - None, - description=( - "Filter by exact dataset string (case-insensitive). Pass the " - "literal value '__none__' to filter to imports with no dataset." - ), - ), - tag_id: Optional[List[UUID]] = Query( - None, - description="Filter to imports tagged with ALL of the given tag ids.", - ), - source_format: Optional[str] = Query( - None, - description=( - "Filter by source format. Use 'audio' for manual recordings or " - "'__non_audio__' for CSV/Excel/legacy imports." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportListResponse: - """List call-import batches for the active workspace, newest first. - - Scoped to (organization_id, workspace_id) so users only see imports - for the workspace they're currently in. Supports a high-level - ``dataset`` filter (powers the segregation dropdown at the top of - the imports page) plus an AND-style multi-tag filter via repeated - ``tag_id`` parameters. - """ - - query = ( - db.query(CallImport) - .filter( - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - ) - ) - if status_filter is not None: - query = query.filter(CallImport.status == status_filter) - - source_filter = (source_format or "").strip().lower() - if source_filter == "__non_audio__": - query = query.filter( - or_(CallImport.source_format.is_(None), CallImport.source_format != "audio") - ) - elif source_filter: - query = query.filter(func.lower(CallImport.source_format) == source_filter) - - if dataset is not None: - if dataset == "__none__": - query = query.filter(CallImport.dataset.is_(None)) - elif dataset.strip(): - query = query.filter( - func.lower(CallImport.dataset) == dataset.strip().lower() - ) - - if tag_id: - from app.models.database import CallImportTagAssignment - - for single_tag_id in tag_id: - sub = ( - db.query(CallImportTagAssignment.call_import_id) - .filter(CallImportTagAssignment.tag_id == single_tag_id) - .subquery() - ) - query = query.filter(CallImport.id.in_(sub)) - - total = query.count() - items = ( - query.order_by(desc(CallImport.created_at)) - .offset((page - 1) * page_size) - .limit(page_size) - .all() - ) - - return CallImportListResponse( - items=[_serialize_call_import(db, item) for item in items], - total=total, - page=page, - page_size=page_size, - ) - - -@router.get( - "/dispatch-diagnostics", - response_model=CallImportDispatchDiagnosticsResponse, - operation_id="getCallImportDispatchDiagnostics", - dependencies=[Depends(require_admin)], -) -async def get_call_import_dispatch_diagnostics( - workspace_id: Optional[UUID] = Query( - None, - description=( - "Optional workspace filter. When omitted, returns every workspace " - "in the organization with active eval dispatch state." - ), - ), - include_idle_workspaces: bool = Query( - False, - description=( - "When true, include org workspaces with zero pending rows and " - "zero in-flight slots." - ), - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportDispatchDiagnosticsResponse: - """Live eval slot usage and fair-dispatch state for operators. - - Org admins use this to diagnose cross-workspace starvation (e.g. one - workspace's 10k run blocking another's pending eval rows) by inspecting - Redis in-flight counters, pending dispatch rows, and scheduler cursors. - """ - del api_key - payload = build_call_import_dispatch_diagnostics( - db, - organization_id, - workspace_id=workspace_id, - include_idle_workspaces=include_idle_workspaces, - ) - return CallImportDispatchDiagnosticsResponse.model_validate(payload) - - -@router.get( - "/datasets", - response_model=List[str], - operation_id="listCallImportDatasets", -) -async def list_call_import_datasets( - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> List[str]: - """Return the distinct, non-null dataset labels in use for the active - workspace. - - Scoped per-workspace so each workspace's Dataset dropdown only shows - its own segregation labels. - """ - rows = ( - db.query(CallImport.dataset) - .filter( - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - CallImport.dataset.isnot(None), - CallImport.dataset != "", - ) - .distinct() - .order_by(CallImport.dataset.asc()) - .all() - ) - return [row[0] for row in rows if row[0]] - - -@router.get( - "/diarisation-prompt-default", - response_model=CallImportDiarisationPromptDefaultResponse, - operation_id="getCallImportDiarisationPromptDefault", -) -async def get_call_import_diarisation_prompt_default( - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), -) -> CallImportDiarisationPromptDefaultResponse: - """Return the canonical LLM diariser prompt. - - The Transcribe / Run Evaluation modals call this on open so they - can pre-fill the prompt textarea. Returning the constant from the - backend (rather than hard-coding it in the frontend) keeps the - fallback used by the worker and the placeholder shown in the UI - in lock-step — operators always see the *actual* default they'd - get if they leave the field blank. - - Registered before ``GET /{call_import_id}`` so the static path is - not mistaken for a UUID import id (which would 422). - """ - del api_key, organization_id - from app.workers.tasks.helpers.llm_diarisation import ( - DEFAULT_DIARIZATION_PROMPT, - ) - - return CallImportDiarisationPromptDefaultResponse( - prompt=DEFAULT_DIARIZATION_PROMPT - ) - - -@router.patch( - "/{call_import_id}", - response_model=CallImportResponse, - operation_id="updateCallImport", -) -async def update_call_import( - call_import_id: UUID, - payload: CallImportUpdate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> CallImportResponse: - """Edit dataset / tag assignments (and schema, pre-import) on a batch. - - ``dataset = ""`` clears the label; ``tag_ids = []`` removes all tag - assignments. Fields omitted from the body are left untouched. - - ``schema_id`` is only honoured while the batch is in - ``uploaded`` / ``mapped`` state — once rows have been materialised - the schema is locked. Changing the schema resets any persisted - mapping (the user must re-MAP) and rewinds status to ``uploaded``. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - body = payload.model_dump(exclude_unset=True) - if "dataset" in body: - call_import.dataset = _normalize_dataset(body["dataset"]) - - if "tag_ids" in body: - tag_ids = body["tag_ids"] or [] - call_import.tags = _resolve_tags(db, organization_id, tag_ids) - - if "schema_id" in body and body["schema_id"] is not None: - if call_import.status not in ( - CallImportStatus.UPLOADED, - CallImportStatus.MAPPED, - ): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - f"Cannot reassign schema on a batch in status " - f"'{call_import.status.value}'." - ), - ) - new_schema = _resolve_schema( - db, organization_id, workspace_id, body["schema_id"] - ) - if call_import.schema_id != new_schema.id: - # Switching schemas invalidates the persisted mapping — - # parameter names won't line up with the new schema, so - # reset to UPLOADED and force a fresh MAP. - call_import.schema_id = new_schema.id - call_import.parameter_mapping = {} - call_import.skipped_columns = [] - call_import.sheet_name = None - call_import.status = CallImportStatus.UPLOADED - - db.commit() - db.refresh(call_import) - return _serialize_call_import(db, call_import) - - -@router.get( - "/{call_import_id}", - response_model=CallImportDetailResponse, - operation_id="getCallImportDetail", -) -async def get_call_import_detail( - call_import_id: UUID, - row_limit: int = Query(500, ge=0, le=5000), - row_offset: int = Query(0, ge=0), - q: Optional[str] = Query( - None, - description=( - "Optional case-insensitive substring filter on " - "``conversation_id``. When set, ``filtered_total_rows`` in " - "the response reflects the post-filter row count so the UI " - "can paginate against the filtered slice." - ), - ), - diarised_status: Optional[str] = Query( - None, - description=( - "Optional filter on ``CallImportRow.diarised_transcript_status``. " - "Accepts one of ``pending``, ``running``, ``completed``, " - "``failed``. When set, ``filtered_total_rows`` reflects the " - "post-filter row count (combined with the ``q`` filter when " - "both are supplied) so the UI can paginate against the same " - "slice it's displaying." - ), - pattern="^(pending|running|completed|failed)$", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportDetailResponse: - """Fetch a single import batch with a slice of its rows. - - ``row_limit=0`` is intentionally allowed so callers that only need the - batch metadata (e.g. the evaluation-detail page rendering the parent's - column mapping) can skip the rows payload entirely. - """ - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if call_import.status == CallImportStatus.PROCESSING: - from app.services.call_imports.bulk_ops import rollup_call_import_batch_status - - prior_status = call_import.status - rollup_call_import_batch_status(db, call_import) - if call_import.status != prior_status: - db.commit() - db.refresh(call_import) - - search_term = (q or "").strip() - diarised_status_filter = (diarised_status or "").strip() or None - filtered_total_rows: Optional[int] = None - has_row_filters = bool(search_term or diarised_status_filter) - - if has_row_filters: - if is_sharding_enabled(): - from app.db_sharding.scatter_gather import count_call_import_rows_filtered - - filtered_total_rows = count_call_import_rows_filtered( - db, - call_import.id, - search_term=search_term, - diarised_status_filter=diarised_status_filter, - ) - else: - rows_query = db.query(CallImportRow).filter( - CallImportRow.call_import_id == call_import.id - ) - if search_term: - rows_query = rows_query.filter( - CallImportRow.conversation_id.ilike(f"%{search_term}%") - ) - if diarised_status_filter: - rows_query = rows_query.filter( - CallImportRow.diarised_transcript_status == diarised_status_filter - ) - filtered_total_rows = rows_query.count() - - if row_limit == 0: - rows: List[CallImportRow] = [] - elif is_sharding_enabled(): - from app.db_sharding.scatter_gather import ( - fetch_call_import_rows_filtered_page, - fetch_call_import_rows_page, - ) - - if has_row_filters: - rows = fetch_call_import_rows_filtered_page( - db, - call_import.id, - search_term=search_term, - diarised_status_filter=diarised_status_filter, - offset=row_offset, - limit=row_limit, - ) - else: - rows = fetch_call_import_rows_page( - db, - call_import.id, - offset=row_offset, - limit=row_limit, - ) - else: - rows_query = db.query(CallImportRow).filter( - CallImportRow.call_import_id == call_import.id - ) - if search_term: - rows_query = rows_query.filter( - CallImportRow.conversation_id.ilike(f"%{search_term}%") - ) - if diarised_status_filter: - rows_query = rows_query.filter( - CallImportRow.diarised_transcript_status == diarised_status_filter - ) - rows = ( - rows_query.order_by(CallImportRow.row_index) - .offset(row_offset) - .limit(row_limit) - .all() - ) - - # Batch-wide diarisation status aggregate. One ``GROUP BY`` query - # across the whole batch — much cheaper than paging through every - # row to recount on the client and lets the UI render a - # transcribe/diarise progress bar without a separate roundtrip. - if is_sharding_enabled(): - from app.db_sharding.scatter_gather import aggregate_diarised_transcript_counts - - diarised_status_counts = aggregate_diarised_transcript_counts( - db, call_import.id - ) - else: - diarised_status_counts: Dict[str, int] = {} - for status_value, count in ( - db.query(CallImportRow.diarised_transcript_status, func.count()) - .filter(CallImportRow.call_import_id == call_import.id) - .group_by(CallImportRow.diarised_transcript_status) - .all() - ): - if isinstance(status_value, str): - diarised_status_counts[status_value] = int(count or 0) - - detail = CallImportDetailResponse.model_validate( - _serialize_call_import(db, call_import).model_dump() - ) - detail.rows = [CallImportRowResponse.model_validate(r) for r in rows] - detail.filtered_total_rows = filtered_total_rows - detail.diarised_pending_rows = diarised_status_counts.get("pending", 0) - detail.diarised_running_rows = diarised_status_counts.get("running", 0) - detail.diarised_completed_rows = diarised_status_counts.get("completed", 0) - detail.diarised_failed_rows = diarised_status_counts.get("failed", 0) - return detail - - -@router.get( - "/{call_import_id}/row-ids", - response_model=CallImportRowIdsResponse, - operation_id="listCallImportRowIds", -) -async def list_call_import_row_ids( - call_import_id: UUID, - q: Optional[str] = Query( - None, - description=( - "Optional case-insensitive substring filter on " - "``conversation_id``. Same semantics as the detail endpoint." - ), - ), - diarised_status: Optional[str] = Query( - None, - description=( - "Optional filter on ``CallImportRow.diarised_transcript_status``. " - "Accepts ``pending`` / ``running`` / ``completed`` / ``failed``." - ), - pattern="^(pending|running|completed|failed)$", - ), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportRowIdsResponse: - """Return every matching ``CallImportRow.id`` for cross-page bulk select. - - Lightweight companion to ``GET /{call_import_id}`` — the detail - endpoint caps ``row_limit`` at 5000 and ships the entire row body - on each page, so harvesting ids that way is wasteful when the - user just wants to bulk-delete or bulk-transcribe everything that - matches the current filters. This endpoint applies the same ``q`` - and ``diarised_status`` filters and returns only the ids. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - search_term = (q or "").strip() - status_filter = (diarised_status or "").strip() or None - - if is_sharding_enabled(): - from app.db_sharding.scatter_gather import list_call_import_row_ids_filtered - - ids = list_call_import_row_ids_filtered( - db, - call_import.id, - search_term=search_term, - diarised_status_filter=status_filter, - ) - return CallImportRowIdsResponse(ids=ids, total=len(ids)) - - rows_query = db.query(CallImportRow.id).filter( - CallImportRow.call_import_id == call_import.id - ) - if search_term: - rows_query = rows_query.filter( - CallImportRow.conversation_id.ilike(f"%{search_term}%") - ) - if status_filter: - rows_query = rows_query.filter( - CallImportRow.diarised_transcript_status == status_filter - ) - - ids = [ - row_id - for (row_id,) in rows_query.order_by(CallImportRow.row_index).all() - ] - return CallImportRowIdsResponse(ids=ids, total=len(ids)) - - -def _revoke_pending_tasks(rows: List[CallImportRow]) -> None: - """Best-effort revoke of in-flight Celery tasks for the given rows. - - Failures are logged and swallowed — Celery's control plane is async and - best-effort by design, and we always do an idempotent S3 cleanup - afterwards so a missed revoke can't leak storage. - """ - task_ids = [ - r.celery_task_id - for r in rows - if r.celery_task_id - and r.status in (CallImportRowStatus.PENDING, CallImportRowStatus.PROCESSING) - ] - if not task_ids: - return - - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke(task_ids, terminate=False) - logger.info("Revoked {} pending call-import tasks", len(task_ids)) - except Exception as exc: # noqa: BLE001 - logger.warning("Failed to revoke pending call-import tasks: {}", exc) - - -def _delete_s3_objects( - organization_id: UUID, - call_import_id: UUID, - rows: List[CallImportRow], -) -> tuple[int, int]: - """Delete every recording associated with ``rows`` plus a prefix sweep. - - The prefix sweep also cleans up the staged source file written at - UPLOAD time (``…/call_imports/{id}/source.{csv,xlsx}``) — both the - per-row recording keys and the source artefact share the same - organization-scoped prefix, so a single sweep covers them all. - - Returns ``(deleted_count, error_count)``. Never raises — callers proceed - with the DB delete regardless; orphans, if any, can be cleaned up by - re-running the same delete (it's idempotent). - """ - from app.services.storage.s3_service import s3_service - - if not s3_service.is_enabled(): - return 0, 0 - - keys = [r.recording_s3_key for r in rows if r.recording_s3_key] - deleted = 0 - errors = 0 - - if keys: - try: - d, errs = s3_service.delete_keys(keys) - deleted += d - errors += len(errs) - if errs: - logger.warning( - "S3 bulk-delete reported {} errors for call_import {}", - len(errs), - call_import_id, - ) - except Exception as exc: # noqa: BLE001 - logger.exception( - "Bulk S3 delete failed for call_import {}: {}", call_import_id, exc - ) - errors += len(keys) - - # Belt-and-braces sweep: catch anything that landed under the import's - # prefix but never made it into a row's recording_s3_key (narrow - # window where the S3 upload succeeded but the DB commit didn't). - sweep_prefix = ( - f"{s3_service.prefix}organizations/{organization_id}/" - f"call_imports/{call_import_id}/" - ) - try: - d, errs = s3_service.delete_keys_by_prefix(sweep_prefix) - deleted += d - errors += len(errs) - except Exception as exc: # noqa: BLE001 - logger.exception( - "S3 prefix sweep failed for {}: {}", sweep_prefix, exc - ) - - return deleted, errors - - -@router.delete( - "/{call_import_id}", - response_model=CallImportDeleteResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="deleteCallImport", -) -async def delete_call_import( - call_import_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportDeleteResponse: - """Delete a call-import batch asynchronously. - - Flips the batch to ``deleting`` and enqueues background teardown so - large imports (thousands of rows + S3 objects) do not block the API. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - return CallImportDeleteResponse( - id=call_import_id, - status="completed", - ) - - if call_import.status == CallImportStatus.DELETING: - return CallImportDeleteResponse( - id=call_import.id, - status="accepted", - ) - - call_import.status = CallImportStatus.DELETING - call_import.error_message = None - db.commit() - - from app.workers.tasks.call_import_bulk_ops import delete_call_import_task - - delete_call_import_task.delay( - str(call_import_id), - str(organization_id), - ) - - return CallImportDeleteResponse( - id=call_import.id, - status="accepted", - ) - - -def _locate_call_import_row_or_404( - catalog_db: Session, - *, - call_import_id: UUID, - row_id: UUID, - organization_id: UUID, -) -> Tuple[Session, CallImportRow, Optional[Session]]: - """Find a call import row on the correct DB session for mutation. - - When sharding is enabled rows live on shard databases; ``get_db`` only - opens the catalog. Returns ``(row_db, row, extra_catalog_to_close)`` - where ``extra_catalog_to_close`` is the catalog session opened by - :func:`locate_call_import_row` (distinct from the route's catalog - session) and must be closed via :func:`close_row_sessions`. - """ - from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row - - try: - row_db, located_catalog, row, _shard_id = locate_call_import_row(row_id) - except LookupError: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import row not found", - ) from None - if ( - row.call_import_id != call_import_id - or row.organization_id != organization_id - ): - close_row_sessions( - row_db, - located_catalog if located_catalog is not row_db else None, - ) - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import row not found", - ) - extra_catalog = located_catalog if located_catalog is not row_db else None - return row_db, row, extra_catalog - - -@router.delete( - "/{call_import_id}/rows/{row_id}", - status_code=status.HTTP_204_NO_CONTENT, - operation_id="deleteCallImportRow", -) -async def delete_call_import_row( - call_import_id: UUID, - row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> Response: - """Delete a single CallImportRow and its S3 recording. - - The parent ``CallImport`` is left in place. After deletion we recompute - its ``total_rows`` / ``completed_rows`` / ``failed_rows`` / ``status`` - so the UI's progress bar stays consistent with reality. - """ - from app.services.storage.s3_service import s3_service - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.db_sharding.row_ops import close_row_sessions - - row_db, row, extra_catalog = _locate_call_import_row_or_404( - db, - call_import_id=call_import.id, - row_id=row_id, - organization_id=organization_id, - ) - try: - _revoke_pending_tasks([row]) - - if row.recording_s3_key and s3_service.is_enabled(): - try: - s3_service.delete_file_by_key(row.recording_s3_key) - except Exception as exc: # noqa: BLE001 — best-effort, DB is source of truth - logger.warning( - "Failed to delete S3 object {} for row {}: {}", - row.recording_s3_key, - row.id, - exc, - ) - - row_db.delete(row) - row_db.commit() - - _recompute_call_import_counters(db, call_import) - db.commit() - finally: - close_row_sessions(row_db, extra_catalog) - - logger.info( - "Deleted call_import_row {} (call_import={}, org={})", - row_id, - call_import.id, - organization_id, - ) - - return Response(status_code=status.HTTP_204_NO_CONTENT) - - -def _recompute_call_import_counters( - db: Session, call_import: CallImport -) -> None: - """Resync ``total/completed/failed_rows`` + status on the parent batch. - - Called after row-level mutations (single delete, bulk delete) so the - UI's progress bar stays consistent with the actual row set. The - rules mirror :func:`delete_call_import_row` so behavior doesn't - diverge between the per-row and bulk paths. - """ - - from app.services.call_imports.bulk_ops import rollup_call_import_batch_status - - rollup_call_import_batch_status(db, call_import) - - -@router.post( - "/{call_import_id}/retry-failed", - response_model=CallImportRetryFailedRowsResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="retryFailedCallImportRows", -) -async def retry_failed_call_import_rows( - call_import_id: UUID, - payload: Optional[CallImportRetryFailedRowsRequest] = Body(None), - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportRetryFailedRowsResponse: - """Re-enqueue every failed import row in this batch. - - Useful when transient provider issues are resolved and the operator wants - a one-click "try failed downloads again" pass without re-uploading the CSV. - - Pass ``provider`` + ``telephony_integration_id`` (or both omitted for - direct-URL retry) to change how recordings are fetched on this pass. - When the body is omitted entirely, the batch keeps its existing pinned - credentials. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if payload is not None: - if payload.telephony_integration_id is not None: - integration = _resolve_telephony_integration( - db, - organization_id, - payload.telephony_integration_id, - payload.provider or "", - ) - call_import.provider = integration.provider - call_import.telephony_integration_id = integration.id - if (integration.provider or "").lower() == "exotel": - schema = _resolve_schema( - db, - organization_id, - call_import.workspace_id, - call_import.schema_id, - ) - _validate_exotel_import_ready( - list(schema.parameters), - dict(call_import.parameter_mapping or {}), - ) - else: - call_import.provider = None - call_import.telephony_integration_id = None - db.flush() - - failed_rows = ( - db.query(CallImportRow) - .filter( - CallImportRow.call_import_id == call_import.id, - CallImportRow.status == CallImportRowStatus.FAILED, - ) - .order_by(CallImportRow.row_index.asc()) - .all() - ) - if not failed_rows: - return CallImportRetryFailedRowsResponse( - requeued=0, - enqueue_failed=0, - skipped=0, - ) - - from app.workers.concurrency.fair_import_dispatch import ( - schedule_fair_import_dispatch, - ) - - # Reset rows to pending BEFORE enqueue so the UI reflects "retry in - # progress" immediately even if the worker queue is backlogged. - for row in failed_rows: - row.status = CallImportRowStatus.PENDING - row.error_message = None - row.celery_task_id = None - - db.flush() - _recompute_call_import_counters(db, call_import) - db.commit() - - try: - schedule_fair_import_dispatch(max_workspace_turns=999) - requeued = len(failed_rows) - enqueue_failed = 0 - skipped = 0 - except Exception as exc: # noqa: BLE001 - logger.exception( - "Failed to schedule fair import dispatch for import {}", - call_import.id, - ) - requeued = 0 - enqueue_failed = len(failed_rows) - skipped = 0 - for row in failed_rows: - db.refresh(row) - if row.status != CallImportRowStatus.PENDING: - skipped += 1 - enqueue_failed -= 1 - continue - row.status = CallImportRowStatus.FAILED - row.error_message = f"Failed to enqueue retry: {exc}" - db.flush() - _recompute_call_import_counters(db, call_import) - db.commit() - - return CallImportRetryFailedRowsResponse( - requeued=requeued, - enqueue_failed=enqueue_failed, - skipped=skipped, - ) - - -@router.post( - "/{call_import_id}/rows/bulk-delete", - response_model=CallImportRowBulkDeleteResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="bulkDeleteCallImportRows", -) -async def bulk_delete_call_import_rows( - call_import_id: UUID, - payload: CallImportRowBulkDelete, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportRowBulkDeleteResponse: - """Delete multiple ``CallImportRow`` rows in one request. - - Unknown / cross-tenant row ids are silently skipped — the response - reports how many actually went away so a UI that holds onto stale - ids (e.g. after another tab already deleted a row) doesn't 404 - the entire bulk action. - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - if not payload.row_ids: - return CallImportRowBulkDeleteResponse(deleted=0, status="completed") - - from app.workers.tasks.call_import_bulk_ops import bulk_delete_call_import_rows_task - - row_id_strs = [str(rid) for rid in payload.row_ids] - - bulk_delete_call_import_rows_task.delay( - str(call_import_id), - str(organization_id), - row_id_strs, - ) - - return CallImportRowBulkDeleteResponse(deleted=0, status="accepted") - - -# --------------------------------------------------------------------------- -# Diarization / transcription endpoints -# --------------------------------------------------------------------------- - - -def _select_rows_for_transcription( - db: Session, - call_import: CallImport, - payload: CallImportTranscribeRequest, - requested_row_ids: Optional[List[UUID]] = None, -) -> tuple[List[CallImportRow], Dict[str, int]]: - """Pick which rows to enqueue for diarisation (delegates to bulk_ops).""" - from app.services.call_imports.bulk_ops import select_rows_for_transcription - - try: - return select_rows_for_transcription( - db, call_import, payload, requested_row_ids=requested_row_ids - ) - except ValueError as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=str(exc), - ) from exc - - -@router.post( - "/{call_import_id}/transcribe", - response_model=CallImportTranscribeResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="transcribeCallImport", -) -async def transcribe_call_import( - call_import_id: UUID, - payload: CallImportTranscribeRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportTranscribeResponse: - """Fan out diarization tasks for many rows in a single call. - - Returns a summary with how many rows were queued and how many were - skipped (broken down by reason) so the UI can show a meaningful - toast even when nothing actually got enqueued (e.g. "All 12 rows - already have transcripts"). - """ - - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.workers.tasks.call_import_bulk_ops import bulk_diarize_call_import_task - - bulk_diarize_call_import_task.delay( - str(call_import_id), - str(organization_id), - payload.model_dump(mode="json"), - [str(rid) for rid in payload.row_ids] if payload.row_ids else None, - ) - - return CallImportTranscribeResponse( - queued=0, - skipped_rows=0, - skipped_reason_counts={}, - accepted=True, - ) - - -@router.post( - "/{call_import_id}/rows/{row_id}/transcribe", - response_model=CallImportTranscribeResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="transcribeCallImportRow", -) -async def transcribe_call_import_row( - call_import_id: UUID, - row_id: UUID, - payload: CallImportTranscribeRequest, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportTranscribeResponse: - """Diarize / transcribe a single row. - - Thin wrapper over the batch endpoint that hard-codes a single - ``row_ids`` filter. Skip counts still surface so the UI can render - "Skipped — transcript present" diagnostics consistently. - """ - - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.services.call_imports.bulk_ops import execute_bulk_diarization - - try: - result = execute_bulk_diarization( - db, - call_import, - payload, - requested_row_ids=[row_id], - ) - except ValueError as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=str(exc), - ) from exc - - return CallImportTranscribeResponse( - queued=result.queued, - skipped_rows=result.skipped_rows, - skipped_reason_counts=result.skipped_reason_counts, - ) - - -# --------------------------------------------------------------------------- -# Cancel-in-flight diarisation -# --------------------------------------------------------------------------- -# -# Long-running multimodal LLM diarisation calls (especially LLM-only mode on -# slow audio) can sit in ``pending`` / ``running`` for tens of minutes when an -# upstream provider stalls. Without an abort affordance the operator's only -# recourse is to wait for Celery's ``time_limit`` to fire — which can be -# several minutes — or to manually mutate the DB. These helpers + the two -# endpoints below give the UI a first-class "Stop diarisation" button. -# -# Why ``terminate=True``: the legacy ``_revoke_pending_tasks`` helper uses -# ``terminate=False`` because it's called from delete-flow paths where the -# task may simply not get to run (a worker pulls it off the queue and drops -# it). For a user-initiated cancel we want SIGTERM to interrupt the worker -# mid-LLM call so the audio HTTP request actually aborts. ``terminate=True`` -# routes SIGTERM to the executing process; ``signal="SIGTERM"`` is the -# default but we spell it out so the intent is obvious to reviewers. - -# Sentinel error message stamped on cancelled rows. Read by the transcribe -# worker's finaliser (see ``app/workers/tasks/transcribe_call_import_row.py``) -# to detect a row that was cancelled mid-flight and AVOID overwriting it -# with whatever partial result the worker had managed to compute before the -# SIGTERM landed. -CANCELLED_BY_USER_ERROR: str = "Diarisation cancelled by user" - - -def _cancellable_diarisation_states() -> Tuple[str, ...]: - """States that a diarisation row can be cancelled from. - - Kept as a tiny helper so adding a future ``"queued"`` / ``"retrying"`` - state only needs one edit. - """ - return ("pending", "running") - - -def _revoke_diarisation_task(row: CallImportRow) -> None: - """Best-effort revoke of a single row's diarisation Celery task. - - Always swallows control-plane exceptions — Celery's control bus is - inherently best-effort and a missed revoke is not catastrophic - because the DB row is already flipped to ``failed`` by the caller - before this runs (so the UI immediately reflects the cancel; if - the task happens to finish anyway, the worker's finaliser skips - over the row via :data:`CANCELLED_BY_USER_ERROR`). - """ - task_id = (row.celery_task_id or "").strip() - if not task_id: - return - try: - from app.workers.celery_app import celery_app - - celery_app.control.revoke( - task_id, terminate=True, signal="SIGTERM" - ) - logger.info( - "Revoked diarisation task {} for call-import row {}", - task_id, - row.id, - ) - except Exception as exc: # noqa: BLE001 — revoke is best-effort - logger.warning( - "Failed to revoke diarisation task {} for row {}: {}", - task_id, - row.id, - exc, - ) - - -def _apply_diarisation_cancel(rows: List[CallImportRow]) -> Tuple[int, int]: - """Cancel diarisation on every cancellable row in ``rows``. - - Returns ``(cancelled, skipped)`` so the caller can build a typed - response without re-querying the DB. The caller is responsible for - ``db.commit()`` after this returns — we deliberately don't commit - here so a batch endpoint can flush all rows in one transaction. - """ - cancellable_states = _cancellable_diarisation_states() - cancelled = 0 - skipped = 0 - for row in rows: - if (row.diarised_transcript_status or "").lower() not in cancellable_states: - skipped += 1 - continue - # Flip the row state BEFORE we revoke so the UI's next poll - # already shows the cancel, even if Celery's control plane is - # slow to ack. - row.diarised_transcript_status = "failed" - row.diarised_transcript_error = CANCELLED_BY_USER_ERROR - _revoke_diarisation_task(row) - # Drop the task id so a follow-up retry (or a stale poll) can't - # accidentally re-revoke or get confused. - row.celery_task_id = None - cancelled += 1 - return cancelled, skipped - - -@router.post( - "/{call_import_id}/rows/{row_id}/cancel-diarisation", - response_model=CallImportRowResponse, - status_code=status.HTTP_200_OK, - operation_id="cancelCallImportRowDiarisation", -) -async def cancel_call_import_row_diarisation( - call_import_id: UUID, - row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportRowResponse: - """Abort an in-flight (or queued) diarisation for a single row. - - Idempotent: calling on a row that's already terminal (``completed`` - / ``failed`` / ``idle``) returns the row unchanged with a 200, so - the UI can fire this from a "Stop" button without having to - pre-check the state. - - Race notes: - - * The row's ``diarised_transcript_status`` is flipped to ``failed`` - with :data:`CANCELLED_BY_USER_ERROR` BEFORE the Celery revoke, - so the polling UI sees the cancel immediately. - * If the worker happens to finish between our DB flip and the - SIGTERM landing, its finaliser will detect the cancelled - sentinel on the row and skip its own status / score writes - (see :mod:`app.workers.tasks.transcribe_call_import_row`). - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.db_sharding.row_ops import close_row_sessions - - row_db, row, extra_catalog = _locate_call_import_row_or_404( - db, - call_import_id=call_import_id, - row_id=row_id, - organization_id=organization_id, - ) - try: - _apply_diarisation_cancel([row]) - row_db.commit() - row_db.refresh(row) - return CallImportRowResponse.model_validate(row) - finally: - close_row_sessions(row_db, extra_catalog) - - -@router.post( - "/{call_import_id}/cancel-diarisation", - response_model=CallImportCancelDiarisationResponse, - status_code=status.HTTP_200_OK, - operation_id="cancelCallImportDiarisation", -) -async def cancel_call_import_diarisation( - call_import_id: UUID, - payload: Optional[CallImportCancelDiarisationRequest] = None, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportCancelDiarisationResponse: - """Abort in-flight diarisation for many rows in a single call. - - Default body (no ``row_ids``) cancels every row in this import - whose ``diarised_transcript_status`` is ``pending`` or - ``running`` — the "stop everything" button. Pass ``row_ids`` to - scope the cancel to the rows the operator has selected. - - Returns ``(cancelled, skipped)`` so the UI can render a tight - toast ("Cancelled 3 rows · 1 skipped (already completed)"). - """ - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - base_query = db.query(CallImportRow).filter( - CallImportRow.call_import_id == call_import_id - ) - - requested_ids = ( - payload.row_ids if payload and payload.row_ids is not None else None - ) - if requested_ids is not None: - if not requested_ids: - # Empty list is "no rows requested" — treat as a no-op - # 200 rather than 400 so the UI can pass through an empty - # selection without a special-case. - return CallImportCancelDiarisationResponse(cancelled=0, skipped=0) - rows = base_query.filter(CallImportRow.id.in_(requested_ids)).all() - found_ids = {r.id for r in rows} - # Treat requested-but-not-found ids as ``skipped`` so the UI's - # numbers reconcile (a stale selection that includes deleted - # rows shouldn't 404 the whole call). - missing = [rid for rid in requested_ids if rid not in found_ids] - skipped_missing = len(missing) - else: - # Implicit "cancel every cancellable row in this import" path. - rows = base_query.filter( - CallImportRow.diarised_transcript_status.in_( - list(_cancellable_diarisation_states()) - ) - ).all() - skipped_missing = 0 - - cancelled, skipped = _apply_diarisation_cancel(rows) - db.commit() - return CallImportCancelDiarisationResponse( - cancelled=cancelled, - skipped=skipped + skipped_missing, - ) - - -def _render_diarised_segments_text( - segments: Optional[List[Dict[str, Any]]], - *, - swap: bool = False, -) -> str: - """Render ``CallImportRow.diarised_segments`` as ``: `` lines. - - Mirrors the worker's ``_render_turns_as_text`` (kept duplicated so - the route doesn't need to import a Celery task module just to - rebuild the rendered transcript). Only ``agent`` and ``user`` are - swapped — multi-party calls keep their ``speaker_N`` labels through - a swap so we don't silently collapse a third speaker into the user - side. - """ - if not segments: - return "" - out: List[str] = [] - for turn in segments: - if not isinstance(turn, dict): - continue - speaker = (turn.get("speaker") or "").strip() - text = (turn.get("text") or "").strip() - if not speaker or not text: - continue - if swap: - if speaker == "agent": - speaker = "user" - elif speaker == "user": - speaker = "agent" - out.append(f"{speaker}: {text}") - return "\n".join(out) - - -@router.post( - "/{call_import_id}/rows/{row_id}/diarised-speaker-swap", - response_model=CallImportRowResponse, - operation_id="toggleCallImportRowSpeakerSwap", -) -async def toggle_call_import_row_speaker_swap( - call_import_id: UUID, - row_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportRowResponse: - """Flip the user <-> agent mapping on a diarised row. - - The worker's "first speaker is the agent" heuristic is right most of - the time but does fail on inbound recordings where the customer - greets first, on recordings where the agent stays silent for the - intro, etc. Rather than rerun the (expensive) STT + pyannote - pipeline for those cases, we let reviewers flip the mapping in - place: the structured ``diarised_segments`` are the source of truth - and we re-render the plain-text ``diarised_transcript`` from them - with the swap applied. The next CSV export will then show the - corrected labels. - - Returns the updated row so the frontend can refresh without an - extra round-trip. - """ - - del api_key - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - from app.db_sharding.row_ops import close_row_sessions - - row_db, row, extra_catalog = _locate_call_import_row_or_404( - db, - call_import_id=call_import_id, - row_id=row_id, - organization_id=organization_id, - ) - try: - segments = ( - row.diarised_segments if isinstance(row.diarised_segments, list) else None - ) - if not segments: - # Without structured turns the swap toggle would have nothing to - # re-render — surface a clear error rather than silently - # flipping a flag the UI never read. - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This row has no structured diarised segments to swap. " - "Re-run diarisation to generate per-speaker turns first." - ), - ) - - new_swap = not bool(row.diarised_speaker_swap) - row.diarised_speaker_swap = new_swap - row.diarised_transcript = ( - _render_diarised_segments_text(segments, swap=new_swap) or None - ) - row_db.commit() - row_db.refresh(row) - return CallImportRowResponse.model_validate(row) - finally: - close_row_sessions(row_db, extra_catalog) - - -# --------------------------------------------------------------------------- -# Cross-run insights for the import detail page -# --------------------------------------------------------------------------- - - -@router.get( - "/{call_import_id}/insights", - response_model=CallImportInsightsResponse, - operation_id="getCallImportInsights", -) -async def get_call_import_insights( - call_import_id: UUID, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -) -> CallImportInsightsResponse: - """Aggregate signals across every evaluation run on this import. - - Powers the Insights tab on the call-import detail page: returns - per-metric "latest run" summaries plus a trend series of mean values - across runs so the UI can render a small line chart per metric. Also - bundles transcript coverage stats since those are the cheapest - pre-eval health-check (e.g. "30 of 50 rows still missing - transcripts"). - """ - - del api_key - - from app.models.database import ( - CallImportEvaluation, - CallImportEvaluationRow, - Metric, - ) - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == call_import_id, - CallImport.organization_id == organization_id, - ) - .first() - ) - if not call_import: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import not found", - ) - - rows = ( - db.query(CallImportRow) - .filter(CallImportRow.call_import_id == call_import_id) - .all() - ) - # A row "has a transcript" if EITHER the production (CSV) or the - # diarised (worker) column is populated — the insights tile reports - # the union so users see total coverage regardless of which source - # produced the value. - rows_with_transcript = sum( - 1 - for r in rows - if (r.transcript or "").strip() - or (r.diarised_transcript or "").strip() - ) - rows_without_transcript = len(rows) - rows_with_transcript - source_counts: Dict[str, int] = {} - for r in rows: - has_production = bool((r.transcript or "").strip()) - has_diarised = bool((r.diarised_transcript or "").strip()) - if has_production: - key = r.transcript_source or "csv" - source_counts[key] = source_counts.get(key, 0) + 1 - if has_diarised: - source_counts["diarised"] = source_counts.get("diarised", 0) + 1 - - evaluations = ( - db.query(CallImportEvaluation) - .filter( - CallImportEvaluation.call_import_id == call_import_id, - CallImportEvaluation.organization_id == organization_id, - ) - .order_by(CallImportEvaluation.created_at.asc()) - .all() - ) - - # Defer heavy lifting to the aggregation helper so this endpoint and - # the per-run aggregate endpoint share the exact same metric - # bucketing math (no chance of "trend" disagreeing with "latest" on - # the same data set). - from app.api.v1.routes.call_import_evaluations import ( - _compute_metric_aggregates, - ) - - metric_history: Dict[str, List[CallImportInsightsRunPoint]] = {} - metric_meta: Dict[str, Metric] = {} - metric_latest: Dict[str, CallImportMetricAggregate] = {} - - for evaluation in evaluations: - eval_rows = ( - db.query(CallImportEvaluationRow) - .filter(CallImportEvaluationRow.evaluation_id == evaluation.id) - .all() - ) - aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) - for agg in aggregates: - if agg.metric_id not in metric_meta: - # ``agg.metric_id`` is normally a UUID string, but the - # aggregator also emits ids that surface in row scores - # without a matching ``Metric`` row (e.g. a metric the - # user deleted mid-run, or LLM-discovered slugs). Those - # are not valid UUIDs, so coerce defensively and skip - # the metric registry lookup when the cast fails — the - # ``meta is None`` branch below already handles the - # display via the values stored on ``agg`` itself. - try: - metric_uuid = UUID(agg.metric_id) - except (ValueError, AttributeError, TypeError): - metric_uuid = None - if metric_uuid is not None: - metric_obj = ( - db.query(Metric) - .filter( - Metric.id == metric_uuid, - Metric.organization_id == organization_id, - ) - .first() - ) - if metric_obj is not None: - metric_meta[agg.metric_id] = metric_obj - history = metric_history.setdefault(agg.metric_id, []) - history.append( - CallImportInsightsRunPoint( - evaluation_id=evaluation.id, - name=evaluation.name, - created_at=evaluation.created_at, - mean=agg.mean, - completed_rows=agg.count, - ) - ) - metric_latest[agg.metric_id] = agg - - metrics_payload: List[CallImportInsightsMetric] = [] - for metric_id, latest in metric_latest.items(): - meta = metric_meta.get(metric_id) - metrics_payload.append( - CallImportInsightsMetric( - metric_id=metric_id, - metric_name=(meta.name if meta else latest.metric_name), - metric_type=(meta.metric_type if meta else latest.metric_type), - latest=latest, - trend=metric_history.get(metric_id, []), - ) - ) - - return CallImportInsightsResponse( - call_import_id=call_import_id, - total_rows=len(rows), - rows_with_transcript=rows_with_transcript, - rows_without_transcript=rows_without_transcript, - transcript_source_counts=source_counts, - evaluation_count=len(evaluations), - metrics=metrics_payload, - ) - - -from app.core.auth.capabilities import CALLS_DELETE, CALLS_IMPORT, CALLS_VIEW -from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities - -apply_workspace_route_capabilities( - router, - view_capability=CALLS_VIEW, - manage_capability=CALLS_IMPORT, - delete_capability=CALLS_DELETE, -) +"""CSV-driven call import routes. + +Users upload a CSV plus a per-batch column mapping (CSV header -> system +field). The backend persists a CallImport batch + one CallImportRow per +line, then fans the rows out to the Celery ``imports`` queue where each +row is downloaded using the telephony credential pinned on the batch. +Exotel credentialed imports require a ``recording_url`` on every row; +direct-URL imports (no credential) also require a mapped recording URL. +""" + +from __future__ import annotations + +import csv +import io +import json +import re +from dataclasses import dataclass, field +from datetime import date, datetime, time, timedelta +from typing import Any, Dict, Iterable, List, Optional, Tuple +from uuid import UUID, uuid4 + +from fastapi import APIRouter, Body, BackgroundTasks, Depends, File, Form, HTTPException, Query, Response, UploadFile, status +from loguru import logger +from sqlalchemy import desc, func, or_ +from sqlalchemy.orm import Session + +from app.config import settings +from app.core.auth import Principal, get_principal +from app.core.auth.rbac import require_admin +from app.database import get_db +from app.db_sharding.sessions import is_sharding_enabled +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.billing.flexprice_service import record_call_import_batch_created +from app.services.call_imports.audit import ( + actor_emails_for_call_import, + emails_for_user_ids, + stamp_call_import_actor, + user_ids_from_call_imports, +) +from app.services.call_imports.dispatch_diagnostics import ( + build_call_import_dispatch_diagnostics, +) +from app.models.database import ( + CallImport, + CallImportRow, + CallImportSchema, + CallImportSchemaParameter, + CallImportTag, + TelephonyIntegration, +) +from app.models.enums import ( + CallImportParameterType, + CallImportRowStatus, + CallImportStatus, +) +from app.models.schemas import ( + CallImportCancelDiarisationRequest, + CallImportCancelDiarisationResponse, + CallImportDetailResponse, + CallImportDeleteResponse, + CallImportDiarisationPromptDefaultResponse, + CallImportDispatchDiagnosticsResponse, + CallImportInsightsMetric, + CallImportInsightsResponse, + CallImportInsightsRunPoint, + CallImportListResponse, + CallImportMappingUpdate, + CallImportMetricAggregate, + CallImportPreviewResponse, + CallImportPreviewSheet, + CallImportRetryFailedRowsRequest, + CallImportRetryFailedRowsResponse, + CallImportResponse, + CallImportRowIdsResponse, + CallImportRowBulkDelete, + CallImportRowBulkDeleteResponse, + CallImportRowResponse, + CallImportStartRequest, + CallImportTranscribeRequest, + CallImportTranscribeResponse, + CallImportUpdate, + CallImportUploadResponse, +) + + +router = APIRouter( + prefix="/call-imports", + tags=["Call Imports"], + dependencies=[Depends(require_enterprise_feature("call_imports"))], +) + + +@dataclass(frozen=True) +class CallImportParseSkip: + """One source row excluded during CSV/Excel parse (identity / recording URL).""" + + source_row: int + reason: str + message: str + + +@dataclass +class CallImportParseResult: + rows: List[Dict[str, Any]] = field(default_factory=list) + skipped: List[CallImportParseSkip] = field(default_factory=list) + + +def parse_skips_to_json(skips: List[CallImportParseSkip]) -> List[Dict[str, Any]]: + """Persistable JSON shape for ``CallImport.source_row_skips``.""" + return [ + { + "source_row": item.source_row, + "reason": item.reason, + "message": item.message, + } + for item in skips + ] + + +def _normalize_dataset(raw: Optional[str]) -> Optional[str]: + """Trim and treat empty strings as 'no dataset' (NULL).""" + if raw is None: + return None + cleaned = raw.strip() + return cleaned or None + + +def _serialize_call_import( + db: Session, + call_import: CallImport, + *, + user_emails: Optional[Dict[UUID, str]] = None, +) -> CallImportResponse: + """Catalog parent fields; counters come from SQL rollup (not Redis merge).""" + from app.services.call_imports.bulk_ops import rollup_call_import_batch_status + from app.services.call_imports.progress_counters import ( + clear_import_progress_redis, + read_import_progress, + ) + + redis_completed, redis_failed = read_import_progress(call_import.id) + if ( + redis_completed + or redis_failed + or int(call_import.completed_rows or 0) > int(call_import.total_rows or 0) + or int(call_import.failed_rows or 0) > int(call_import.total_rows or 0) + ): + rollup_call_import_batch_status(db, call_import) + db.flush() + + clear_import_progress_redis(call_import.id) + db.refresh(call_import) + total = int(call_import.total_rows or 0) + completed = min(int(call_import.completed_rows or 0), total) if total else int( + call_import.completed_rows or 0 + ) + failed = min(int(call_import.failed_rows or 0), total) if total else int( + call_import.failed_rows or 0 + ) + if user_emails is None: + user_emails = emails_for_user_ids( + db, user_ids_from_call_imports([call_import]) + ) + created_email, updated_email = actor_emails_for_call_import( + call_import, user_emails + ) + base = CallImportResponse.model_validate(call_import) + return base.model_copy( + update={ + "completed_rows": completed, + "failed_rows": failed, + "created_by_email": created_email, + "last_updated_by_email": updated_email, + } + ) + + +def _resolve_tags( + db: Session, organization_id: UUID, tag_ids: Optional[List[UUID]] +) -> List[CallImportTag]: + """Look up tag rows by id, scoped to the organization. + + Raises HTTPException(400) if any id is unknown for the org. + """ + if not tag_ids: + return [] + rows = ( + db.query(CallImportTag) + .filter( + CallImportTag.organization_id == organization_id, + CallImportTag.id.in_(tag_ids), + ) + .all() + ) + found_ids = {row.id for row in rows} + missing = [str(tag_id) for tag_id in tag_ids if tag_id not in found_ids] + if missing: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unknown call_import_tag id(s): {missing}", + ) + return rows + + +MAX_UPLOAD_BYTES = 15 * 1024 * 1024 # 15 MB upload cap (CSV or Excel) + +# File extensions accepted by the upload + preview endpoints. Keep in +# lockstep with the frontend ``accept`` attribute on the file picker. +CSV_EXTENSIONS = (".csv",) +XLSX_EXTENSIONS = (".xlsx", ".xlsm") +ALLOWED_EXTENSIONS = CSV_EXTENSIONS + XLSX_EXTENSIONS + +AUDIO_CONTENT_TYPES = { + "wav": "audio/wav", + "mp3": "audio/mpeg", + "flac": "audio/flac", + "m4a": "audio/mp4", +} + + +def _file_format(filename: Optional[str]) -> Optional[str]: + """Classify ``filename`` as ``'csv'`` / ``'xlsx'`` or ``None`` if unsupported.""" + if not filename: + return None + name = filename.lower() + if name.endswith(CSV_EXTENSIONS): + return "csv" + if name.endswith(XLSX_EXTENSIONS): + return "xlsx" + return None + + +def _audio_extension(filename: Optional[str]) -> Optional[str]: + """Return the validated lower-case extension for a manual recording.""" + if not filename or "." not in filename: + return None + ext = filename.rsplit(".", 1)[-1].lower().strip() + allowed = {fmt.lower().lstrip(".") for fmt in settings.ALLOWED_AUDIO_FORMATS} + return ext if ext in allowed else None + + +def _audio_content_type(ext: str, upload_content_type: Optional[str]) -> str: + """Prefer the browser-supplied audio content type, with a safe fallback.""" + supplied = (upload_content_type or "").strip() + if supplied and supplied != "application/octet-stream": + return supplied + return AUDIO_CONTENT_TYPES.get(ext.lower(), "application/octet-stream") + + +def _audio_s3_key( + organization_id: UUID, call_import_id: UUID, row_id: UUID, ext: str +) -> str: + """Build the canonical S3 key for a manually uploaded recording.""" + from app.services.storage.s3_service import s3_service + + return ( + f"{s3_service.prefix}organizations/{organization_id}/call_imports/" + f"{call_import_id}/{row_id}.{ext}" + ) + + +def _filename_stem(filename: Optional[str]) -> str: + """Extract a cross-platform filename stem from an UploadFile name.""" + raw = (filename or "").strip() + basename = re.split(r"[\\/]", raw)[-1] if raw else "" + if "." in basename: + basename = basename.rsplit(".", 1)[0] + return basename.strip() + + +def _sanitize_conversation_id(raw: str) -> str: + """Turn a filename stem into a stable conversation_id.""" + cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", raw.strip()) + cleaned = re.sub(r"_+", "_", cleaned).strip("._-") + return (cleaned or "recording")[:255] + + +def _dedupe_conversation_id( + base: str, counts: Dict[str, int] +) -> str: + """Make conversation ids unique within one manual upload batch.""" + count = counts.get(base, 0) + 1 + counts[base] = count + if count == 1: + return base + suffix = f"-{count}" + return f"{base[: 255 - len(suffix)]}{suffix}" + + +def _normalize_header(name: str) -> str: + return (name or "").strip().lower() + + +def _header_lookup(fieldnames: List[str]) -> Dict[str, str]: + """Map normalized header -> original header for case-insensitive lookup.""" + return {_normalize_header(h): h for h in fieldnames or []} + + +def _resolve_mapped_header( + mapping_value: Optional[str], header_lookup: Dict[str, str] +) -> Optional[str]: + """Translate a user-supplied CSV header into the actual column key. + + The frontend sends headers exactly as they appear in the source file, + but we still normalize on the server so trailing whitespace / casing + doesn't break matching. Returns the canonical fieldname or ``None`` + if not present in the file. + """ + if not mapping_value: + return None + return header_lookup.get(_normalize_header(mapping_value)) + + +def _xlsx_cell_to_str(value: Any) -> str: + """Coerce an openpyxl cell value to the string the rest of the + pipeline expects. + + openpyxl returns native Python types (int, float, datetime, bool, + None). The CSV path always works with strings, so we mirror that: + integers stringify cleanly (no ``.0`` suffix on whole-number floats), + datetimes use ISO-8601, booleans use SQL-style ``TRUE`` / ``FALSE``. + """ + if value is None: + return "" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + if isinstance(value, int): + return str(value) + if isinstance(value, float): + if value.is_integer(): + return str(int(value)) + return str(value) + if isinstance(value, datetime): + return value.isoformat() + if isinstance(value, date): + return value.isoformat() + if isinstance(value, time): + return value.isoformat() + if isinstance(value, timedelta): + return str(value) + return str(value) + + +def _parse_recording_date_cell(cell: str) -> date: + """Parse day-first dates with one/two digit day-month parts.""" + match = re.fullmatch(r"\s*(\d{1,2})[/-](\d{1,2})[/-](\d{4})\s*", cell) + if match: + day, month, year = (int(part) for part in match.groups()) + return date(year, month, day) + + # Native Excel date cells arrive from ``_xlsx_cell_to_str`` as ISO + # datetimes (e.g. ``2026-01-04T00:00:00``). Accept that resolved date, + # while keeping plain ISO dates rejected for hand-entered text/CSV cells. + if "T" in cell: + return datetime.fromisoformat(cell.replace("Z", "+00:00")).date() + + raise ValueError("expected D/M/YYYY or D-M-YYYY") + + +def _coerce_parameter_value( + raw: str, + param_type: CallImportParameterType, + *, + row_idx: int, + param_name: str, +) -> Any: + """Validate + coerce a single CSV cell against its declared type. + + Returns the typed Python value to surface in ``raw_columns``. Empty + strings are returned as ``None`` regardless of the parameter type so + optional cells stay null end-to-end. Coercion failures raise a + 400 with a row-anchored message. + """ + cell = (raw or "").strip() + if not cell: + return None + + if param_type == CallImportParameterType.CONVERSATION_ID: + return cell + if param_type == CallImportParameterType.RECORDING_URL: + # Recording URLs are exercised by the worker (which downloads + # them); we only do a light "starts with http" check here so a + # paste-error surfaces immediately at upload time. + lower = cell.lower() + if not (lower.startswith("http://") or lower.startswith("https://")): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + "valid recording URL (must start with http:// or https://)." + ), + ) + return cell + if param_type == CallImportParameterType.RECORDING_DATE: + try: + parsed_date = _parse_recording_date_cell(cell) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + f"valid recording date ({cell!r}); expected day-first " + "D/M/YYYY or D-M-YYYY." + ), + ) + return parsed_date.strftime("%d/%m/%Y") + if param_type == CallImportParameterType.TRANSCRIPT: + return cell + if param_type == CallImportParameterType.TEXT: + return cell + if param_type == CallImportParameterType.NUMBER: + try: + value = float(cell) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + f"valid number ({cell!r})." + ), + ) + if value.is_integer(): + return int(value) + return value + if param_type == CallImportParameterType.BOOLEAN: + truthy = {"true", "yes", "y", "1", "t"} + falsy = {"false", "no", "n", "0", "f"} + norm = cell.lower() + if norm in truthy: + return True + if norm in falsy: + return False + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + f"valid boolean ({cell!r})." + ), + ) + if param_type == CallImportParameterType.DATETIME: + try: + parsed = datetime.fromisoformat(cell.replace("Z", "+00:00")) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + f"valid ISO-8601 date/time ({cell!r})." + ), + ) + return parsed.isoformat() + if param_type == CallImportParameterType.URL: + lower = cell.lower() + if not (lower.startswith("http://") or lower.startswith("https://")): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {row_idx + 1}: value for '{param_name}' is not a " + "valid URL (must start with http:// or https://)." + ), + ) + return cell + # Unknown types: store as text and let the next migration catch up. + return cell + + +def _recording_url_cell_is_valid_http(raw: str) -> bool: + cell = (raw or "").strip() + if not cell: + return False + lower = cell.lower() + return lower.startswith("http://") or lower.startswith("https://") + + +def _parameter_is_required(param: CallImportSchemaParameter) -> bool: + """Return whether a schema parameter must be mapped on every upload.""" + if param.is_required: + return True + try: + param_type = CallImportParameterType(param.type) + except ValueError: + return False + return param_type in ( + CallImportParameterType.CONVERSATION_ID, + CallImportParameterType.RECORDING_URL, + ) + + +def _apply_schema_mapping( + fieldnames: List[str], + rows_iter: Iterable[Dict[str, str]], + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, str], + skipped_columns: List[str], + *, + source_label: str = "CSV", + validate_only: bool = False, +) -> CallImportParseResult: + """Schema-driven row projection: parameter -> CSV header -> typed value. + + Validates that every required schema parameter is mapped to a CSV + header that actually exists in the file, and that every CSV header + is either mapped to a parameter or explicitly listed in + ``skipped_columns``. Returns one dict per non-empty data row with: + + * ``conversation_id`` (str, mandatory) + * ``recording_date`` (Optional[str], DD/MM/YYYY date) + * ``recording_url`` (Optional[str]) + * ``transcript`` (Optional[str]) + * ``parameter_values`` (Dict[str, Any]) of typed values keyed by + parameter name (drives ``raw_columns`` so the export can + reproduce the source). + + ``validate_only=True`` runs the header / mapping / skipped-column + checks (every check that doesn't need to read row data) and then + returns an empty list — used by the MAP stage to validate a + mapping payload against the cached sheet snapshot without + re-fetching the source bytes from S3. + """ + header_lookup = _header_lookup(list(fieldnames)) + + # 1. Look up the conversation_id parameter so we can address it + # directly while building each row. + conv_param = next( + (p for p in parameters if p.type == CallImportParameterType.CONVERSATION_ID), + None, + ) + if conv_param is None: + # The schema invariant should have caught this on create/update, + # but a defensive 400 here keeps us safe against hand-rolled + # API callers that bypassed validation. + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Selected schema is missing the mandatory conversation_id parameter.", + ) + # 2. Resolve every mapped parameter to a canonical fieldname. + # Required parameters MUST resolve; optional ones may resolve to + # None if the user left them blank (no mapping). + canonical_by_param: Dict[str, Optional[str]] = {} + recording_date_param_name: Optional[str] = None + rec_url_param_name: Optional[str] = None + transcript_param_name: Optional[str] = None + for param in parameters: + mapped_header = parameter_mapping.get(param.name) + canonical = ( + _resolve_mapped_header(mapped_header, header_lookup) + if mapped_header + else None + ) + if _parameter_is_required(param) and canonical is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"{source_label} does not contain the column " + f"'{mapped_header or ''}' mapped to required parameter " + f"'{param.name}'." + ), + ) + canonical_by_param[param.name] = canonical + if param.type == CallImportParameterType.RECORDING_DATE: + recording_date_param_name = param.name + elif param.type == CallImportParameterType.RECORDING_URL: + rec_url_param_name = param.name + elif param.type == CallImportParameterType.TRANSCRIPT: + transcript_param_name = param.name + + # 3. Every CSV column must either be mapped to a parameter or + # explicitly skipped. Catches "I forgot to skip the email + # column" gracefully instead of dropping data silently. + mapped_canonicals = {c for c in canonical_by_param.values() if c} + skipped_canonicals = { + _resolve_mapped_header(h, header_lookup) + for h in skipped_columns + } + skipped_canonicals.discard(None) + unhandled = [ + h + for h in fieldnames + if h not in mapped_canonicals and h not in skipped_canonicals + ] + if unhandled: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"{source_label} columns must either be mapped to a schema " + f"parameter or explicitly skipped. Unhandled: {unhandled}." + ), + ) + + conv_canonical = canonical_by_param[conv_param.name] + rec_canonical = ( + canonical_by_param.get(rec_url_param_name) + if rec_url_param_name + else None + ) + recording_date_canonical = ( + canonical_by_param.get(recording_date_param_name) + if recording_date_param_name + else None + ) + transcript_canonical = ( + canonical_by_param.get(transcript_param_name) + if transcript_param_name + else None + ) + + if validate_only: + # MAP-stage validation: every header check above has already + # run; the row loop only matters at IMPORT time. Skip it (and + # the "no data rows" guard at the bottom of the function) so + # the caller gets a clean pass when the mapping is shaped right. + return CallImportParseResult() + + parsed: List[Dict[str, Any]] = [] + skipped: List[CallImportParseSkip] = [] + for idx, row in enumerate(rows_iter): + # Drop fully-blank lines - matches the legacy parser behavior so + # trailing-newline edge cases don't fail an otherwise-good upload. + non_blank = any( + (row.get(c) or "").strip() + for c in mapped_canonicals + if c + ) + if not non_blank: + continue + + source_row = idx + 1 + conv_value = (row.get(conv_canonical) or "").strip() if conv_canonical else "" + if not conv_value: + skipped.append( + CallImportParseSkip( + source_row=source_row, + reason="missing_conversation_id", + message=( + f"Row {source_row} is missing the '{conv_param.name}' " + "(conversation_id) value." + ), + ) + ) + continue + + if rec_canonical and rec_url_param_name: + rec_param = next( + (p for p in parameters if p.name == rec_url_param_name), + None, + ) + if rec_param is not None and _parameter_is_required(rec_param): + rec_raw = (row.get(rec_canonical) or "").strip() + if not rec_raw: + skipped.append( + CallImportParseSkip( + source_row=source_row, + reason="missing_recording_url", + message=( + f"Row {source_row} is missing the required " + f"'{rec_url_param_name}' value." + ), + ) + ) + continue + if not _recording_url_cell_is_valid_http(rec_raw): + skipped.append( + CallImportParseSkip( + source_row=source_row, + reason="invalid_recording_url", + message=( + f"Row {source_row}: value for " + f"'{rec_url_param_name}' is not a valid recording " + "URL (must start with http:// or https://)." + ), + ) + ) + continue + + # Materialize every mapped parameter into the per-row snapshot, + # running per-type coercion so a bad cell aborts the upload + # rather than silently storing garbage. + parameter_values: Dict[str, Any] = {} + row_skipped = False + for param in parameters: + canonical = canonical_by_param[param.name] + if canonical is None: + continue + try: + param_type = CallImportParameterType(param.type) + except ValueError: + param_type = CallImportParameterType.TEXT + coerced = _coerce_parameter_value( + row.get(canonical) or "", + param_type, + row_idx=idx, + param_name=param.name, + ) + if _parameter_is_required(param) and coerced is None: + if param_type == CallImportParameterType.RECORDING_URL: + skipped.append( + CallImportParseSkip( + source_row=source_row, + reason="missing_recording_url", + message=( + f"Row {source_row} is missing the required " + f"'{param.name}' value." + ), + ) + ) + row_skipped = True + break + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Row {source_row} is missing the required " + f"'{param.name}' value." + ), + ) + parameter_values[param.name] = coerced + if row_skipped: + continue + + rec_value = ( + (row.get(rec_canonical) or "").strip() if rec_canonical else "" + ) + transcript_value = ( + (row.get(transcript_canonical) or "").strip() + if transcript_canonical + else "" + ) + recording_date_value = ( + parameter_values.get(recording_date_param_name) + if recording_date_param_name + else None + ) + + parsed.append( + { + "conversation_id": conv_value, + "recording_date": recording_date_value, + "recording_url": rec_value or None, + "transcript": transcript_value or None, + "parameter_values": parameter_values, + } + ) + + if not parsed and not skipped: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"{source_label} did not contain any data rows.", + ) + + return CallImportParseResult(rows=parsed, skipped=skipped) + + +def _raise_if_no_importable_rows( + result: CallImportParseResult, *, source_label: str = "CSV" +) -> None: + """Sync upload / API callers fail fast when every data row was skipped.""" + if result.rows: + return + if result.skipped: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"No importable rows. {len(result.skipped)} row(s) skipped due " + "to missing or invalid conversation ID or recording URL." + ), + ) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"{source_label} did not contain any data rows.", + ) + + +def _parse_csv( + file_bytes: bytes, + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, str], + skipped_columns: List[str], +) -> CallImportParseResult: + """Parse a CSV file using the resolved schema parameters.""" + if not file_bytes: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Uploaded CSV is empty.", + ) + + try: + text_stream = io.StringIO(file_bytes.decode("utf-8-sig")) + except UnicodeDecodeError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="CSV must be UTF-8 encoded.", + ) + + reader = csv.DictReader(text_stream) + if not reader.fieldnames: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="CSV is missing a header row.", + ) + + return _apply_schema_mapping( + list(reader.fieldnames), + reader, + parameters, + parameter_mapping, + skipped_columns, + source_label="CSV", + ) + + +def _open_xlsx_workbook(file_bytes: bytes): + """Open an xlsx/xlsm workbook from in-memory bytes (read-only stream). + + Imports openpyxl lazily so the module loads even in environments that + haven't installed the optional dep yet (e.g. lightweight tooling + images). Surfaces a clean 400 if openpyxl is missing or the file is + not a valid Office Open XML workbook. + """ + if not file_bytes: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Uploaded Excel file is empty.", + ) + try: + from openpyxl import load_workbook # type: ignore + from openpyxl.utils.exceptions import InvalidFileException # type: ignore + except ImportError as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=( + "Excel uploads require the 'openpyxl' package which is " + "not installed in this environment." + ), + ) from exc + + try: + return load_workbook( + io.BytesIO(file_bytes), + read_only=True, + data_only=True, + ) + except InvalidFileException as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"File is not a valid .xlsx workbook: {exc}", + ) from exc + except Exception as exc: # zipfile.BadZipFile etc. + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Could not open Excel workbook: {exc}", + ) from exc + + +def _xlsx_sheet_headers_and_rows( + worksheet, +) -> Tuple[List[str], List[Dict[str, str]]]: + """Read row 1 as headers and the rest as dicts of stringified cells. + + Empty trailing header cells are dropped. Duplicate headers preserve + the first occurrence (matches ``csv.DictReader`` behavior, which + silently drops duplicates). + """ + iterator = worksheet.iter_rows(values_only=True) + try: + header_row = next(iterator) + except StopIteration: + return [], [] + + headers: List[str] = [] + seen: set[str] = set() + for cell in header_row: + name = _xlsx_cell_to_str(cell).strip() + if not name: + # Stop at the first blank header — treats trailing empty + # columns as not part of the table (matches typical Excel + # workbook conventions). + break + norm = name.lower() + if norm in seen: + continue + seen.add(norm) + headers.append(name) + + rows: List[Dict[str, str]] = [] + for row in iterator: + if row is None: + continue + # Pad / truncate to the header length so dict construction is + # stable even when a row has fewer / extra cells than the header. + cells = list(row[: len(headers)]) + if len(cells) < len(headers): + cells.extend([None] * (len(headers) - len(cells))) + if not any(_xlsx_cell_to_str(c).strip() for c in cells): + # Skip fully-blank rows (openpyxl read_only routinely yields + # trailing empties when the worksheet's used range exceeds + # the actual data). + continue + rows.append( + { + header: _xlsx_cell_to_str(value) + for header, value in zip(headers, cells) + } + ) + + return headers, rows + + +def _parse_xlsx( + file_bytes: bytes, + sheet_name: Optional[str], + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, str], + skipped_columns: List[str], +) -> CallImportParseResult: + """Parse a single worksheet from an xlsx/xlsm workbook. + + ``sheet_name`` must match one of the workbook's sheets (case + insensitive whitespace-trimmed match). Returns the same shape as + :func:`_parse_csv` so the upload handler can persist either format + through the same code path. + """ + if not sheet_name or not sheet_name.strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is required when uploading an Excel workbook.", + ) + + workbook = _open_xlsx_workbook(file_bytes) + try: + sheet_names = list(workbook.sheetnames) + target_norm = sheet_name.strip().lower() + match = next( + (s for s in sheet_names if s.strip().lower() == target_norm), + None, + ) + if match is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Sheet '{sheet_name}' not found in workbook. " + f"Available sheets: {sheet_names}" + ), + ) + worksheet = workbook[match] + headers, rows = _xlsx_sheet_headers_and_rows(worksheet) + finally: + workbook.close() + + if not headers: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Sheet '{sheet_name}' is missing a header row.", + ) + + return _apply_schema_mapping( + headers, + rows, + parameters, + parameter_mapping, + skipped_columns, + source_label=f"Sheet '{sheet_name}'", + ) + + +def _csv_preview_sheets( + file_bytes: bytes, filename: Optional[str] +) -> List[CallImportPreviewSheet]: + """Build the synthetic single-sheet preview entry for a CSV upload.""" + if not file_bytes: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Uploaded CSV is empty.", + ) + try: + text_stream = io.StringIO(file_bytes.decode("utf-8-sig")) + except UnicodeDecodeError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="CSV must be UTF-8 encoded.", + ) + reader = csv.DictReader(text_stream) + if not reader.fieldnames: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="CSV is missing a header row.", + ) + headers = list(reader.fieldnames) + row_count = 0 + for row in reader: + # Match the parse-time skip: ignore fully blank rows so the + # count the user sees lines up with what /upload will ingest. + if any((v or "").strip() for v in row.values()): + row_count += 1 + + sheet_label = (filename or "sheet1").rsplit("/", 1)[-1] or "sheet1" + return [ + CallImportPreviewSheet( + name=sheet_label, + headers=headers, + row_count=row_count, + ) + ] + + +def _xlsx_preview_sheets(file_bytes: bytes) -> List[CallImportPreviewSheet]: + """List every worksheet in the workbook with its headers and row count.""" + workbook = _open_xlsx_workbook(file_bytes) + sheets: List[CallImportPreviewSheet] = [] + try: + for name in workbook.sheetnames: + worksheet = workbook[name] + headers, rows = _xlsx_sheet_headers_and_rows(worksheet) + sheets.append( + CallImportPreviewSheet( + name=name, + headers=headers, + row_count=len(rows), + ) + ) + finally: + workbook.close() + return sheets + + +def _parse_json_form_field(name: str, raw: Optional[str], default): + """Decode a JSON-encoded form field with a friendly 400 on bad JSON.""" + if raw is None or raw == "": + return default + try: + return json.loads(raw) + except json.JSONDecodeError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"{name} must be valid JSON: {exc}", + ) + + +# --------------------------------------------------------------------------- +# Shared helpers used by the staged endpoints (UPLOAD / MAP / IMPORT) and the +# legacy one-shot ``POST /upload`` shim. Extracted here so each stage and the +# back-compat path operate on the exact same validation + persistence code. +# --------------------------------------------------------------------------- + + +def _source_content_type(fmt: str) -> str: + """Return the canonical ``Content-Type`` for a parsed file format.""" + if fmt == "csv": + return "text/csv" + if fmt == "xlsx": + return ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ) + return "application/octet-stream" + + +def _source_s3_key( + organization_id: UUID, call_import_id: UUID, fmt: str +) -> str: + """Build the canonical S3 key for an upload's source file. + + Mirrors the per-row recording key convention used by + ``process_call_import_row`` so a single prefix sweep on delete still + cleans up both the source artefact and every fetched recording. + """ + from app.services.storage.s3_service import s3_service + + ext = "xlsx" if fmt == "xlsx" else "csv" + return ( + f"{s3_service.prefix}organizations/{organization_id}/call_imports/" + f"{call_import_id}/source.{ext}" + ) + + +def _build_available_sheets( + file_bytes: bytes, fmt: str, filename: Optional[str] +) -> List[CallImportPreviewSheet]: + """Snapshot of sheets + headers cached on the batch at UPLOAD time.""" + if fmt == "csv": + return _csv_preview_sheets(file_bytes, filename) + return _xlsx_preview_sheets(file_bytes) + + +def _resolve_schema( + db: Session, + organization_id: UUID, + workspace_id: UUID, + schema_id: UUID, +) -> CallImportSchema: + """Fetch + validate a schema row in the active workspace. + + Eager-loads ``parameters`` so callers can iterate without re-querying. + """ + from sqlalchemy.orm import selectinload as _selectinload + + schema = ( + db.query(CallImportSchema) + .options(_selectinload(CallImportSchema.parameters)) + .filter( + CallImportSchema.id == schema_id, + CallImportSchema.organization_id == organization_id, + CallImportSchema.workspace_id == workspace_id, + ) + .first() + ) + if not schema: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Call import schema not found in the active workspace.", + ) + if not list(schema.parameters): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Selected schema has no parameters defined.", + ) + return schema + + +def _validate_direct_url_import_ready( + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, Any], +) -> None: + """Ensure direct-URL import has a mapped recording_url column.""" + rec_url_param = next( + ( + p + for p in parameters + if p.type == CallImportParameterType.RECORDING_URL.value + ), + None, + ) + if rec_url_param is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Direct URL import requires a schema parameter of type " + "'recording_url'." + ), + ) + mapped_header = (parameter_mapping or {}).get(rec_url_param.name) + if not (mapped_header or "").strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Direct URL import requires the 'recording_url' parameter to " + "be mapped to a source column." + ), + ) + + +def _validate_exotel_import_ready( + parameters: List[CallImportSchemaParameter], + parameter_mapping: Dict[str, Any], +) -> None: + """Ensure Exotel credentialed import has a mapped recording_url column.""" + rec_url_param = next( + ( + p + for p in parameters + if p.type == CallImportParameterType.RECORDING_URL.value + ), + None, + ) + if rec_url_param is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Exotel import requires a schema parameter of type " + "'recording_url'." + ), + ) + mapped_header = (parameter_mapping or {}).get(rec_url_param.name) + if not (mapped_header or "").strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Exotel import requires the 'recording_url' parameter to " + "be mapped to a source column." + ), + ) + + +def _resolve_telephony_integration( + db: Session, + organization_id: UUID, + telephony_integration_id: UUID, + provider: str, +) -> TelephonyIntegration: + """Fetch + validate a telephony credential against the requested provider.""" + integration = ( + db.query(TelephonyIntegration) + .filter( + TelephonyIntegration.id == telephony_integration_id, + TelephonyIntegration.organization_id == organization_id, + ) + .first() + ) + if not integration: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Telephony credential not found for this organization.", + ) + if (integration.provider or "").lower() != provider.lower(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Selected credential is for provider '{integration.provider}', " + f"but request specified '{provider}'." + ), + ) + if not integration.is_active: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Selected telephony credential is inactive.", + ) + return integration + + +def _clean_parameter_mapping( + mapping_payload: Any, + parameters: List[CallImportSchemaParameter], + schema_name: str, +) -> Dict[str, str]: + """Trim values and drop empties; reject unknown parameter names. + + Accepts an already-decoded value (dict-shaped) so the same helper + works for the JSON-form upload path and the JSON-body PATCH path. + """ + if not isinstance(mapping_payload, dict) or not all( + isinstance(k, str) and (v is None or isinstance(v, str)) + for k, v in mapping_payload.items() + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "parameter_mapping must be an object of " + "{parameter_name: csv_header}." + ), + ) + + valid_param_names = {p.name for p in parameters} + cleaned: Dict[str, str] = {} + for raw_name, raw_header in mapping_payload.items(): + if raw_name not in valid_param_names: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"parameter_mapping references unknown parameter " + f"'{raw_name}' on schema '{schema_name}'." + ), + ) + header = (raw_header or "").strip() + if header: + cleaned[raw_name] = header + return cleaned + + +def _clean_skipped_columns(skipped_payload: Any) -> List[str]: + """Dedupe (case-insensitively) and drop blanks; preserve original casing.""" + if not isinstance(skipped_payload, list) or not all( + isinstance(item, str) for item in skipped_payload + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="skipped_columns must be a list of header strings.", + ) + cleaned: List[str] = [] + seen: set[str] = set() + for item in skipped_payload: + norm = _normalize_header(item) + if not norm or norm in seen: + continue + seen.add(norm) + cleaned.append(item) + return cleaned + + +def _parse_source_file( + file_bytes: bytes, + fmt: str, + sheet_name: Optional[str], + parameters: List[CallImportSchemaParameter], + cleaned_mapping: Dict[str, str], + cleaned_skipped: List[str], +) -> CallImportParseResult: + """Run the format-appropriate parser against a buffer of file bytes.""" + if fmt == "csv": + return _parse_csv(file_bytes, parameters, cleaned_mapping, cleaned_skipped) + return _parse_xlsx( + file_bytes, sheet_name, parameters, cleaned_mapping, cleaned_skipped + ) + + +def _materialize_rows( + db: Session, + call_import: CallImport, + parsed_rows: List[Dict[str, Any]], + organization_id: UUID, +) -> List[CallImportRow]: + """Insert one ``CallImportRow`` per parsed row, returning the new models.""" + row_models: List[CallImportRow] = [] + for idx, row in enumerate(parsed_rows): + # Stamp ``transcript_source='csv'`` when the upload actually + # provided a transcript so the UI badge ("From CSV") works from + # day one. Blank cells stay NULL so the row reads as "no + # production transcript yet". + csv_transcript = row["transcript"] + row_model = CallImportRow( + call_import_id=call_import.id, + organization_id=organization_id, + workspace_id=call_import.workspace_id, + row_index=idx, + conversation_id=row["conversation_id"], + recording_date=( + _parse_recording_date_cell(row["recording_date"]) + if row.get("recording_date") + else None + ), + recording_url=row["recording_url"], + transcript=csv_transcript, + transcript_source=( + "csv" if csv_transcript and csv_transcript.strip() else None + ), + raw_columns=row["parameter_values"] or None, + status=CallImportRowStatus.PENDING, + ) + db.add(row_model) + row_models.append(row_model) + return row_models + + +def _enqueue_row_tasks( + db: Session, + call_import: CallImport, + row_models: List[CallImportRow], +) -> None: + """Schedule fair round-robin dispatch for pending import rows.""" + del db, call_import, row_models + from app.workers.concurrency.fair_import_dispatch import ( + schedule_fair_import_dispatch, + ) + + schedule_fair_import_dispatch(max_workspace_turns=999) + + +def _ensure_blob_storage_enabled() -> None: + """Hard-fail UPLOAD if cloud blob storage isn't configured (no local fallback).""" + from app.services.storage.s3_service import s3_service + + if not s3_service.is_enabled(): + err = ( + s3_service.get_status_message() + or "Cloud blob storage is not enabled or not configured" + ) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=( + "Call uploads require cloud blob storage so the file can be " + f"persisted between stages: {err}" + ), + ) + + +def _validate_sheet_choice( + fmt: str, + sheet_name: Optional[str], + available_sheets: Optional[List[Dict[str, Any]]], +) -> Optional[str]: + """Normalize / validate ``sheet_name`` against the persisted snapshot. + + Returns the canonical sheet name (matching the workbook's casing) + so downstream parsing addresses the right worksheet. + """ + if fmt == "csv": + if sheet_name and sheet_name.strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is not applicable to CSV uploads.", + ) + return None + + cleaned = (sheet_name or "").strip() or None + if cleaned is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is required when the source is an Excel workbook.", + ) + + if not available_sheets: + # Nothing to validate against (e.g. legacy batch without snapshot); + # let downstream parsing error out instead of silently importing. + return cleaned + + target = cleaned.strip().lower() + for entry in available_sheets: + name = entry.get("name") if isinstance(entry, dict) else None + if isinstance(name, str) and name.strip().lower() == target: + return name + sheet_names = [ + entry.get("name") for entry in available_sheets if isinstance(entry, dict) + ] + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Sheet '{cleaned}' not found in the staged file. " + f"Available sheets: {sheet_names}" + ), + ) + + +def _tag_response_payload(tags: Optional[List[CallImportTag]]) -> List[Dict[str, Any]]: + """Shape a CallImport's tag relationship for the upload response.""" + return [ + { + "id": tag.id, + "name": tag.name, + "color": tag.color, + "created_at": tag.created_at, + "updated_at": tag.updated_at, + } + for tag in (tags or []) + ] + + +@router.post( + "/preview", + response_model=CallImportPreviewResponse, + operation_id="previewCallImportFile", +) +async def preview_call_import_file( + file: UploadFile = File(...), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> CallImportPreviewResponse: + """Inspect an uploaded CSV / Excel file and return its sheets + headers. + + Drives the column-mapping UI without forcing the frontend to parse + CSV / xlsx itself — keeps client and server in lockstep on quoted + fields, encodings, and Excel cell coercion. CSVs return a single + synthetic sheet named after the filename; Excel workbooks return one + entry per worksheet (in workbook order). + """ + del api_key, organization_id, workspace_id, db # auth only + + fmt = _file_format(file.filename) + if fmt is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Unsupported file format. Allowed extensions: " + f"{', '.join(ALLOWED_EXTENSIONS)}." + ), + ) + + file_bytes = await file.read() + if len(file_bytes) > MAX_UPLOAD_BYTES: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", + ) + + if fmt == "csv": + sheets = _csv_preview_sheets(file_bytes, file.filename) + else: + sheets = _xlsx_preview_sheets(file_bytes) + + return CallImportPreviewResponse(format=fmt, sheets=sheets) + + +@router.post( + "", + response_model=CallImportResponse, + status_code=status.HTTP_201_CREATED, + operation_id="createCallImport", +) +async def create_call_import( + file: UploadFile = File( + ..., + description="CSV / Excel file to stage. Persisted to S3 between stages.", + ), + dataset: str = Form( + ..., + description=( + "Required free-text dataset label. Collected up-front so the " + "batch is filterable from the moment it lands." + ), + ), + tag_ids: Optional[List[UUID]] = Form( + None, + description="Optional list of CallImportTag ids to attach to the new batch.", + ), + schema_id: Optional[UUID] = Form( + None, + description=( + "Optional schema pre-pick. The user can still change it during " + "the MAP stage; provided here only so the detail page can pre-" + "select the schema dropdown." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportResponse: + """UPLOAD stage of the staged call-import flow. + + Persists the source file to S3 and creates a ``CallImport`` row with + ``status='uploaded'``. No mapping, no provider, no rows yet — the + user moves through MAP and IMPORT as separate idempotent steps. + + Dataset is collected here (rather than at IMPORT) so the batch is + filterable from the moment it appears in the list view. + """ + del api_key + + fmt = _file_format(file.filename) + if fmt is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Unsupported file format. Allowed extensions: " + f"{', '.join(ALLOWED_EXTENSIONS)}." + ), + ) + + normalized_dataset = _normalize_dataset(dataset) + if not normalized_dataset: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="dataset is required and must be a non-empty string.", + ) + + file_bytes = await file.read() + if len(file_bytes) > MAX_UPLOAD_BYTES: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", + ) + + # Parse-now so we (a) reject garbage uploads up-front instead of + # later in the MAP step, and (b) capture the sheets snapshot the + # MAP UI needs without having to re-fetch the file from S3. + sheets = _build_available_sheets(file_bytes, fmt, file.filename) + + # Optional schema pre-pick: validated only if supplied (the user is + # allowed to set it for the first time during MAP). + if schema_id is not None: + _resolve_schema(db, organization_id, workspace_id, schema_id) + + tag_rows = _resolve_tags(db, organization_id, tag_ids) + + _ensure_blob_storage_enabled() + + # Pre-generate the id so we can compute a deterministic S3 key + # before the row is persisted, keeping ``source_s3_key`` consistent + # with the prefix sweep used at delete-time. + import uuid as _uuid + + call_import_id = _uuid.uuid4() + s3_key = _source_s3_key(organization_id, call_import_id, fmt) + content_type = _source_content_type(fmt) + + from app.services.storage.s3_service import s3_service, StorageError + + try: + s3_service.upload_file_by_key(file_bytes, s3_key, content_type=content_type) + except StorageError as exc: + logger.exception( + "Failed to upload source file to S3 for new call import {}", + call_import_id, + ) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=f"Failed to persist upload to S3: {exc}", + ) + + call_import = CallImport( + id=call_import_id, + organization_id=organization_id, + workspace_id=workspace_id, + # Provider + credential aren't known until the IMPORT stage; leave + # them NULL so the staged-vs-legacy distinction is visible at a + # glance from the DB. + provider=None, + telephony_integration_id=None, + original_filename=file.filename, + sheet_name=None, + dataset=normalized_dataset, + schema_id=schema_id, + parameter_mapping={}, + skipped_columns=[], + column_mapping={}, + extra_columns=[], + custom_column_mapping={}, + source_s3_key=s3_key, + source_format=fmt, + source_size_bytes=len(file_bytes), + source_content_type=content_type, + available_sheets=[sheet.model_dump() for sheet in sheets], + total_rows=0, + completed_rows=0, + failed_rows=0, + status=CallImportStatus.UPLOADED, + ) + if tag_rows: + call_import.tags = tag_rows + + stamp_call_import_actor(call_import, principal, creating=True) + db.add(call_import) + try: + db.commit() + except Exception: + db.rollback() + # Best-effort cleanup of the uploaded S3 object so a failed + # commit doesn't leak storage. + try: + s3_service.delete_file_by_key(s3_key) + except Exception as cleanup_exc: # noqa: BLE001 + logger.warning( + "Failed to clean up orphaned S3 object {} after DB rollback: {}", + s3_key, + cleanup_exc, + ) + raise + + db.refresh(call_import) + return _serialize_call_import(db, call_import) + + +@router.patch( + "/{call_import_id}/mapping", + response_model=CallImportResponse, + operation_id="updateCallImportMapping", +) +async def update_call_import_mapping( + call_import_id: UUID, + payload: CallImportMappingUpdate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportResponse: + """MAP stage of the staged call-import flow. + + Validates ``parameter_mapping`` + ``skipped_columns`` against the + sheet headers captured at UPLOAD time and persists them on the + batch. Idempotent: callers may submit this multiple times while + the batch is in ``uploaded`` or ``mapped`` state. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + if call_import.status not in ( + CallImportStatus.UPLOADED, + CallImportStatus.MAPPED, + ): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Cannot edit mapping on a batch in status " + f"'{call_import.status.value}'. Mapping can only be edited " + "before the IMPORT stage." + ), + ) + + if not call_import.source_s3_key or not call_import.source_format: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This batch was not uploaded through the staged flow and " + "cannot have its mapping edited." + ), + ) + + schema = _resolve_schema( + db, organization_id, workspace_id, payload.schema_id + ) + parameters = list(schema.parameters) + + canonical_sheet = _validate_sheet_choice( + call_import.source_format, + payload.sheet_name, + call_import.available_sheets, + ) + + # Pull the headers for the selected sheet straight out of the + # snapshot so we don't have to re-download the file from S3 just to + # validate the mapping. + headers: List[str] = [] + if call_import.available_sheets: + if canonical_sheet is None: + # CSV: single synthetic sheet. + entry = call_import.available_sheets[0] + headers = list(entry.get("headers") or []) + else: + for entry in call_import.available_sheets: + if not isinstance(entry, dict): + continue + name = entry.get("name") + if isinstance(name, str) and name == canonical_sheet: + headers = list(entry.get("headers") or []) + break + + cleaned_mapping = _clean_parameter_mapping( + payload.parameter_mapping, parameters, schema.name + ) + cleaned_skipped = _clean_skipped_columns(payload.skipped_columns) + + # Run the same per-column validation as the parse path so the user + # gets an immediate 400 if a required parameter is left unmapped or + # a header is neither mapped nor skipped — without needing to read + # the file. ``validate_only`` skips the row loop (and the empty-rows + # guard) since the row data lives in S3, not in this request. + if headers: + _apply_schema_mapping( + headers, + iter(()), + parameters, + cleaned_mapping, + cleaned_skipped, + source_label=( + f"Sheet '{canonical_sheet}'" + if canonical_sheet is not None + else "CSV" + ), + validate_only=True, + ) + + call_import.schema_id = schema.id + call_import.parameter_mapping = dict(cleaned_mapping) + call_import.skipped_columns = list(cleaned_skipped) + call_import.sheet_name = canonical_sheet + call_import.status = CallImportStatus.MAPPED + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + return _serialize_call_import(db, call_import) + + +@router.post( + "/{call_import_id}/import", + response_model=CallImportUploadResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="startCallImport", +) +async def start_call_import( + call_import_id: UUID, + payload: CallImportStartRequest, + background_tasks: BackgroundTasks, + legacy: bool = Query( + False, + description=( + "Deprecated escape hatch for import-only processing. " + "New batches should use Run Evaluation instead." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportUploadResponse: + """Deprecated IMPORT stage — use Run Evaluation for new batches. + + Recording fetch is part of the unified evaluation pipeline. This + endpoint remains available only with ``?legacy=true`` for backward + compatibility. + """ + del api_key, background_tasks + + if not legacy: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "Standalone import is deprecated. Use Run Evaluation — " + "recording fetch is part of the evaluation pipeline. " + "Append ?legacy=true to use the import-only path." + ), + ) + + from sqlalchemy.orm import selectinload as _selectinload + + call_import = ( + db.query(CallImport) + .options(_selectinload(CallImport.tags)) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + if call_import.status != CallImportStatus.MAPPED: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Cannot start import for a batch in status " + f"'{call_import.status.value}'. Map the columns first." + ), + ) + + if not call_import.source_s3_key or not call_import.source_format: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This batch has no staged source file and cannot be imported " + "through the staged flow." + ), + ) + + if not call_import.schema_id: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Cannot start import without a mapped schema.", + ) + + schema = _resolve_schema( + db, organization_id, workspace_id, call_import.schema_id + ) + parameters = list(schema.parameters) + + if payload.telephony_integration_id is not None: + integration = _resolve_telephony_integration( + db, + organization_id, + payload.telephony_integration_id, + payload.provider or "", + ) + if (integration.provider or "").lower() == "exotel": + _validate_exotel_import_ready( + parameters, dict(call_import.parameter_mapping or {}) + ) + else: + _validate_direct_url_import_ready( + parameters, dict(call_import.parameter_mapping or {}) + ) + integration = None + + _ensure_blob_storage_enabled() + + if integration is not None: + call_import.provider = integration.provider + call_import.telephony_integration_id = integration.id + else: + call_import.provider = None + call_import.telephony_integration_id = None + + call_import.total_rows = 0 + call_import.completed_rows = 0 + call_import.failed_rows = 0 + call_import.error_message = None + call_import.status = CallImportStatus.PROCESSING + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + + from app.workers.tasks.call_import_bulk_ops import ( + materialize_call_import_rows_task, + ) + + materialize_call_import_rows_task.delay( + str(call_import_id), + str(organization_id), + str(workspace_id), + schedule_import_dispatch=True, + ) + + return CallImportUploadResponse( + id=call_import.id, + total_rows=0, + status=call_import.status, + dataset=call_import.dataset, + tags=_tag_response_payload(call_import.tags), + message=( + "Import accepted. Rows are being materialized in the background; " + "recordings will be fetched asynchronously." + ), + ) + + +@router.post( + "/upload", + response_model=CallImportUploadResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="uploadCallImportCsv", + deprecated=True, +) +async def upload_call_import_csv( + background_tasks: BackgroundTasks, + file: UploadFile = File(...), + provider: Optional[str] = Form( + None, + description=( + "Telephony provider key (e.g. 'exotel', 'plivo'). Must match the " + "selected telephony_integration_id's provider. Omit together " + "with telephony_integration_id for direct-URL import." + ), + ), + telephony_integration_id: Optional[UUID] = Form( + None, + description=( + "Specific TelephonyIntegration credential row to use when " + "downloading recordings for this batch. Omit together with " + "provider for direct-URL import." + ), + ), + schema_id: UUID = Form( + ..., + description=( + "Reusable Input Parameter schema this upload is mapped against. " + "Must belong to the active workspace." + ), + ), + parameter_mapping: str = Form( + ..., + description=( + "JSON-encoded ``{schema_parameter_name: source_header}`` map " + "covering every required schema parameter. Optional parameters " + "may be omitted or set to an empty string." + ), + ), + skipped_columns: Optional[str] = Form( + None, + description=( + "JSON-encoded list of source header strings the uploader has " + "explicitly skipped. Every header in the file must either be " + "mapped or appear here; otherwise the upload is rejected so a " + "forgotten column never silently drops." + ), + ), + dataset: Optional[str] = Form( + None, + description=( + "Optional free-text dataset label for high-level segregation. " + "Empty strings are stored as NULL." + ), + ), + tag_ids: Optional[List[UUID]] = Form( + None, + description="Optional list of CallImportTag ids to attach to the new batch.", + ), + sheet_name: Optional[str] = Form( + None, + description=( + "Worksheet to import when the file is an Excel workbook " + "(.xlsx / .xlsm). REQUIRED for Excel uploads. Ignored for CSV " + "uploads (rejected with 400 if non-empty so typos surface " + "instead of silently importing the wrong source)." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportUploadResponse: + """Legacy one-shot upload kept for backward compatibility. + + DEPRECATED: prefer the staged flow + (``POST /`` → ``PATCH /{id}/mapping`` → ``POST /{id}/import``) so + each step is idempotent and resumable. This endpoint runs all three + stages inline in a single transaction so existing scripts / + integrations keep working unchanged. + """ + del api_key + + fmt = _file_format(file.filename) + if fmt is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Unsupported file format. Allowed extensions: " + f"{', '.join(ALLOWED_EXTENSIONS)}." + ), + ) + + sheet_name_clean = (sheet_name or "").strip() or None + if fmt == "csv" and sheet_name_clean is not None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is not applicable to CSV uploads.", + ) + if fmt == "xlsx" and sheet_name_clean is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sheet_name is required when uploading an Excel workbook.", + ) + + file_bytes = await file.read() + if len(file_bytes) > MAX_UPLOAD_BYTES: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", + ) + + schema = _resolve_schema(db, organization_id, workspace_id, schema_id) + parameters = list(schema.parameters) + + mapping_payload = _parse_json_form_field( + "parameter_mapping", parameter_mapping, {} + ) + cleaned_mapping = _clean_parameter_mapping( + mapping_payload, parameters, schema.name + ) + + skipped_payload = _parse_json_form_field("skipped_columns", skipped_columns, []) + cleaned_skipped = _clean_skipped_columns(skipped_payload) + + has_provider = bool((provider or "").strip()) + has_integration = telephony_integration_id is not None + if has_provider != has_integration: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "provider and telephony_integration_id must both be provided " + "or both omitted for direct-URL import." + ), + ) + + if telephony_integration_id is not None: + integration = _resolve_telephony_integration( + db, organization_id, telephony_integration_id, provider or "" + ) + if (integration.provider or "").lower() == "exotel": + _validate_exotel_import_ready(parameters, cleaned_mapping) + else: + _validate_direct_url_import_ready(parameters, cleaned_mapping) + integration = None + + parsed_rows = _parse_source_file( + file_bytes, fmt, sheet_name_clean, parameters, cleaned_mapping, cleaned_skipped + ) + _raise_if_no_importable_rows(parsed_rows, source_label=fmt) + + tag_rows = _resolve_tags(db, organization_id, tag_ids) + + call_import = CallImport( + organization_id=organization_id, + workspace_id=workspace_id, + provider=integration.provider if integration is not None else None, + telephony_integration_id=integration.id if integration is not None else None, + original_filename=file.filename, + sheet_name=sheet_name_clean, + dataset=_normalize_dataset(dataset), + schema_id=schema.id, + parameter_mapping=dict(cleaned_mapping), + skipped_columns=list(cleaned_skipped), + # Legacy columns are left empty on new uploads; the detail page + # falls back to ``parameter_mapping`` when ``schema_id`` is set. + column_mapping={}, + extra_columns=[], + custom_column_mapping={}, + total_rows=len(parsed_rows.rows), + completed_rows=0, + failed_rows=0, + status=CallImportStatus.PENDING, + source_row_skips=parse_skips_to_json(parsed_rows.skipped), + ) + if tag_rows: + call_import.tags = tag_rows + stamp_call_import_actor(call_import, principal, creating=True) + db.add(call_import) + db.flush() # populate call_import.id + if integration is None: + # The model's historical Python default is "exotel"; direct-URL + # imports intentionally have no telephony provider. + call_import.provider = None + + row_models = _materialize_rows( + db, call_import, parsed_rows.rows, organization_id + ) + + call_import.status = CallImportStatus.PROCESSING + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + + background_tasks.add_task( + record_call_import_batch_created, + organization_id, + call_import.id, + workspace_id=workspace_id, + total_rows=call_import.total_rows, + source="csv", + provider=call_import.provider, + ) + + _enqueue_row_tasks(db, call_import, row_models) + + return CallImportUploadResponse( + id=call_import.id, + total_rows=call_import.total_rows, + status=call_import.status, + dataset=call_import.dataset, + tags=_tag_response_payload(call_import.tags), + message=( + f"Accepted {call_import.total_rows} rows for import. " + "Recordings will be fetched asynchronously." + ), + ) + + +@router.post( + "/audio-upload", + response_model=CallImportUploadResponse, + status_code=status.HTTP_201_CREATED, + operation_id="uploadCallImportAudio", +) +async def upload_call_import_audio( + background_tasks: BackgroundTasks, + files: List[UploadFile] = File( + ..., + description="One or more manual call recording audio files.", + ), + dataset: str = Form( + ..., + description="Required free-text dataset label for the manual upload batch.", + ), + tag_ids: Optional[List[UUID]] = Form( + None, + description="Optional list of CallImportTag ids to attach to the new batch.", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportUploadResponse: + """Persist manually uploaded recordings as completed CallImport rows. + + The rows skip the provider-download worker entirely because the audio + bytes are already in hand. From this point onward they behave exactly + like completed CSV-import rows: playback reads ``recording_s3_key`` and + the existing diarisation/evaluation endpoints can operate on them. + """ + + normalized_dataset = _normalize_dataset(dataset) + if not normalized_dataset: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="dataset is required and must be a non-empty string.", + ) + if not files: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="At least one audio file is required.", + ) + + _ensure_blob_storage_enabled() + tag_rows = _resolve_tags(db, organization_id, tag_ids) + + max_bytes = int(settings.MAX_FILE_SIZE_MB) * 1024 * 1024 + prepared: List[Dict[str, Any]] = [] + conversation_counts: Dict[str, int] = {} + + for idx, upload in enumerate(files): + filename = upload.filename or f"recording-{idx + 1}" + ext = _audio_extension(filename) + if not ext: + allowed = ", ".join(settings.ALLOWED_AUDIO_FORMATS) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unsupported audio file '{filename}'. Allowed formats: {allowed}.", + ) + + contents = await upload.read() + if not contents: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Audio file '{filename}' is empty.", + ) + if len(contents) > max_bytes: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=( + f"Audio file '{filename}' exceeds " + f"{settings.MAX_FILE_SIZE_MB} MB." + ), + ) + + base_conversation_id = _sanitize_conversation_id(_filename_stem(filename)) + conversation_id = _dedupe_conversation_id( + base_conversation_id, + conversation_counts, + ) + prepared.append( + { + "filename": filename, + "extension": ext, + "content_type": _audio_content_type(ext, upload.content_type), + "contents": contents, + "conversation_id": conversation_id, + } + ) + + original_filename = ( + prepared[0]["filename"] + if len(prepared) == 1 + else f"{len(prepared)} manual recordings" + ) + total_size = sum(len(item["contents"]) for item in prepared) + uploaded_keys: List[str] = [] + + from app.services.storage.s3_service import s3_service + + call_import = CallImport( + organization_id=organization_id, + workspace_id=workspace_id, + provider=None, + telephony_integration_id=None, + original_filename=original_filename, + source_format="audio", + source_size_bytes=total_size, + source_content_type="audio/*", + dataset=normalized_dataset, + total_rows=len(prepared), + completed_rows=len(prepared), + failed_rows=0, + status=CallImportStatus.COMPLETED, + ) + if tag_rows: + call_import.tags = tag_rows + + stamp_call_import_actor(call_import, principal, creating=True) + try: + db.add(call_import) + db.flush() + # The model's historical Python default is "exotel"; manual uploads + # intentionally have no telephony provider. + call_import.provider = None + + row_mappings: List[Dict[str, Any]] = [] + for idx, item in enumerate(prepared): + row_id = uuid4() + key = _audio_s3_key( + organization_id, + call_import.id, + row_id, + item["extension"], + ) + s3_service.upload_file_by_key( + item["contents"], + key, + content_type=item["content_type"], + ) + uploaded_keys.append(key) + + row_mappings.append( + { + "id": row_id, + "call_import_id": call_import.id, + "organization_id": organization_id, + "workspace_id": workspace_id, + "row_index": idx, + "conversation_id": item["conversation_id"], + "recording_url": None, + "transcript": None, + "transcript_source": None, + "raw_columns": {"conversation_id": item["conversation_id"]}, + "status": CallImportRowStatus.COMPLETED, + "recording_s3_key": key, + "recording_content_type": item["content_type"], + "recording_size_bytes": len(item["contents"]), + } + ) + + if is_sharding_enabled(): + from app.db_sharding.row_ops import ( + bulk_insert_mappings_on_shards, + register_shard_slices, + ) + + bulk_insert_mappings_on_shards(db, call_import.id, row_mappings) + register_shard_slices(db, call_import.id, len(row_mappings)) + else: + for mapping in row_mappings: + db.add(CallImportRow(**mapping)) + + db.commit() + except Exception as exc: + db.rollback() + if uploaded_keys and s3_service.is_enabled(): + try: + s3_service.delete_keys(uploaded_keys) + except Exception: + logger.exception( + "Failed to clean up manual audio upload keys after error" + ) + logger.exception("Failed to persist manual call recording upload") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to upload manual recordings: {exc}", + ) from exc + + db.refresh(call_import) + background_tasks.add_task( + record_call_import_batch_created, + organization_id, + call_import.id, + workspace_id=workspace_id, + total_rows=call_import.total_rows, + source="audio", + provider=None, + ) + return CallImportUploadResponse( + id=call_import.id, + total_rows=call_import.total_rows, + status=call_import.status, + dataset=call_import.dataset, + tags=_tag_response_payload(call_import.tags), + message=( + f"Uploaded {call_import.total_rows} manual recording" + f"{'' if call_import.total_rows == 1 else 's'}." + ), + ) + + +@router.get( + "", + response_model=CallImportListResponse, + operation_id="listCallImports", +) +async def list_call_imports( + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + status_filter: Optional[CallImportStatus] = Query(None, alias="status"), + dataset: Optional[str] = Query( + None, + description=( + "Filter by exact dataset string (case-insensitive). Pass the " + "literal value '__none__' to filter to imports with no dataset." + ), + ), + tag_id: Optional[List[UUID]] = Query( + None, + description="Filter to imports tagged with ALL of the given tag ids.", + ), + source_format: Optional[str] = Query( + None, + description=( + "Filter by source format. Use 'audio' for manual recordings or " + "'__non_audio__' for CSV/Excel/legacy imports." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> CallImportListResponse: + """List call-import batches for the active workspace, newest first. + + Scoped to (organization_id, workspace_id) so users only see imports + for the workspace they're currently in. Supports a high-level + ``dataset`` filter (powers the segregation dropdown at the top of + the imports page) plus an AND-style multi-tag filter via repeated + ``tag_id`` parameters. + """ + + query = ( + db.query(CallImport) + .filter( + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + ) + if status_filter is not None: + query = query.filter(CallImport.status == status_filter) + + source_filter = (source_format or "").strip().lower() + if source_filter == "__non_audio__": + query = query.filter( + or_(CallImport.source_format.is_(None), CallImport.source_format != "audio") + ) + elif source_filter: + query = query.filter(func.lower(CallImport.source_format) == source_filter) + + if dataset is not None: + if dataset == "__none__": + query = query.filter(CallImport.dataset.is_(None)) + elif dataset.strip(): + query = query.filter( + func.lower(CallImport.dataset) == dataset.strip().lower() + ) + + if tag_id: + from app.models.database import CallImportTagAssignment + + for single_tag_id in tag_id: + sub = ( + db.query(CallImportTagAssignment.call_import_id) + .filter(CallImportTagAssignment.tag_id == single_tag_id) + .subquery() + ) + query = query.filter(CallImport.id.in_(sub)) + + total = query.count() + items = ( + query.order_by(desc(CallImport.created_at)) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) + + email_map = emails_for_user_ids(db, user_ids_from_call_imports(items)) + return CallImportListResponse( + items=[ + _serialize_call_import(db, item, user_emails=email_map) + for item in items + ], + total=total, + page=page, + page_size=page_size, + ) + + +@router.get( + "/dispatch-diagnostics", + response_model=CallImportDispatchDiagnosticsResponse, + operation_id="getCallImportDispatchDiagnostics", + dependencies=[Depends(require_admin)], +) +async def get_call_import_dispatch_diagnostics( + workspace_id: Optional[UUID] = Query( + None, + description=( + "Optional workspace filter. When omitted, returns every workspace " + "in the organization with active eval dispatch state." + ), + ), + include_idle_workspaces: bool = Query( + False, + description=( + "When true, include org workspaces with zero pending rows and " + "zero in-flight slots." + ), + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportDispatchDiagnosticsResponse: + """Live eval slot usage and fair-dispatch state for operators. + + Org admins use this to diagnose cross-workspace starvation (e.g. one + workspace's 10k run blocking another's pending eval rows) by inspecting + Redis in-flight counters, pending dispatch rows, and scheduler cursors. + """ + del api_key + payload = build_call_import_dispatch_diagnostics( + db, + organization_id, + workspace_id=workspace_id, + include_idle_workspaces=include_idle_workspaces, + ) + return CallImportDispatchDiagnosticsResponse.model_validate(payload) + + +@router.get( + "/datasets", + response_model=List[str], + operation_id="listCallImportDatasets", +) +async def list_call_import_datasets( + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> List[str]: + """Return the distinct, non-null dataset labels in use for the active + workspace. + + Scoped per-workspace so each workspace's Dataset dropdown only shows + its own segregation labels. + """ + rows = ( + db.query(CallImport.dataset) + .filter( + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + CallImport.dataset.isnot(None), + CallImport.dataset != "", + ) + .distinct() + .order_by(CallImport.dataset.asc()) + .all() + ) + return [row[0] for row in rows if row[0]] + + +@router.get( + "/diarisation-prompt-default", + response_model=CallImportDiarisationPromptDefaultResponse, + operation_id="getCallImportDiarisationPromptDefault", +) +async def get_call_import_diarisation_prompt_default( + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), +) -> CallImportDiarisationPromptDefaultResponse: + """Return the canonical LLM diariser prompt. + + The Transcribe / Run Evaluation modals call this on open so they + can pre-fill the prompt textarea. Returning the constant from the + backend (rather than hard-coding it in the frontend) keeps the + fallback used by the worker and the placeholder shown in the UI + in lock-step — operators always see the *actual* default they'd + get if they leave the field blank. + + Registered before ``GET /{call_import_id}`` so the static path is + not mistaken for a UUID import id (which would 422). + """ + del api_key, organization_id + from app.workers.tasks.helpers.llm_diarisation import ( + DEFAULT_DIARIZATION_PROMPT, + ) + + return CallImportDiarisationPromptDefaultResponse( + prompt=DEFAULT_DIARIZATION_PROMPT + ) + + +@router.patch( + "/{call_import_id}", + response_model=CallImportResponse, + operation_id="updateCallImport", +) +async def update_call_import( + call_import_id: UUID, + payload: CallImportUpdate, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportResponse: + """Edit dataset / tag assignments (and schema, pre-import) on a batch. + + ``dataset = ""`` clears the label; ``tag_ids = []`` removes all tag + assignments. Fields omitted from the body are left untouched. + + ``schema_id`` is only honoured while the batch is in + ``uploaded`` / ``mapped`` state — once rows have been materialised + the schema is locked. Changing the schema resets any persisted + mapping (the user must re-MAP) and rewinds status to ``uploaded``. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + body = payload.model_dump(exclude_unset=True) + if "dataset" in body: + call_import.dataset = _normalize_dataset(body["dataset"]) + + if "tag_ids" in body: + tag_ids = body["tag_ids"] or [] + call_import.tags = _resolve_tags(db, organization_id, tag_ids) + + if "schema_id" in body and body["schema_id"] is not None: + if call_import.status not in ( + CallImportStatus.UPLOADED, + CallImportStatus.MAPPED, + ): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Cannot reassign schema on a batch in status " + f"'{call_import.status.value}'." + ), + ) + new_schema = _resolve_schema( + db, organization_id, workspace_id, body["schema_id"] + ) + if call_import.schema_id != new_schema.id: + # Switching schemas invalidates the persisted mapping — + # parameter names won't line up with the new schema, so + # reset to UPLOADED and force a fresh MAP. + call_import.schema_id = new_schema.id + call_import.parameter_mapping = {} + call_import.skipped_columns = [] + call_import.sheet_name = None + call_import.status = CallImportStatus.UPLOADED + + stamp_call_import_actor(call_import, principal) + db.commit() + db.refresh(call_import) + return _serialize_call_import(db, call_import) + + +@router.get( + "/{call_import_id}", + response_model=CallImportDetailResponse, + operation_id="getCallImportDetail", +) +async def get_call_import_detail( + call_import_id: UUID, + row_limit: int = Query(500, ge=0, le=5000), + row_offset: int = Query(0, ge=0), + q: Optional[str] = Query( + None, + description=( + "Optional case-insensitive substring filter on " + "``conversation_id``. When set, ``filtered_total_rows`` in " + "the response reflects the post-filter row count so the UI " + "can paginate against the filtered slice." + ), + ), + diarised_status: Optional[str] = Query( + None, + description=( + "Optional filter on ``CallImportRow.diarised_transcript_status``. " + "Accepts one of ``pending``, ``running``, ``completed``, " + "``failed``. When set, ``filtered_total_rows`` reflects the " + "post-filter row count (combined with the ``q`` filter when " + "both are supplied) so the UI can paginate against the same " + "slice it's displaying." + ), + pattern="^(pending|running|completed|failed)$", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportDetailResponse: + """Fetch a single import batch with a slice of its rows. + + ``row_limit=0`` is intentionally allowed so callers that only need the + batch metadata (e.g. the evaluation-detail page rendering the parent's + column mapping) can skip the rows payload entirely. + """ + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + if call_import.status == CallImportStatus.PROCESSING: + from app.services.call_imports.bulk_ops import rollup_call_import_batch_status + + prior_status = call_import.status + rollup_call_import_batch_status(db, call_import) + if call_import.status != prior_status: + db.commit() + db.refresh(call_import) + + search_term = (q or "").strip() + diarised_status_filter = (diarised_status or "").strip() or None + filtered_total_rows: Optional[int] = None + has_row_filters = bool(search_term or diarised_status_filter) + + if has_row_filters: + if is_sharding_enabled(): + from app.db_sharding.scatter_gather import count_call_import_rows_filtered + + filtered_total_rows = count_call_import_rows_filtered( + db, + call_import.id, + search_term=search_term, + diarised_status_filter=diarised_status_filter, + ) + else: + rows_query = db.query(CallImportRow).filter( + CallImportRow.call_import_id == call_import.id + ) + if search_term: + rows_query = rows_query.filter( + CallImportRow.conversation_id.ilike(f"%{search_term}%") + ) + if diarised_status_filter: + rows_query = rows_query.filter( + CallImportRow.diarised_transcript_status == diarised_status_filter + ) + filtered_total_rows = rows_query.count() + + if row_limit == 0: + rows: List[CallImportRow] = [] + elif is_sharding_enabled(): + from app.db_sharding.scatter_gather import ( + fetch_call_import_rows_filtered_page, + fetch_call_import_rows_page, + ) + + if has_row_filters: + rows = fetch_call_import_rows_filtered_page( + db, + call_import.id, + search_term=search_term, + diarised_status_filter=diarised_status_filter, + offset=row_offset, + limit=row_limit, + ) + else: + rows = fetch_call_import_rows_page( + db, + call_import.id, + offset=row_offset, + limit=row_limit, + ) + else: + rows_query = db.query(CallImportRow).filter( + CallImportRow.call_import_id == call_import.id + ) + if search_term: + rows_query = rows_query.filter( + CallImportRow.conversation_id.ilike(f"%{search_term}%") + ) + if diarised_status_filter: + rows_query = rows_query.filter( + CallImportRow.diarised_transcript_status == diarised_status_filter + ) + rows = ( + rows_query.order_by(CallImportRow.row_index) + .offset(row_offset) + .limit(row_limit) + .all() + ) + + # Batch-wide diarisation status aggregate. One ``GROUP BY`` query + # across the whole batch — much cheaper than paging through every + # row to recount on the client and lets the UI render a + # transcribe/diarise progress bar without a separate roundtrip. + if is_sharding_enabled(): + from app.db_sharding.scatter_gather import aggregate_diarised_transcript_counts + + diarised_status_counts = aggregate_diarised_transcript_counts( + db, call_import.id + ) + else: + diarised_status_counts: Dict[str, int] = {} + for status_value, count in ( + db.query(CallImportRow.diarised_transcript_status, func.count()) + .filter(CallImportRow.call_import_id == call_import.id) + .group_by(CallImportRow.diarised_transcript_status) + .all() + ): + if isinstance(status_value, str): + diarised_status_counts[status_value] = int(count or 0) + + detail = CallImportDetailResponse.model_validate( + _serialize_call_import(db, call_import).model_dump() + ) + detail.rows = [CallImportRowResponse.model_validate(r) for r in rows] + detail.filtered_total_rows = filtered_total_rows + detail.diarised_pending_rows = diarised_status_counts.get("pending", 0) + detail.diarised_running_rows = diarised_status_counts.get("running", 0) + detail.diarised_completed_rows = diarised_status_counts.get("completed", 0) + detail.diarised_failed_rows = diarised_status_counts.get("failed", 0) + return detail + + +@router.get( + "/{call_import_id}/row-ids", + response_model=CallImportRowIdsResponse, + operation_id="listCallImportRowIds", +) +async def list_call_import_row_ids( + call_import_id: UUID, + q: Optional[str] = Query( + None, + description=( + "Optional case-insensitive substring filter on " + "``conversation_id``. Same semantics as the detail endpoint." + ), + ), + diarised_status: Optional[str] = Query( + None, + description=( + "Optional filter on ``CallImportRow.diarised_transcript_status``. " + "Accepts ``pending`` / ``running`` / ``completed`` / ``failed``." + ), + pattern="^(pending|running|completed|failed)$", + ), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportRowIdsResponse: + """Return every matching ``CallImportRow.id`` for cross-page bulk select. + + Lightweight companion to ``GET /{call_import_id}`` — the detail + endpoint caps ``row_limit`` at 5000 and ships the entire row body + on each page, so harvesting ids that way is wasteful when the + user just wants to bulk-delete or bulk-transcribe everything that + matches the current filters. This endpoint applies the same ``q`` + and ``diarised_status`` filters and returns only the ids. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + search_term = (q or "").strip() + status_filter = (diarised_status or "").strip() or None + + if is_sharding_enabled(): + from app.db_sharding.scatter_gather import list_call_import_row_ids_filtered + + ids = list_call_import_row_ids_filtered( + db, + call_import.id, + search_term=search_term, + diarised_status_filter=status_filter, + ) + return CallImportRowIdsResponse(ids=ids, total=len(ids)) + + rows_query = db.query(CallImportRow.id).filter( + CallImportRow.call_import_id == call_import.id + ) + if search_term: + rows_query = rows_query.filter( + CallImportRow.conversation_id.ilike(f"%{search_term}%") + ) + if status_filter: + rows_query = rows_query.filter( + CallImportRow.diarised_transcript_status == status_filter + ) + + ids = [ + row_id + for (row_id,) in rows_query.order_by(CallImportRow.row_index).all() + ] + return CallImportRowIdsResponse(ids=ids, total=len(ids)) + + +def _revoke_pending_tasks(rows: List[CallImportRow]) -> None: + """Best-effort revoke of in-flight Celery tasks for the given rows. + + Failures are logged and swallowed — Celery's control plane is async and + best-effort by design, and we always do an idempotent S3 cleanup + afterwards so a missed revoke can't leak storage. + """ + task_ids = [ + r.celery_task_id + for r in rows + if r.celery_task_id + and r.status in (CallImportRowStatus.PENDING, CallImportRowStatus.PROCESSING) + ] + if not task_ids: + return + + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke(task_ids, terminate=False) + logger.info("Revoked {} pending call-import tasks", len(task_ids)) + except Exception as exc: # noqa: BLE001 + logger.warning("Failed to revoke pending call-import tasks: {}", exc) + + +def _delete_s3_objects( + organization_id: UUID, + call_import_id: UUID, + rows: List[CallImportRow], +) -> tuple[int, int]: + """Delete every recording associated with ``rows`` plus a prefix sweep. + + The prefix sweep also cleans up the staged source file written at + UPLOAD time (``…/call_imports/{id}/source.{csv,xlsx}``) — both the + per-row recording keys and the source artefact share the same + organization-scoped prefix, so a single sweep covers them all. + + Returns ``(deleted_count, error_count)``. Never raises — callers proceed + with the DB delete regardless; orphans, if any, can be cleaned up by + re-running the same delete (it's idempotent). + """ + from app.services.storage.s3_service import s3_service + + if not s3_service.is_enabled(): + return 0, 0 + + keys = [r.recording_s3_key for r in rows if r.recording_s3_key] + deleted = 0 + errors = 0 + + if keys: + try: + d, errs = s3_service.delete_keys(keys) + deleted += d + errors += len(errs) + if errs: + logger.warning( + "S3 bulk-delete reported {} errors for call_import {}", + len(errs), + call_import_id, + ) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Bulk S3 delete failed for call_import {}: {}", call_import_id, exc + ) + errors += len(keys) + + # Belt-and-braces sweep: catch anything that landed under the import's + # prefix but never made it into a row's recording_s3_key (narrow + # window where the S3 upload succeeded but the DB commit didn't). + sweep_prefix = ( + f"{s3_service.prefix}organizations/{organization_id}/" + f"call_imports/{call_import_id}/" + ) + try: + d, errs = s3_service.delete_keys_by_prefix(sweep_prefix) + deleted += d + errors += len(errs) + except Exception as exc: # noqa: BLE001 + logger.exception( + "S3 prefix sweep failed for {}: {}", sweep_prefix, exc + ) + + return deleted, errors + + +@router.delete( + "/{call_import_id}", + response_model=CallImportDeleteResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="deleteCallImport", +) +async def delete_call_import( + call_import_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportDeleteResponse: + """Delete a call-import batch asynchronously. + + Flips the batch to ``deleting`` and enqueues background teardown so + large imports (thousands of rows + S3 objects) do not block the API. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + return CallImportDeleteResponse( + id=call_import_id, + status="completed", + ) + + if call_import.status == CallImportStatus.DELETING: + return CallImportDeleteResponse( + id=call_import.id, + status="accepted", + ) + + call_import.status = CallImportStatus.DELETING + call_import.error_message = None + stamp_call_import_actor(call_import, principal) + db.commit() + + from app.workers.tasks.call_import_bulk_ops import delete_call_import_task + + delete_call_import_task.delay( + str(call_import_id), + str(organization_id), + ) + + return CallImportDeleteResponse( + id=call_import.id, + status="accepted", + ) + + +def _locate_call_import_row_or_404( + catalog_db: Session, + *, + call_import_id: UUID, + row_id: UUID, + organization_id: UUID, +) -> Tuple[Session, CallImportRow, Optional[Session]]: + """Find a call import row on the correct DB session for mutation. + + When sharding is enabled rows live on shard databases; ``get_db`` only + opens the catalog. Returns ``(row_db, row, extra_catalog_to_close)`` + where ``extra_catalog_to_close`` is the catalog session opened by + :func:`locate_call_import_row` (distinct from the route's catalog + session) and must be closed via :func:`close_row_sessions`. + """ + from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row + + try: + row_db, located_catalog, row, _shard_id = locate_call_import_row(row_id) + except LookupError: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import row not found", + ) from None + if ( + row.call_import_id != call_import_id + or row.organization_id != organization_id + ): + close_row_sessions( + row_db, + located_catalog if located_catalog is not row_db else None, + ) + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import row not found", + ) + extra_catalog = located_catalog if located_catalog is not row_db else None + return row_db, row, extra_catalog + + +@router.delete( + "/{call_import_id}/rows/{row_id}", + status_code=status.HTTP_204_NO_CONTENT, + operation_id="deleteCallImportRow", +) +async def delete_call_import_row( + call_import_id: UUID, + row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> Response: + """Delete a single CallImportRow and its S3 recording. + + The parent ``CallImport`` is left in place. After deletion we recompute + its ``total_rows`` / ``completed_rows`` / ``failed_rows`` / ``status`` + so the UI's progress bar stays consistent with reality. + """ + from app.services.storage.s3_service import s3_service + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.db_sharding.row_ops import close_row_sessions + + row_db, row, extra_catalog = _locate_call_import_row_or_404( + db, + call_import_id=call_import.id, + row_id=row_id, + organization_id=organization_id, + ) + try: + _revoke_pending_tasks([row]) + + if row.recording_s3_key and s3_service.is_enabled(): + try: + s3_service.delete_file_by_key(row.recording_s3_key) + except Exception as exc: # noqa: BLE001 — best-effort, DB is source of truth + logger.warning( + "Failed to delete S3 object {} for row {}: {}", + row.recording_s3_key, + row.id, + exc, + ) + + row_db.delete(row) + row_db.commit() + + _recompute_call_import_counters(db, call_import) + stamp_call_import_actor(call_import, principal) + db.commit() + finally: + close_row_sessions(row_db, extra_catalog) + + logger.info( + "Deleted call_import_row {} (call_import={}, org={})", + row_id, + call_import.id, + organization_id, + ) + + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +def _recompute_call_import_counters( + db: Session, call_import: CallImport +) -> None: + """Resync ``total/completed/failed_rows`` + status on the parent batch. + + Called after row-level mutations (single delete, bulk delete) so the + UI's progress bar stays consistent with the actual row set. The + rules mirror :func:`delete_call_import_row` so behavior doesn't + diverge between the per-row and bulk paths. + """ + + from app.services.call_imports.bulk_ops import rollup_call_import_batch_status + + rollup_call_import_batch_status(db, call_import) + + +@router.post( + "/{call_import_id}/retry-failed", + response_model=CallImportRetryFailedRowsResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="retryFailedCallImportRows", +) +async def retry_failed_call_import_rows( + call_import_id: UUID, + payload: Optional[CallImportRetryFailedRowsRequest] = Body(None), + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportRetryFailedRowsResponse: + """Re-enqueue every failed import row in this batch. + + Useful when transient provider issues are resolved and the operator wants + a one-click "try failed downloads again" pass without re-uploading the CSV. + + Pass ``provider`` + ``telephony_integration_id`` (or both omitted for + direct-URL retry) to change how recordings are fetched on this pass. + When the body is omitted entirely, the batch keeps its existing pinned + credentials. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + if payload is not None: + if payload.telephony_integration_id is not None: + integration = _resolve_telephony_integration( + db, + organization_id, + payload.telephony_integration_id, + payload.provider or "", + ) + call_import.provider = integration.provider + call_import.telephony_integration_id = integration.id + if (integration.provider or "").lower() == "exotel": + schema = _resolve_schema( + db, + organization_id, + call_import.workspace_id, + call_import.schema_id, + ) + _validate_exotel_import_ready( + list(schema.parameters), + dict(call_import.parameter_mapping or {}), + ) + else: + call_import.provider = None + call_import.telephony_integration_id = None + db.flush() + + failed_rows = ( + db.query(CallImportRow) + .filter( + CallImportRow.call_import_id == call_import.id, + CallImportRow.status == CallImportRowStatus.FAILED, + ) + .order_by(CallImportRow.row_index.asc()) + .all() + ) + if not failed_rows: + return CallImportRetryFailedRowsResponse( + requeued=0, + enqueue_failed=0, + skipped=0, + ) + + from app.workers.concurrency.fair_import_dispatch import ( + schedule_fair_import_dispatch, + ) + + # Reset rows to pending BEFORE enqueue so the UI reflects "retry in + # progress" immediately even if the worker queue is backlogged. + for row in failed_rows: + row.status = CallImportRowStatus.PENDING + row.error_message = None + row.celery_task_id = None + + db.flush() + _recompute_call_import_counters(db, call_import) + stamp_call_import_actor(call_import, principal) + db.commit() + + try: + schedule_fair_import_dispatch(max_workspace_turns=999) + requeued = len(failed_rows) + enqueue_failed = 0 + skipped = 0 + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to schedule fair import dispatch for import {}", + call_import.id, + ) + requeued = 0 + enqueue_failed = len(failed_rows) + skipped = 0 + for row in failed_rows: + db.refresh(row) + if row.status != CallImportRowStatus.PENDING: + skipped += 1 + enqueue_failed -= 1 + continue + row.status = CallImportRowStatus.FAILED + row.error_message = f"Failed to enqueue retry: {exc}" + db.flush() + _recompute_call_import_counters(db, call_import) + db.commit() + + return CallImportRetryFailedRowsResponse( + requeued=requeued, + enqueue_failed=enqueue_failed, + skipped=skipped, + ) + + +@router.post( + "/{call_import_id}/rows/bulk-delete", + response_model=CallImportRowBulkDeleteResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="bulkDeleteCallImportRows", +) +async def bulk_delete_call_import_rows( + call_import_id: UUID, + payload: CallImportRowBulkDelete, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportRowBulkDeleteResponse: + """Delete multiple ``CallImportRow`` rows in one request. + + Unknown / cross-tenant row ids are silently skipped — the response + reports how many actually went away so a UI that holds onto stale + ids (e.g. after another tab already deleted a row) doesn't 404 + the entire bulk action. + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + if not payload.row_ids: + return CallImportRowBulkDeleteResponse(deleted=0, status="completed") + + from app.workers.tasks.call_import_bulk_ops import bulk_delete_call_import_rows_task + + row_id_strs = [str(rid) for rid in payload.row_ids] + + stamp_call_import_actor(call_import, principal) + db.commit() + + bulk_delete_call_import_rows_task.delay( + str(call_import_id), + str(organization_id), + row_id_strs, + ) + + return CallImportRowBulkDeleteResponse(deleted=0, status="accepted") + + +# --------------------------------------------------------------------------- +# Diarization / transcription endpoints +# --------------------------------------------------------------------------- + + +def _select_rows_for_transcription( + db: Session, + call_import: CallImport, + payload: CallImportTranscribeRequest, + requested_row_ids: Optional[List[UUID]] = None, +) -> tuple[List[CallImportRow], Dict[str, int]]: + """Pick which rows to enqueue for diarisation (delegates to bulk_ops).""" + from app.services.call_imports.bulk_ops import select_rows_for_transcription + + try: + return select_rows_for_transcription( + db, call_import, payload, requested_row_ids=requested_row_ids + ) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc), + ) from exc + + +@router.post( + "/{call_import_id}/transcribe", + response_model=CallImportTranscribeResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="transcribeCallImport", +) +async def transcribe_call_import( + call_import_id: UUID, + payload: CallImportTranscribeRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportTranscribeResponse: + """Fan out diarization tasks for many rows in a single call. + + Returns a summary with how many rows were queued and how many were + skipped (broken down by reason) so the UI can show a meaningful + toast even when nothing actually got enqueued (e.g. "All 12 rows + already have transcripts"). + """ + + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.workers.tasks.call_import_bulk_ops import bulk_diarize_call_import_task + + stamp_call_import_actor(call_import, principal) + db.commit() + + bulk_diarize_call_import_task.delay( + str(call_import_id), + str(organization_id), + payload.model_dump(mode="json"), + [str(rid) for rid in payload.row_ids] if payload.row_ids else None, + ) + + return CallImportTranscribeResponse( + queued=0, + skipped_rows=0, + skipped_reason_counts={}, + accepted=True, + ) + + +@router.post( + "/{call_import_id}/rows/{row_id}/transcribe", + response_model=CallImportTranscribeResponse, + status_code=status.HTTP_202_ACCEPTED, + operation_id="transcribeCallImportRow", +) +async def transcribe_call_import_row( + call_import_id: UUID, + row_id: UUID, + payload: CallImportTranscribeRequest, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportTranscribeResponse: + """Diarize / transcribe a single row. + + Thin wrapper over the batch endpoint that hard-codes a single + ``row_ids`` filter. Skip counts still surface so the UI can render + "Skipped — transcript present" diagnostics consistently. + """ + + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.services.call_imports.bulk_ops import execute_bulk_diarization + + try: + result = execute_bulk_diarization( + db, + call_import, + payload, + requested_row_ids=[row_id], + ) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc), + ) from exc + + stamp_call_import_actor(call_import, principal) + db.commit() + + return CallImportTranscribeResponse( + queued=result.queued, + skipped_rows=result.skipped_rows, + skipped_reason_counts=result.skipped_reason_counts, + ) + + +# --------------------------------------------------------------------------- +# Cancel-in-flight diarisation +# --------------------------------------------------------------------------- +# +# Long-running multimodal LLM diarisation calls (especially LLM-only mode on +# slow audio) can sit in ``pending`` / ``running`` for tens of minutes when an +# upstream provider stalls. Without an abort affordance the operator's only +# recourse is to wait for Celery's ``time_limit`` to fire — which can be +# several minutes — or to manually mutate the DB. These helpers + the two +# endpoints below give the UI a first-class "Stop diarisation" button. +# +# Why ``terminate=True``: the legacy ``_revoke_pending_tasks`` helper uses +# ``terminate=False`` because it's called from delete-flow paths where the +# task may simply not get to run (a worker pulls it off the queue and drops +# it). For a user-initiated cancel we want SIGTERM to interrupt the worker +# mid-LLM call so the audio HTTP request actually aborts. ``terminate=True`` +# routes SIGTERM to the executing process; ``signal="SIGTERM"`` is the +# default but we spell it out so the intent is obvious to reviewers. + +# Sentinel error message stamped on cancelled rows. Read by the transcribe +# worker's finaliser (see ``app/workers/tasks/transcribe_call_import_row.py``) +# to detect a row that was cancelled mid-flight and AVOID overwriting it +# with whatever partial result the worker had managed to compute before the +# SIGTERM landed. +CANCELLED_BY_USER_ERROR: str = "Diarisation cancelled by user" + + +def _cancellable_diarisation_states() -> Tuple[str, ...]: + """States that a diarisation row can be cancelled from. + + Kept as a tiny helper so adding a future ``"queued"`` / ``"retrying"`` + state only needs one edit. + """ + return ("pending", "running") + + +def _revoke_diarisation_task(row: CallImportRow) -> None: + """Best-effort revoke of a single row's diarisation Celery task. + + Always swallows control-plane exceptions — Celery's control bus is + inherently best-effort and a missed revoke is not catastrophic + because the DB row is already flipped to ``failed`` by the caller + before this runs (so the UI immediately reflects the cancel; if + the task happens to finish anyway, the worker's finaliser skips + over the row via :data:`CANCELLED_BY_USER_ERROR`). + """ + task_id = (row.celery_task_id or "").strip() + if not task_id: + return + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke( + task_id, terminate=True, signal="SIGTERM" + ) + logger.info( + "Revoked diarisation task {} for call-import row {}", + task_id, + row.id, + ) + except Exception as exc: # noqa: BLE001 — revoke is best-effort + logger.warning( + "Failed to revoke diarisation task {} for row {}: {}", + task_id, + row.id, + exc, + ) + + +def _apply_diarisation_cancel(rows: List[CallImportRow]) -> Tuple[int, int]: + """Cancel diarisation on every cancellable row in ``rows``. + + Returns ``(cancelled, skipped)`` so the caller can build a typed + response without re-querying the DB. The caller is responsible for + ``db.commit()`` after this returns — we deliberately don't commit + here so a batch endpoint can flush all rows in one transaction. + """ + cancellable_states = _cancellable_diarisation_states() + cancelled = 0 + skipped = 0 + for row in rows: + if (row.diarised_transcript_status or "").lower() not in cancellable_states: + skipped += 1 + continue + # Flip the row state BEFORE we revoke so the UI's next poll + # already shows the cancel, even if Celery's control plane is + # slow to ack. + row.diarised_transcript_status = "failed" + row.diarised_transcript_error = CANCELLED_BY_USER_ERROR + _revoke_diarisation_task(row) + # Drop the task id so a follow-up retry (or a stale poll) can't + # accidentally re-revoke or get confused. + row.celery_task_id = None + cancelled += 1 + return cancelled, skipped + + +@router.post( + "/{call_import_id}/rows/{row_id}/cancel-diarisation", + response_model=CallImportRowResponse, + status_code=status.HTTP_200_OK, + operation_id="cancelCallImportRowDiarisation", +) +async def cancel_call_import_row_diarisation( + call_import_id: UUID, + row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportRowResponse: + """Abort an in-flight (or queued) diarisation for a single row. + + Idempotent: calling on a row that's already terminal (``completed`` + / ``failed`` / ``idle``) returns the row unchanged with a 200, so + the UI can fire this from a "Stop" button without having to + pre-check the state. + + Race notes: + + * The row's ``diarised_transcript_status`` is flipped to ``failed`` + with :data:`CANCELLED_BY_USER_ERROR` BEFORE the Celery revoke, + so the polling UI sees the cancel immediately. + * If the worker happens to finish between our DB flip and the + SIGTERM landing, its finaliser will detect the cancelled + sentinel on the row and skip its own status / score writes + (see :mod:`app.workers.tasks.transcribe_call_import_row`). + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.db_sharding.row_ops import close_row_sessions + + row_db, row, extra_catalog = _locate_call_import_row_or_404( + db, + call_import_id=call_import_id, + row_id=row_id, + organization_id=organization_id, + ) + try: + _apply_diarisation_cancel([row]) + row_db.commit() + stamp_call_import_actor(call_import, principal) + db.commit() + row_db.refresh(row) + return CallImportRowResponse.model_validate(row) + finally: + close_row_sessions(row_db, extra_catalog) + + +@router.post( + "/{call_import_id}/cancel-diarisation", + response_model=CallImportCancelDiarisationResponse, + status_code=status.HTTP_200_OK, + operation_id="cancelCallImportDiarisation", +) +async def cancel_call_import_diarisation( + call_import_id: UUID, + payload: Optional[CallImportCancelDiarisationRequest] = None, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportCancelDiarisationResponse: + """Abort in-flight diarisation for many rows in a single call. + + Default body (no ``row_ids``) cancels every row in this import + whose ``diarised_transcript_status`` is ``pending`` or + ``running`` — the "stop everything" button. Pass ``row_ids`` to + scope the cancel to the rows the operator has selected. + + Returns ``(cancelled, skipped)`` so the UI can render a tight + toast ("Cancelled 3 rows · 1 skipped (already completed)"). + """ + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + base_query = db.query(CallImportRow).filter( + CallImportRow.call_import_id == call_import_id + ) + + requested_ids = ( + payload.row_ids if payload and payload.row_ids is not None else None + ) + if requested_ids is not None: + if not requested_ids: + # Empty list is "no rows requested" — treat as a no-op + # 200 rather than 400 so the UI can pass through an empty + # selection without a special-case. + return CallImportCancelDiarisationResponse(cancelled=0, skipped=0) + rows = base_query.filter(CallImportRow.id.in_(requested_ids)).all() + found_ids = {r.id for r in rows} + # Treat requested-but-not-found ids as ``skipped`` so the UI's + # numbers reconcile (a stale selection that includes deleted + # rows shouldn't 404 the whole call). + missing = [rid for rid in requested_ids if rid not in found_ids] + skipped_missing = len(missing) + else: + # Implicit "cancel every cancellable row in this import" path. + rows = base_query.filter( + CallImportRow.diarised_transcript_status.in_( + list(_cancellable_diarisation_states()) + ) + ).all() + skipped_missing = 0 + + cancelled, skipped = _apply_diarisation_cancel(rows) + stamp_call_import_actor(call_import, principal) + db.commit() + return CallImportCancelDiarisationResponse( + cancelled=cancelled, + skipped=skipped + skipped_missing, + ) + + +def _render_diarised_segments_text( + segments: Optional[List[Dict[str, Any]]], + *, + swap: bool = False, +) -> str: + """Render ``CallImportRow.diarised_segments`` as ``: `` lines. + + Mirrors the worker's ``_render_turns_as_text`` (kept duplicated so + the route doesn't need to import a Celery task module just to + rebuild the rendered transcript). Only ``agent`` and ``user`` are + swapped — multi-party calls keep their ``speaker_N`` labels through + a swap so we don't silently collapse a third speaker into the user + side. + """ + if not segments: + return "" + out: List[str] = [] + for turn in segments: + if not isinstance(turn, dict): + continue + speaker = (turn.get("speaker") or "").strip() + text = (turn.get("text") or "").strip() + if not speaker or not text: + continue + if swap: + if speaker == "agent": + speaker = "user" + elif speaker == "user": + speaker = "agent" + out.append(f"{speaker}: {text}") + return "\n".join(out) + + +@router.post( + "/{call_import_id}/rows/{row_id}/diarised-speaker-swap", + response_model=CallImportRowResponse, + operation_id="toggleCallImportRowSpeakerSwap", +) +async def toggle_call_import_row_speaker_swap( + call_import_id: UUID, + row_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +) -> CallImportRowResponse: + """Flip the user <-> agent mapping on a diarised row. + + The worker's "first speaker is the agent" heuristic is right most of + the time but does fail on inbound recordings where the customer + greets first, on recordings where the agent stays silent for the + intro, etc. Rather than rerun the (expensive) STT + pyannote + pipeline for those cases, we let reviewers flip the mapping in + place: the structured ``diarised_segments`` are the source of truth + and we re-render the plain-text ``diarised_transcript`` from them + with the swap applied. The next CSV export will then show the + corrected labels. + + Returns the updated row so the frontend can refresh without an + extra round-trip. + """ + + del api_key + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + from app.db_sharding.row_ops import close_row_sessions + + row_db, row, extra_catalog = _locate_call_import_row_or_404( + db, + call_import_id=call_import_id, + row_id=row_id, + organization_id=organization_id, + ) + try: + segments = ( + row.diarised_segments if isinstance(row.diarised_segments, list) else None + ) + if not segments: + # Without structured turns the swap toggle would have nothing to + # re-render — surface a clear error rather than silently + # flipping a flag the UI never read. + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This row has no structured diarised segments to swap. " + "Re-run diarisation to generate per-speaker turns first." + ), + ) + + new_swap = not bool(row.diarised_speaker_swap) + row.diarised_speaker_swap = new_swap + row.diarised_transcript = ( + _render_diarised_segments_text(segments, swap=new_swap) or None + ) + row_db.commit() + stamp_call_import_actor(call_import, principal) + db.commit() + row_db.refresh(row) + return CallImportRowResponse.model_validate(row) + finally: + close_row_sessions(row_db, extra_catalog) + + +# --------------------------------------------------------------------------- +# Cross-run insights for the import detail page +# --------------------------------------------------------------------------- + + +@router.get( + "/{call_import_id}/insights", + response_model=CallImportInsightsResponse, + operation_id="getCallImportInsights", +) +async def get_call_import_insights( + call_import_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportInsightsResponse: + """Aggregate signals across every evaluation run on this import. + + Powers the Insights tab on the call-import detail page: returns + per-metric "latest run" summaries plus a trend series of mean values + across runs so the UI can render a small line chart per metric. Also + bundles transcript coverage stats since those are the cheapest + pre-eval health-check (e.g. "30 of 50 rows still missing + transcripts"). + """ + + del api_key + + from app.models.database import ( + CallImportEvaluation, + CallImportEvaluationRow, + Metric, + ) + + call_import = ( + db.query(CallImport) + .filter( + CallImport.id == call_import_id, + CallImport.organization_id == organization_id, + ) + .first() + ) + if not call_import: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import not found", + ) + + rows = ( + db.query(CallImportRow) + .filter(CallImportRow.call_import_id == call_import_id) + .all() + ) + # A row "has a transcript" if EITHER the production (CSV) or the + # diarised (worker) column is populated — the insights tile reports + # the union so users see total coverage regardless of which source + # produced the value. + rows_with_transcript = sum( + 1 + for r in rows + if (r.transcript or "").strip() + or (r.diarised_transcript or "").strip() + ) + rows_without_transcript = len(rows) - rows_with_transcript + source_counts: Dict[str, int] = {} + for r in rows: + has_production = bool((r.transcript or "").strip()) + has_diarised = bool((r.diarised_transcript or "").strip()) + if has_production: + key = r.transcript_source or "csv" + source_counts[key] = source_counts.get(key, 0) + 1 + if has_diarised: + source_counts["diarised"] = source_counts.get("diarised", 0) + 1 + + evaluations = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .order_by(CallImportEvaluation.created_at.asc()) + .all() + ) + + # Defer heavy lifting to the aggregation helper so this endpoint and + # the per-run aggregate endpoint share the exact same metric + # bucketing math (no chance of "trend" disagreeing with "latest" on + # the same data set). + from app.api.v1.routes.call_import_evaluations import ( + _compute_metric_aggregates, + ) + + metric_history: Dict[str, List[CallImportInsightsRunPoint]] = {} + metric_meta: Dict[str, Metric] = {} + metric_latest: Dict[str, CallImportMetricAggregate] = {} + + for evaluation in evaluations: + eval_rows = ( + db.query(CallImportEvaluationRow) + .filter(CallImportEvaluationRow.evaluation_id == evaluation.id) + .all() + ) + aggregates = _compute_metric_aggregates(db, evaluation, eval_rows) + for agg in aggregates: + if agg.metric_id not in metric_meta: + # ``agg.metric_id`` is normally a UUID string, but the + # aggregator also emits ids that surface in row scores + # without a matching ``Metric`` row (e.g. a metric the + # user deleted mid-run, or LLM-discovered slugs). Those + # are not valid UUIDs, so coerce defensively and skip + # the metric registry lookup when the cast fails — the + # ``meta is None`` branch below already handles the + # display via the values stored on ``agg`` itself. + try: + metric_uuid = UUID(agg.metric_id) + except (ValueError, AttributeError, TypeError): + metric_uuid = None + if metric_uuid is not None: + metric_obj = ( + db.query(Metric) + .filter( + Metric.id == metric_uuid, + Metric.organization_id == organization_id, + ) + .first() + ) + if metric_obj is not None: + metric_meta[agg.metric_id] = metric_obj + history = metric_history.setdefault(agg.metric_id, []) + history.append( + CallImportInsightsRunPoint( + evaluation_id=evaluation.id, + name=evaluation.name, + created_at=evaluation.created_at, + mean=agg.mean, + completed_rows=agg.count, + ) + ) + metric_latest[agg.metric_id] = agg + + metrics_payload: List[CallImportInsightsMetric] = [] + for metric_id, latest in metric_latest.items(): + meta = metric_meta.get(metric_id) + metrics_payload.append( + CallImportInsightsMetric( + metric_id=metric_id, + metric_name=(meta.name if meta else latest.metric_name), + metric_type=(meta.metric_type if meta else latest.metric_type), + latest=latest, + trend=metric_history.get(metric_id, []), + ) + ) + + return CallImportInsightsResponse( + call_import_id=call_import_id, + total_rows=len(rows), + rows_with_transcript=rows_with_transcript, + rows_without_transcript=rows_without_transcript, + transcript_source_counts=source_counts, + evaluation_count=len(evaluations), + metrics=metrics_payload, + ) + + +from app.core.auth.capabilities import CALLS_DELETE, CALLS_IMPORT, CALLS_VIEW +from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities + +apply_workspace_route_capabilities( + router, + view_capability=CALLS_VIEW, + manage_capability=CALLS_IMPORT, + delete_capability=CALLS_DELETE, +) diff --git a/app/config.py b/app/config.py index 91ecdbb3..94b51f18 100644 --- a/app/config.py +++ b/app/config.py @@ -107,7 +107,7 @@ class Settings(BaseSettings): RATE_LIMIT_PER_MINUTE: int = 60 # Authentication - AUTH_PROVIDERS: List[str] = ["api_key"] + AUTH_PROVIDERS: Annotated[List[str], NoDecode] = ["api_key"] AUTH_LOCAL_ALLOW_SIGNUP: bool = True AUTH_LOCAL_TOKEN_TTL_MINUTES: int = 15 AUTH_REFRESH_TOKEN_TTL_DAYS: int = 7 diff --git a/app/migrations/059_call_import_audit_users.py b/app/migrations/059_call_import_audit_users.py new file mode 100644 index 00000000..41ec06db --- /dev/null +++ b/app/migrations/059_call_import_audit_users.py @@ -0,0 +1,69 @@ +""" +Migration: last_updated_by_user_id on call imports and evaluations. + +Supports surfacing who created / last modified a batch or evaluation run +via FK to users (email resolved at read time). +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Add last_updated_by_user_id to call_imports and call_import_evaluations" +) + + +def _column_exists(db: Session, table: str, column: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table, "column_name": column}, + ).first() + return row is not None + + +def _table_exists(db: Session, table: str) -> bool: + row = db.execute( + text( + "SELECT 1 FROM information_schema.tables WHERE table_name = :t" + ), + {"t": table}, + ).first() + return row is not None + + +def upgrade(db: Session): + for table in ("call_imports", "call_import_evaluations"): + if not _table_exists(db, table): + print(f"{table} does not exist, skipping...") + continue + if _column_exists(db, table, "last_updated_by_user_id"): + print(f"{table}.last_updated_by_user_id already exists, skipping...") + continue + db.execute( + text( + f""" + ALTER TABLE {table} + ADD COLUMN last_updated_by_user_id UUID NULL + REFERENCES users(id) ON DELETE SET NULL + """ + ) + ) + print(f"Added {table}.last_updated_by_user_id") + + +def downgrade(db: Session): + for table in ("call_import_evaluations", "call_imports"): + if not _table_exists(db, table): + continue + if not _column_exists(db, table, "last_updated_by_user_id"): + continue + db.execute( + text(f"ALTER TABLE {table} DROP COLUMN last_updated_by_user_id") + ) + print(f"Dropped {table}.last_updated_by_user_id") diff --git a/app/models/database.py b/app/models/database.py index a21a4819..3ce548de 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -1,2729 +1,2735 @@ -"""SQLAlchemy database models.""" - -from sqlalchemy import ( - BigInteger, - Boolean, - Column, - Date, - DateTime, - DDL, - Enum, - event, - Float, - ForeignKey, - Integer, - JSON, - String, - Text, - UniqueConstraint, - select, - text, -) -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import relationship -from sqlalchemy.sql import func -import uuid -import enum -from app.models.enums import ( - EvaluationType, EvaluationStatus, EvaluatorResultStatus, RoleEnum, InvitationStatus, - LanguageEnum, CallTypeEnum, CallMediumEnum, GenderEnum, AccentEnum, BackgroundNoiseEnum, - IntegrationPlatform, ModelProvider, VoiceBundleType, TestAgentConversationStatus, - MetricType, MetricCategory, MetricTrigger, CallRecordingStatus, AlertMetricType, AlertAggregation, - AlertOperator, AlertNotifyFrequency, AlertStatus, AlertHistoryStatus, CronJobStatus, - PromptOptimizationStatus, CallImportStatus, CallImportRowStatus, -) - -def get_enum_values(enum_class): - """Helper to get values from enum class for SQLAlchemy.""" - return [e.value for e in enum_class] - -from app.database import Base - - -# Enums moved to enums.py - - -class Organization(Base): - """Organization model for multi-tenancy.""" - - __tablename__ = "organizations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - name = Column(String(255), nullable=False) - voice_playground_threshold_overrides = Column(JSON, nullable=True) - # AlignEval-style judge alignment thresholds. - # Shape: {"min_labels_to_evaluate": int, "min_labels_to_optimize": int} - # Falls back to system defaults (20 / 50) when null. - judge_alignment_settings = Column(JSON, nullable=True) - # Per-org LLM gateway overrides (enabled, gateway_type, base_url, keys). - llm_gateway_settings = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - # Relationships - api_keys = relationship("APIKey", back_populates="organization") - members = relationship("OrganizationMember", back_populates="organization", cascade="all, delete-orphan") - invitations = relationship("Invitation", back_populates="organization", cascade="all, delete-orphan") - workspaces = relationship( - "Workspace", - back_populates="organization", - cascade="all, delete-orphan", - ) - workspace_roles = relationship( - "WorkspaceRole", - back_populates="organization", - cascade="all, delete-orphan", - ) - - -class Workspace(Base): - """Workspace - in-org isolation boundary for call imports and metrics. - - Every organization has at least one workspace (``is_default = True``, - seeded by migration 033). Users pick an "active workspace" in the UI; - list endpoints filter by it so users only see calls/metrics from the - project they're currently working in. Access is governed by - ``workspace_members`` and org-scoped ``workspace_roles`` (capability - bundles); org admins implicitly access all workspaces. - """ - - __tablename__ = "workspaces" - __table_args__ = ( - UniqueConstraint("organization_id", "slug", name="uq_workspaces_org_slug"), - ) - - # ``server_default`` is required so that raw-SQL INSERTs (e.g. the - # per-org Default seed in migration 033) can omit ``id`` and let the - # database fill it in. Without it, ``create_all`` produces a column - # with NOT NULL but no DEFAULT, and the migration crashes with - # ``null value in column "id"``. - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid4, - server_default=text("gen_random_uuid()"), - ) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - slug = Column(String(255), nullable=False) - # At most one default per org. Enforced on Postgres by the partial - # unique index attached via the after_create event below; on - # SQLite (test runs) we rely on the route-level _check_slug_unique - # check + the Default-workspace conftest fixture instead, because - # SQLite doesn't support partial indexes the same way. - is_default = Column(Boolean, nullable=False, default=False, server_default="false") - # Reusable PDF/report branding metadata scoped to this workspace. Images - # live in S3. Shape: {"heading": str|null, "images": [{id, s3_key, - # content_type, filename, size_bytes, updated_at}, ...]}. - report_branding = Column(JSON, nullable=True) - created_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True - ) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - organization = relationship("Organization", back_populates="workspaces") - members = relationship( - "WorkspaceMember", - back_populates="workspace", - cascade="all, delete-orphan", - ) - - -class WorkspaceRole(Base): - """Org-scoped workspace role (system or custom) as a capability bundle.""" - - __tablename__ = "workspace_roles" - __table_args__ = ( - UniqueConstraint("organization_id", "name", name="uq_workspace_roles_org_name"), - ) - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid4, - server_default=text("gen_random_uuid()"), - ) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - capabilities = Column(JSON, nullable=False, default=list) - is_system = Column(Boolean, nullable=False, default=False, server_default="false") - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - organization = relationship("Organization", back_populates="workspace_roles") - members = relationship("WorkspaceMember", back_populates="role") - - -class WorkspaceMember(Base): - """User membership in a workspace with an assigned workspace role.""" - - __tablename__ = "workspace_members" - __table_args__ = ( - UniqueConstraint("workspace_id", "user_id", name="uq_workspace_members_ws_user"), - ) - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid4, - server_default=text("gen_random_uuid()"), - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - user_id = Column( - UUID(as_uuid=True), - ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - role_id = Column( - UUID(as_uuid=True), - ForeignKey("workspace_roles.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - added_by_user_id = Column( - UUID(as_uuid=True), - ForeignKey("users.id", ondelete="SET NULL"), - nullable=True, - ) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - workspace = relationship("Workspace", back_populates="members") - user = relationship("User", foreign_keys=[user_id]) - role = relationship("WorkspaceRole", back_populates="members") - added_by = relationship("User", foreign_keys=[added_by_user_id]) - - -# Partial unique index: "at most one default workspace per org". This -# is attached as an after_create event (rather than declared in -# ``__table_args__``) because SQLAlchemy's ``Index(..., -# postgresql_where=...)`` silently degrades to a *full* unique index on -# SQLite - which then forbids any second workspace per org and breaks -# the test suite. ``execute_if(dialect="postgresql")`` makes this DDL -# a no-op on SQLite while still emitting it on Postgres (prod, CI). -event.listen( - Workspace.__table__, - "after_create", - DDL( - "CREATE UNIQUE INDEX IF NOT EXISTS uq_workspaces_org_default " - "ON workspaces (organization_id) WHERE is_default" - ).execute_if(dialect="postgresql"), -) - - -class User(Base): - """User model for authentication and profile management.""" - - __tablename__ = "users" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - email = Column(String(255), unique=True, nullable=False, index=True) - name = Column(String(255), nullable=True) - first_name = Column(String(255), nullable=True) - last_name = Column(String(255), nullable=True) - password_hash = Column(String(255), nullable=True) # Nullable for users created via invitation - external_id = Column(String(255), unique=True, nullable=True, index=True) - auth_provider = Column(String(50), nullable=True) - mfa_enabled = Column(Boolean, default=False, nullable=False) - last_login_at = Column(DateTime(timezone=True), nullable=True) - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - # Relationships - organization_memberships = relationship("OrganizationMember", back_populates="user", cascade="all, delete-orphan") - api_keys = relationship("APIKey", back_populates="user") - invitations = relationship("Invitation", back_populates="invited_user", foreign_keys="Invitation.invited_user_id") - refresh_tokens = relationship("RefreshToken", back_populates="user", cascade="all, delete-orphan") - - -class RefreshToken(Base): - """Opaque refresh token for extending local-password sessions.""" - - __tablename__ = "refresh_tokens" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True) - token_hash = Column(String(64), unique=True, nullable=False, index=True) - expires_at = Column(DateTime(timezone=True), nullable=False) - revoked_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - user = relationship("User", back_populates="refresh_tokens") - - -class OrganizationMember(Base): - """Organization membership with role.""" - - __tablename__ = "organization_members" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False, index=True) - role = Column(String, nullable=False, default=RoleEnum.READER.value) - - # User preferences for this organization - default_agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) - - joined_at = Column(DateTime(timezone=True), server_default=func.now()) - - # Unique constraint: one membership per user per organization - __table_args__ = ( - UniqueConstraint('organization_id', 'user_id', name='uq_org_user'), - ) - - # Relationships - organization = relationship("Organization", back_populates="members") - user = relationship("User", back_populates="organization_memberships") - default_agent = relationship("Agent", foreign_keys=[default_agent_id]) - - -class Invitation(Base): - """Invitation model for inviting users to organizations.""" - - __tablename__ = "invitations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - invited_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Null if user doesn't exist yet - invited_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False) - email = Column(String(255), nullable=False) # Email of invited user - role = Column(String, nullable=False, default=RoleEnum.READER.value) - status = Column(String, nullable=False, default=InvitationStatus.PENDING.value) - - - - token = Column(String(255), unique=True, nullable=False, index=True) # Invitation token - expires_at = Column(DateTime(timezone=True), nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - accepted_at = Column(DateTime(timezone=True), nullable=True) - - # Relationships - organization = relationship("Organization", back_populates="invitations") - invited_user = relationship("User", foreign_keys=[invited_user_id], back_populates="invitations") - invited_by = relationship("User", foreign_keys=[invited_by_id]) - - -class APIKey(Base): - """API Key model for authentication.""" - - __tablename__ = "api_keys" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - key = Column(String(255), unique=True, nullable=False, index=True) - name = Column(String(255), nullable=True) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Optional: link to user - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - last_used = Column(DateTime(timezone=True), nullable=True) - - # Relationships - organization = relationship("Organization", back_populates="api_keys") - user = relationship("User", back_populates="api_keys") - - -class AudioFile(Base): - """Audio file model.""" - - __tablename__ = "audio_files" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - filename = Column(String(255), nullable=False) - file_path = Column(String(512), nullable=False) - file_size = Column(Integer, nullable=False) # Size in bytes - duration = Column(Float, nullable=True) # Duration in seconds - sample_rate = Column(Integer, nullable=True) - channels = Column(Integer, nullable=True) - format = Column(String(10), nullable=False) # wav, mp3, flac, etc. - uploaded_at = Column(DateTime(timezone=True), server_default=func.now()) - - # Relationships - evaluations = relationship("Evaluation", back_populates="audio_file") - - -class Evaluation(Base): - """Evaluation job model.""" - - __tablename__ = "evaluations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every legacy audio evaluation belongs to a - # workspace within its org. Stamped from the X-Workspace-Id header - # (falling back to the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - audio_id = Column(UUID(as_uuid=True), ForeignKey("audio_files.id"), nullable=False) - reference_text = Column(String, nullable=True) # For WER calculation - evaluation_type = Column(String, nullable=False) - model_name = Column(String(100), nullable=True) - status = Column(String, default=EvaluationStatus.PENDING.value, nullable=False) - - - - metrics_requested = Column(JSON, nullable=True) # List of requested metrics - created_at = Column(DateTime(timezone=True), server_default=func.now()) - started_at = Column(DateTime(timezone=True), nullable=True) - completed_at = Column(DateTime(timezone=True), nullable=True) - error_message = Column(String, nullable=True) - - # Relationships - audio_file = relationship("AudioFile", back_populates="evaluations") - result = relationship("EvaluationResult", back_populates="evaluation", uselist=False) - - -class EvaluationResult(Base): - """Evaluation result model.""" - - __tablename__ = "evaluation_results" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column(UUID(as_uuid=True), ForeignKey("evaluations.id"), nullable=False, unique=True) - # Workspace isolation: mirrors the parent Evaluation's workspace. - # Denormalized for fast filter-by-workspace listings without a join. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - transcript = Column(String, nullable=True) - metrics = Column(JSON, nullable=False) # {"wer": 0.05, "latency_ms": 1250, ...} - raw_output = Column(JSON, nullable=True) # Full model output - processing_time = Column(Float, nullable=True) # Processing time in seconds - model_used = Column(String(100), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - # Relationships - evaluation = relationship("Evaluation", back_populates="result") - - -# ============================================ -# VAIOPS MODELS - Voice AI Ops -# ============================================ - -# Enums moved to enums.py - - -class Agent(Base): - """Test Agent - The voice AI agent being evaluated""" - __tablename__ = "agents" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - agent_id = Column(String(6), unique=True, nullable=True, index=True) # 6-digit ID - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every agent belongs to a workspace within its - # org. Stamped from the X-Workspace-Id header (falling back to the - # org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - name = Column(String, nullable=False) - phone_number = Column(String, nullable=True) # Optional, required only for phone_call - language = Column(String, nullable=False, default=LanguageEnum.ENGLISH.value) - description = Column(String) - provider_prompt = Column(Text, nullable=True) - provider_prompt_synced_at = Column(DateTime(timezone=True), nullable=True) - call_type = Column(String, nullable=False, default=CallTypeEnum.OUTBOUND.value) - call_medium = Column(String, nullable=False, default=CallMediumEnum.PHONE_CALL.value) - telephony_phone_number_id = Column( - UUID(as_uuid=True), - ForeignKey("telephony_phone_numbers.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - - - - - # Voice configuration - either voice_bundle_id OR ai_provider_id OR voice_ai_integration_id (mutually exclusive) - voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True, index=True) - ai_provider_id = Column(UUID(as_uuid=True), ForeignKey("aiproviders.id"), nullable=True, index=True) - - # Voice AI agent integration (Retell, Vapi, etc.) - voice_ai_integration_id = Column(UUID(as_uuid=True), ForeignKey("integrations.id"), nullable=True, index=True) - voice_ai_agent_id = Column(String, nullable=True) # Agent ID from the external provider (Retell/Vapi) - prompt_variables = Column(JSON, nullable=True) - silence_hangup_secs = Column(Integer, nullable=False, server_default="15") - - created_at = Column(DateTime, server_default=func.now()) - updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) - created_by = Column(String) - - -class Persona(Base): - """Persona - TTS provider-tied voice identity for testing""" - __tablename__ = "personas" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every persona belongs to a workspace within - # its org. Stamped from the X-Workspace-Id header (falling back to - # the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - name = Column(String, nullable=False) - gender = Column(String, nullable=False, default=GenderEnum.NEUTRAL.value) - tts_provider = Column(String(100), nullable=True) - tts_voice_id = Column(String(255), nullable=True) - tts_voice_name = Column(String(255), nullable=True) - is_custom = Column(Boolean, default=False) - description = Column(Text, nullable=True) - tts_config = Column(JSON, nullable=True) - llm_temperature = Column(Float, nullable=True) - llm_max_tokens = Column(Integer, nullable=True) - response_delay_ms = Column(Integer, nullable=True) - max_turns = Column(Integer, nullable=True) - allow_interruptions = Column(Boolean, nullable=True) - - created_at = Column(DateTime, server_default=func.now()) - updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) - created_by = Column(String) - - -class Scenario(Base): - """Scenario - The conversation scenario/test case""" - __tablename__ = "scenarios" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every scenario belongs to a workspace within - # its org. Stamped from the X-Workspace-Id header (falling back to - # the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) - name = Column(String, nullable=False) - description = Column(String) - required_info = Column(JSON) - - created_at = Column(DateTime, server_default=func.now()) - updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) - created_by = Column(String) - - -# Enums moved to enums.py - - -class Integration(Base): - """Integration model for connecting with external voice AI platforms.""" - __tablename__ = "integrations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - platform = Column(String, nullable=False) - - - - name = Column(String, nullable=True) # Optional friendly name - api_key = Column(String, nullable=False) # Encrypted Private API key for the platform - public_key = Column(String, nullable=True) # Optional Public API key (e.g. for Vapi) - is_active = Column(Boolean, default=True, nullable=False) - # Multiple credentials per (org, platform) are allowed. is_default marks - # the row used when a caller does not explicitly select a credential. - # A partial unique index in migration 028 enforces at most one default - # per (org, platform) at the DB level. - is_default = Column(Boolean, default=False, nullable=False) - # inherit | gateway | direct — per-credential LLM routing override - routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated - - -class ManualTranscription(Base): - """Manual transcription model for storing transcriptions from S3 audio files.""" - - __tablename__ = "manual_transcriptions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - name = Column(String(255), nullable=True) # User-friendly name for the transcription - audio_file_key = Column(String(512), nullable=False) # S3 key or file path - transcript = Column(String, nullable=False) # Full transcript text - speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] - stt_model = Column(String(100), nullable=True) # STT model used (e.g., "whisper-1", "google-speech-v2") - stt_provider = Column(String, nullable=True) # Provider used - - - - language = Column(String(10), nullable=True) # Detected or specified language - processing_time = Column(Float, nullable=True) # Processing time in seconds - raw_output = Column(JSON, nullable=True) # Full model output for reference - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class ConversationEvaluation(Base): - """Conversation evaluation model for evaluating manual transcriptions against agent objectives.""" - - __tablename__ = "conversation_evaluations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - transcription_id = Column(UUID(as_uuid=True), ForeignKey("manual_transcriptions.id"), nullable=False, index=True) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) - - # Evaluation results - objective_achieved = Column(Boolean, nullable=False) # Binary: was the conversation objective achieved? - objective_achieved_reason = Column(String, nullable=True) # Explanation for the binary result - additional_metrics = Column(JSON, nullable=True) # Additional evaluation metrics (e.g., professionalism, clarity, etc.) - overall_score = Column(Float, nullable=True) # Overall score (0.0 to 1.0) - - # LLM metadata - llm_provider = Column(Enum(ModelProvider, native_enum=False), nullable=True) - - llm_model = Column(String(100), nullable=True) - llm_response = Column(JSON, nullable=True) # Full LLM response for reference - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class AIProvider(Base): - """AI Provider - Stores API keys for different AI platforms.""" - __tablename__ = "aiproviders" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider = Column(String, nullable=False) - - - - api_key = Column(String, nullable=False) # Encrypted API key - name = Column(String, nullable=True) # Optional friendly name - # Azure OpenAI resource endpoint (e.g. https://my-resource.openai.azure.com). - # Only used when provider is azure; other providers ignore this column. - endpoint_url = Column(String, nullable=True) - is_active = Column(Boolean, default=True, nullable=False) - # Multiple AIProvider rows per (org, provider) are allowed. is_default - # marks the row resolved when no explicit credential id is selected. - # A partial unique index in migration 028 enforces at most one default. - is_default = Column(Boolean, default=False, nullable=False) - # inherit | gateway | direct — per-credential LLM routing override - routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") - # Bifrost custom model ID used when routing via gateway - gateway_model = Column(String(255), nullable=True) - # inherit | litellm_shim | native_openai — Bifrost API surface override - gateway_interface = Column(String(20), nullable=False, default="inherit", server_default="inherit") - # Optional per-credential Bifrost/gateway base URL override - gateway_base_url = Column(String(512), nullable=True) - # Optional auth header for Bifrost (e.g. x-bf-vk, Authorization, x-api-key) - gateway_auth_header = Column(String(64), nullable=True) - # Env var name whose value is sent as the gateway auth secret - gateway_auth_secret_env = Column(String(128), nullable=True) - # Encrypted inline gateway auth secret (alternative to env var) - gateway_auth_secret = Column(String, nullable=True) - # Arbitrary HTTP headers sent with gateway-routed LiteLLM calls - gateway_extra_headers = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated - - -# Enums moved to enums.py - - -class VoiceBundle(Base): - """VoiceBundle - Composable unit combining STT, LLM, and TTS for voice AI testing, or S2S models.""" - __tablename__ = "voicebundles" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - name = Column(String, nullable=False) - description = Column(String, nullable=True) - - # Bundle type: either STT+LLM+TTS or S2S - # Using String instead of Enum to avoid SQLAlchemy enum conversion issues - # The enum conversion is handled in the Pydantic schemas - bundle_type = Column(String(50), nullable=False, default=VoiceBundleType.STT_LLM_TTS.value) - - # STT Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S - stt_provider = Column(String, nullable=True) - # Optional explicit credential row (aiproviders.id or integrations.id). - # When NULL the credential resolver picks the default row for the - # provider. No FK is set because the target table varies by provider. - stt_credential_id = Column(UUID(as_uuid=True), nullable=True) - - stt_model = Column(String, nullable=True) # e.g., "whisper-1", "google-speech-v2" - - # LLM Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S - llm_provider = Column(String, nullable=True) - llm_credential_id = Column(UUID(as_uuid=True), nullable=True) - - llm_model = Column(String, nullable=True) # e.g., "gpt-4", "claude-3-opus" - llm_temperature = Column(Float, nullable=True, default=0.7) - llm_max_tokens = Column(Integer, nullable=True) - llm_config = Column(JSON, nullable=True) # Additional LLM configuration (extensible) - - # TTS Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S - tts_provider = Column(String, nullable=True) - tts_credential_id = Column(UUID(as_uuid=True), nullable=True) - - tts_model = Column(String, nullable=True) # e.g., "tts-1", "neural-voice" - tts_voice = Column(String, nullable=True) # Voice selection if applicable - tts_config = Column(JSON, nullable=True) # Additional TTS configuration (extensible) - - # S2S Configuration - required for S2S type, optional for STT_LLM_TTS - s2s_provider = Column(String, nullable=True) - s2s_credential_id = Column(UUID(as_uuid=True), nullable=True) - - - - s2s_model = Column(String, nullable=True) # e.g., "gpt-4o-transcribe", speech-to-speech model - s2s_config = Column(JSON, nullable=True) # Additional S2S configuration (extensible) - - # Additional configuration for extensibility - extra_metadata = Column(JSON, nullable=True) # For future extensions (renamed from 'metadata' to avoid SQLAlchemy conflict) - - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -# Enums moved to enums.py - - -class TestAgentConversation(Base): - """Test Agent Conversation - Records conversations between test AI agent and voice AI agent.""" - __tablename__ = "test_agent_conversations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every playground conversation belongs to a - # workspace within its org. Stamped from the X-Workspace-Id header - # (falling back to the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Configuration - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) - scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=False) - voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) - - # Conversation data - status = Column(String, nullable=False, default=TestAgentConversationStatus.INITIALIZING.value) - - - - live_transcription = Column(JSON, nullable=True) # Array of conversation turns with timestamps - conversation_audio_key = Column(String, nullable=True) # S3 key for recorded conversation audio - full_transcript = Column(String, nullable=True) # Full conversation transcript - - # Metadata - started_at = Column(DateTime(timezone=True), server_default=func.now()) - ended_at = Column(DateTime(timezone=True), nullable=True) - duration_seconds = Column(Float, nullable=True) - - # Additional metadata - conversation_metadata = Column(JSON, nullable=True) # Additional conversation metadata - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class EvaluatorSuite(Base): - """Evaluator suite — one agent + one persona + N scenario combinations.""" - - __tablename__ = "evaluator_suites" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - name = Column(String, nullable=True) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) - metric_ids = Column(JSON, nullable=True) - llm_provider = Column(String, nullable=True) - llm_model = Column(String, nullable=True) - llm_config = Column(JSON, nullable=True) - tags = Column(JSON, nullable=True) - default_runs_per_combination = Column(Integer, nullable=False, default=1) - round_robin_index = Column(Integer, nullable=False, default=0) - is_active = Column(Boolean, nullable=False, default=False) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class Evaluator(Base): - """Evaluator - Configuration for testing agents with specific persona and scenario combinations, or custom prompt evaluators.""" - __tablename__ = "evaluators" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluator_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every evaluator belongs to a workspace within - # its org. Stamped from the X-Workspace-Id header (falling back to - # the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Display name (required for custom evaluators, optional for standard) - name = Column(String, nullable=True) - - # Parent suite (nullable for legacy/custom evaluators) - suite_id = Column( - UUID(as_uuid=True), - ForeignKey("evaluator_suites.id", ondelete="CASCADE"), - nullable=True, - index=True, - ) - - # Standard evaluator configuration (nullable for custom evaluators) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) - scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) - - # Custom evaluator prompt (used instead of agent/persona/scenario) - custom_prompt = Column(Text, nullable=True) - - # Custom evaluator metric selection. When set, the worker filters the - # enabled-org metrics down to only these IDs (list of metric UUID strings). - # Standard evaluators leave this NULL and use all enabled agent metrics. - metric_ids = Column(JSON, nullable=True) - - # LLM configuration for evaluation (overrides hardcoded defaults) - llm_provider = Column(String, nullable=True) # e.g. "openai", "anthropic", "google" - llm_model = Column(String, nullable=True) # e.g. "gpt-4.1", "claude-sonnet-4-20250514" - llm_config = Column(JSON, nullable=True) - - # Tags for categorization - tags = Column(JSON, nullable=True) # Array of tag strings - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -# Enums moved to enums.py - - -class Metric(Base): - """Metric - Configuration for evaluation metrics. - - Supports a 2-level hierarchy via ``parent_metric_id``: a "category" - parent metric (e.g. "Call Outcome") owns N child sub-metric labels - (e.g. "happy_completion", "angry_hangup"). ``selection_mode`` is set - only on parents and controls how the LLM scores children together - (``single_choice`` = pick exactly one; ``multi_label`` = independent - yes/no with logical consistency). Children are always boolean. - """ - __tablename__ = "metrics" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: two-shape column. - # - # * ``workspace_id = `` — workspace-scoped metric. Only - # visible inside that workspace (the default behavior; existing - # rows all look like this). - # * ``workspace_id IS NULL`` — org-shared metric. Surfaces in - # every workspace's listing under this org so users don't have - # to recreate the same metric per workspace. - # - # Children always inherit their parent's ``workspace_id`` (including - # NULL) so a category metric's whole subtree shares one scope; the - # add-child / promote-discovered endpoints enforce this. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=True, - index=True, - ) - - # Basic information - name = Column(String, nullable=False) - description = Column(String, nullable=True) - # Free-form illustrative example used to sharpen the LLM judge's - # rubric. Today this is consumed by child sub-labels of a - # categorization parent metric so each label can carry "what does - # this look like in a transcript?" text alongside the rubric in - # ``description``. The column lives on every Metric row for - # forward-compat: a standalone metric could later surface its own - # example without another migration. - example = Column(Text, nullable=True) - - # Configuration - metric_type = Column(String, nullable=False, default=MetricType.RATING.value) - metric_category = Column( - String(30), - nullable=False, - default=MetricCategory.QUALITY.value, - server_default=MetricCategory.QUALITY.value, - ) - trigger = Column(String, nullable=False, default=MetricTrigger.ALWAYS.value) - metric_origin = Column(String(30), nullable=False, default="default") - supported_surfaces = Column(JSON, nullable=False, default=list) # ["agent", "voice_playground", "blind_test"] - enabled_surfaces = Column(JSON, nullable=False, default=list) # subset of supported_surfaces - custom_data_type = Column(String(30), nullable=True) # "boolean" | "enum" | "number_range" - custom_config = Column(JSON, nullable=True) # enum options / number range config - tags = Column(JSON, nullable=True) # ["tone", "latency", ...] - - # Hierarchy: NULL = standalone or parent. When set, this row is a - # child sub-metric of the referenced parent. ON DELETE CASCADE so - # deleting a category removes its children atomically. - parent_metric_id = Column( - UUID(as_uuid=True), - ForeignKey("metrics.id", ondelete="CASCADE"), - nullable=True, - index=True, - ) - # Set only on parent rows (``parent_metric_id IS NULL``). Either - # ``single_choice`` or ``multi_label``. NULL = legacy / non-hierarchical - # metric (no children). - selection_mode = Column(String(20), nullable=True) - - # When true on a parent metric (any selection_mode), the LLM is - # invited during call-import evaluation to emit additional - # candidate sub-labels beyond the user-defined children. The - # candidates surface in a "Discovered labels" panel where the user - # manually promotes them into real child Metric rows. For - # ``single_choice`` parents the discovered entries are - # supplemental — the chosen child is still picked from the - # predefined children so the exactly-one-true invariant holds. - # The validator rejects this flag on standalone / child metrics. - allow_discovery = Column( - Boolean, nullable=False, default=False, server_default="false" - ) - - # When True, this metric is a "transcript-compare judge": the - # call-import evaluator feeds BOTH the production transcript - # (``call_import_rows.transcript``, CSV-supplied) and the diarised - # transcript (``call_import_rows.diarised_transcript``, worker- - # produced by the STT/diarisation pipeline) to the LLM as a - # labeled pair instead of feeding one transcript. The parent - # evaluation's ``CallImportEvaluation.transcript_source`` is - # ignored for these metrics — they always read both columns. - # Rows where either transcript is missing are skipped per-metric - # with ``skipped="comparison_missing_transcript"`` so the rest of - # the row's metrics still produce scores. The Pydantic validator - # rejects ``compare_transcripts`` combined with ``parent_metric_id`` - # or ``selection_mode`` (i.e. it can't simultaneously be part of - # a parent/child hierarchy). The call-import worker also - # auto-promotes a metric to comparison mode when its description - # references the production / diarised transcripts in well-known - # phrases (see ``_metric_text_references_production`` in - # ``app.workers.tasks.evaluate_call_import_row``). - compare_transcripts = Column( - Boolean, nullable=False, default=False, server_default="false" - ) - - parent = relationship( - "Metric", - remote_side=[id], - backref="children", - ) - - # When true, the LLM-judge is asked to also return a short free-form - # rationale alongside the value (stored under ``metric_scores[id].rationale``). - # Adds a second " - LLM Rationale" column in the call-import CSV export. - capture_rationale = Column(Boolean, nullable=False, default=False) - - enabled = Column(Boolean, nullable=False, default=True) - - # Metadata - is_default = Column(Boolean, nullable=False, default=False) # Pre-defined metrics - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class EvaluatorResult(Base): - """EvaluatorResult - Results from running an evaluator with transcription and metric evaluations.""" - __tablename__ = "evaluator_results" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - result_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every evaluator result belongs to a workspace - # within its org. Stamped from the active workspace at creation time - # (either the X-Workspace-Id header or the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # References - evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True, index=True) # Optional - can be None for test calls without persona/scenario - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Nullable for custom evaluators - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) # Optional - can be None for test calls - scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) # Optional - can be None for test calls - - # Result data - name = Column(String, nullable=True) # Scenario name or test call name (optional) - timestamp = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) - duration_seconds = Column(Float, nullable=True) # Call duration - status = Column(String(20), nullable=False, default=EvaluatorResultStatus.QUEUED.value) - - # Audio and transcription - audio_s3_key = Column(String, nullable=True) # S3 key for audio file - transcription = Column(String, nullable=True) # Full transcription - speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] - - # Metric scores - JSON object with metric_id as key and score as value - # Format: {"metric_id_1": {"value": 85, "type": "rating"}, "metric_id_2": {"value": true, "type": "boolean"}} - metric_scores = Column(JSON, nullable=True) - - # Celery task tracking - celery_task_id = Column(String, nullable=True, index=True) # Celery task ID for tracking - - # Error information - error_message = Column(String, nullable=True) - - # Call event tracking (similar to CallRecording) - call_event = Column(String, nullable=True, index=True) # Latest call event (e.g., call_started, call_ended) - provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) - provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" - call_data = Column(JSON, nullable=True) # Full call details from provider (like CallRecording) - - # Data-plane shard routing (payload rows on shard DBs when sharding enabled) - shard_id = Column(String(64), nullable=True, index=True) - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -# Enums moved to enums.py - - -class CallRecordingSource(str, enum.Enum): - """Source of the call recording data.""" - - PLAYGROUND = "playground" - WEBHOOK = "webhook" - - -class CallRecording(Base): - """Call Recording model for tracking voice provider calls.""" - __tablename__ = "call_recordings" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every recording belongs to a workspace within - # its org. For playground-origin rows this is stamped from the active - # workspace at creation time; for webhook-origin rows the worker - # looks up the recording's agent and inherits its workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - call_short_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID - status = Column(Enum(CallRecordingStatus), nullable=False, default=CallRecordingStatus.PENDING, index=True) - call_event = Column(String, nullable=True, index=True) # Latest webhook event (e.g., call_started, call_ended) - source = Column(Enum(CallRecordingSource), nullable=False, default=CallRecordingSource.PLAYGROUND, index=True) - call_data = Column(JSON, nullable=True) # JSON blob for provider response - provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) - provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Reference to our agent - - # Link to EvaluatorResult for metric evaluations - evaluator_result_id = Column( - UUID(as_uuid=True), - ForeignKey("evaluator_results.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - - shard_id = Column(String(64), nullable=True, index=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class EvaluatorResultPayload(Base): - """Heavy evaluator result fields stored on data shards when sharding is enabled.""" - - __tablename__ = "evaluator_result_payloads" - - evaluator_result_id = Column(UUID(as_uuid=True), primary_key=True) - workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) - audio_s3_key = Column(String, nullable=True) - transcription = Column(String, nullable=True) - speaker_segments = Column(JSON, nullable=True) - metric_scores = Column(JSON, nullable=True) - call_data = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class CallRecordingPayload(Base): - """Heavy call recording fields stored on data shards when sharding is enabled.""" - - __tablename__ = "call_recording_payloads" - - call_recording_id = Column(UUID(as_uuid=True), primary_key=True) - workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) - call_data = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class Alert(Base): - """Alert model for configuring monitoring alerts.""" - __tablename__ = "alerts" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - - # Basic information - name = Column(String(255), nullable=False) - description = Column(String, nullable=True) - - # Metric condition configuration - metric_type = Column(String, nullable=False, default=AlertMetricType.NUMBER_OF_CALLS.value) - aggregation = Column(String, nullable=False, default=AlertAggregation.SUM.value) - operator = Column(String, nullable=False, default=AlertOperator.GREATER_THAN.value) - threshold_value = Column(Float, nullable=False) - time_window_minutes = Column(Integer, nullable=False, default=60) # Time window for aggregation - - # Agent selection (JSON array of agent UUIDs, null means all agents) - agent_ids = Column(JSON, nullable=True) - - # Notification configuration - notify_frequency = Column(String, nullable=False, default=AlertNotifyFrequency.IMMEDIATE.value) - notify_emails = Column(JSON, nullable=True) # Array of email addresses - notify_webhooks = Column(JSON, nullable=True) # Array of webhook URLs (Slack, etc.) - - # Status - status = Column(String, nullable=False, default=AlertStatus.ACTIVE.value) - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - # Relationships - alert_history = relationship("AlertHistory", back_populates="alert", cascade="all, delete-orphan") - - -class AlertHistory(Base): - """Alert history model for tracking triggered alerts.""" - __tablename__ = "alert_history" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - alert_id = Column(UUID(as_uuid=True), ForeignKey("alerts.id"), nullable=False, index=True) - - # Trigger information - triggered_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) - triggered_value = Column(Float, nullable=False) # The actual value that triggered the alert - threshold_value = Column(Float, nullable=False) # The threshold at time of trigger - - # Status tracking - status = Column(String, nullable=False, default=AlertHistoryStatus.TRIGGERED.value) - - # Notification tracking - notified_at = Column(DateTime(timezone=True), nullable=True) - notification_details = Column(JSON, nullable=True) # Details of sent notifications - - # Resolution - acknowledged_at = Column(DateTime(timezone=True), nullable=True) - acknowledged_by = Column(String, nullable=True) - resolved_at = Column(DateTime(timezone=True), nullable=True) - resolved_by = Column(String, nullable=True) - resolution_notes = Column(String, nullable=True) - - # Additional context - context_data = Column(JSON, nullable=True) # Additional data about the trigger - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - # Relationships - alert = relationship("Alert", back_populates="alert_history") - - -class CronJob(Base): - """Cron job model for scheduling automated evaluator runs.""" - __tablename__ = "cron_jobs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - - # Basic information - name = Column(String(255), nullable=False) - cron_expression = Column(String(100), nullable=False) # e.g., "0 9 * * 1-5" - timezone = Column(String(100), nullable=False, default="UTC") - - # Run configuration - max_runs = Column(Integer, nullable=False, default=10) - current_runs = Column(Integer, nullable=False, default=0) - - # Evaluators to trigger (JSON array of evaluator UUIDs) - evaluator_ids = Column(JSON, nullable=False) - - # Status - status = Column(String, nullable=False, default=CronJobStatus.ACTIVE.value) - - # Run tracking - next_run_at = Column(DateTime(timezone=True), nullable=True) - last_run_at = Column(DateTime(timezone=True), nullable=True) - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class TTSComparisonStatus(str, enum.Enum): - PENDING = "pending" - GENERATING = "generating" - EVALUATING = "evaluating" - COMPLETED = "completed" - FAILED = "failed" - - -class TTSSampleStatus(str, enum.Enum): - PENDING = "pending" - GENERATING = "generating" - COMPLETED = "completed" - FAILED = "failed" - - -class TTSReportJobStatus(str, enum.Enum): - PENDING = "pending" - PROCESSING = "processing" - COMPLETED = "completed" - FAILED = "failed" - - -class TTSComparison(Base): - """TTS Comparison session for A/B testing voice providers.""" - __tablename__ = "tts_comparisons" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every voice playground comparison belongs to - # a workspace within its org. Children (samples, report jobs, blind - # test shares) inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - simulation_id = Column(String(6), unique=True, index=True, nullable=True) - - name = Column(String(255), nullable=True) - status = Column(String(50), nullable=False, default=TTSComparisonStatus.PENDING.value) - - # 'benchmark' = traditional TTS A/B benchmark (provider-generated audio). - # 'blind_test_only' = standalone blind test built from existing recordings - # / uploads / past TTS samples; no TTS generation happens. - mode = Column(String(32), nullable=False, default="benchmark") - - provider_a = Column(String(100), nullable=True) - model_a = Column(String(100), nullable=True) - voices_a = Column(JSON, nullable=True) - - provider_b = Column(String(100), nullable=True) - model_b = Column(String(100), nullable=True) - voices_b = Column(JSON, nullable=True) - - sample_texts = Column(JSON, nullable=False) - num_runs = Column(Integer, nullable=False, default=1) - - blind_test_results = Column(JSON, nullable=True) - evaluation_summary = Column(JSON, nullable=True) - - eval_stt_provider = Column(String(100), nullable=True) - eval_stt_model = Column(String(100), nullable=True) - - celery_task_id = Column(String, nullable=True, index=True) - error_message = Column(String, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - samples = relationship("TTSSample", back_populates="comparison", cascade="all, delete-orphan") - - -class TTSSample(Base): - """Individual TTS audio sample within a comparison.""" - __tablename__ = "tts_samples" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: mirrors the parent TTSComparison's workspace. - # Denormalized for fast filter-by-workspace listings without a join. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - provider = Column(String(100), nullable=True) - model = Column(String(100), nullable=True) - voice_id = Column(String(255), nullable=True) - voice_name = Column(String(255), nullable=True) - side = Column(String(1), nullable=True) # "A" or "B" - sample_index = Column(Integer, nullable=False) - run_index = Column(Integer, nullable=False, default=0) - - # 'tts' (default, audio is synthesized by a provider), 'recording' (audio - # is reused from a CallImportRow recording), or 'upload' (audio was - # uploaded by the user). Non-tts samples are marked completed up-front - # by the API and skipped by the generation worker. - source_type = Column(String(32), nullable=False, default="tts") - # When source_type == 'recording', references CallImportRow.id (no FK - # constraint to keep cascading deletes simple if a call import is later - # removed; the audio_s3_key is what's actually used). - source_ref_id = Column(UUID(as_uuid=True), nullable=True) - - text = Column(String, nullable=False) - audio_s3_key = Column(String(512), nullable=True) - duration_seconds = Column(Float, nullable=True) - latency_ms = Column(Float, nullable=True) - ttfb_ms = Column(Float, nullable=True) - - evaluation_metrics = Column(JSON, nullable=True) - status = Column(String(50), nullable=False, default=TTSSampleStatus.PENDING.value) - error_message = Column(String, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - comparison = relationship("TTSComparison", back_populates="samples") - - -class TTSReportJob(Base): - """Asynchronous PDF report generation jobs for Voice Playground.""" - __tablename__ = "tts_report_jobs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: mirrors the parent TTSComparison's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) - - status = Column(String(50), nullable=False, default=TTSReportJobStatus.PENDING.value) - format = Column(String(20), nullable=False, default="pdf") - filename = Column(String(255), nullable=True) - s3_key = Column(String(512), nullable=True) - error_message = Column(String, nullable=True) - celery_task_id = Column(String, nullable=True, index=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - comparison = relationship("TTSComparison") - - -class TTSBlindTestShareStatus(str, enum.Enum): - OPEN = "open" - CLOSED = "closed" - - -class TTSBlindTestShare(Base): - """A publicly sharable blind test for a TTSComparison. - - The share_token is the capability: anyone holding it can open the public - form and submit a response. Each comparison has at most one share row. - """ - __tablename__ = "tts_blind_test_shares" - __table_args__ = ( - UniqueConstraint("comparison_id", name="uq_blind_test_shares_comparison"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - comparison_id = Column( - UUID(as_uuid=True), - ForeignKey("tts_comparisons.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: mirrors the parent TTSComparison's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - share_token = Column(String(64), unique=True, nullable=False, index=True) - - title = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - - # Internal notes visible only to the share creator (e.g. which voice - # corresponds to which side, source notes for standalone blind tests). - # Never exposed via the public blind test payload. - creator_notes = Column(Text, nullable=True) - - # JSON list: [{ "key": str, "label": str, "type": "rating"|"comment", "scale": int? }] - custom_metrics = Column(JSON, nullable=False) - - status = Column(String(20), nullable=False, default=TTSBlindTestShareStatus.OPEN.value) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - closed_at = Column(DateTime(timezone=True), nullable=True) - created_by = Column(String, nullable=True) - - comparison = relationship("TTSComparison") - responses = relationship( - "TTSBlindTestResponse", - back_populates="share", - cascade="all, delete-orphan", - ) - - -class TTSBlindTestResponse(Base): - """A single rater's submission against a TTSBlindTestShare.""" - __tablename__ = "tts_blind_test_responses" - __table_args__ = ( - UniqueConstraint("share_id", "rater_email", name="uq_blind_test_response_share_email"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - share_id = Column( - UUID(as_uuid=True), - ForeignKey("tts_blind_test_shares.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - # Workspace isolation: mirrors the parent TTSBlindTestShare's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - rater_name = Column(String(255), nullable=False) - rater_email = Column(String(320), nullable=False, index=True) - - # JSON list keyed by sample_index. Server stores in TRUE A/B orientation - # (already de-flipped from whatever the rater's UI showed): - # [{ - # "sample_index": int, - # "preferred": "A" | "B", - # "ratings_a": { metric_key: number }, - # "ratings_b": { metric_key: number }, - # "comment": str? - # }] - responses = Column(JSON, nullable=False) - - ip = Column(String(64), nullable=True) - user_agent = Column(String(512), nullable=True) - - submitted_at = Column(DateTime(timezone=True), server_default=func.now()) - - share = relationship("TTSBlindTestShare", back_populates="responses") - - -class PromptPartial(Base): - """Prompt Partial - Reusable prompt templates with version history.""" - __tablename__ = "prompt_partials" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every prompt partial belongs to a workspace - # within its org. Versions inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - description = Column(String, nullable=True) - content = Column(Text, nullable=False) - tags = Column(JSON, nullable=True) - current_version = Column(Integer, nullable=False, default=1) - # Cached LLM-generated flowchart for imported production agent prompts. - # Shape: AgentFlowGraph JSON (nodes[], edges[]). - agent_flowchart = Column(JSON, nullable=True) - agent_flowchart_status = Column(String(20), nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - versions = relationship("PromptPartialVersion", back_populates="prompt_partial", cascade="all, delete-orphan", order_by="PromptPartialVersion.version.desc()") - - -class PromptPartialVersion(Base): - """Version history for a prompt partial.""" - __tablename__ = "prompt_partial_versions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - prompt_partial_id = Column(UUID(as_uuid=True), ForeignKey("prompt_partials.id", ondelete="CASCADE"), nullable=False, index=True) - # Workspace isolation: mirrors the parent PromptPartial's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - version = Column(Integer, nullable=False) - content = Column(Text, nullable=False) - change_summary = Column(String, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - created_by = Column(String, nullable=True) - - prompt_partial = relationship("PromptPartial", back_populates="versions") - - __table_args__ = ( - UniqueConstraint('prompt_partial_id', 'version', name='uq_prompt_partial_version'), - ) - - -class CustomTTSVoice(Base): - """Organization-scoped custom TTS voice metadata.""" - __tablename__ = "custom_tts_voices" - __table_args__ = ( - UniqueConstraint("organization_id", "provider", "voice_id", name="uq_custom_tts_voice_org_provider_voice_id"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider = Column(String(100), nullable=False, index=True) - voice_id = Column(String(255), nullable=False) - name = Column(String(255), nullable=False) - gender = Column(String(50), nullable=True) - accent = Column(String(100), nullable=True) - description = Column(Text, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - -class PromptOptimizationRun(Base): - """A single GEPA prompt optimization run for an agent.""" - __tablename__ = "prompt_optimization_runs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every optimization run belongs to a workspace - # within its org. Candidates inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) - evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True) - voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) - - seed_prompt = Column(Text, nullable=False) - best_prompt = Column(Text, nullable=True) - best_score = Column(Float, nullable=True) - - status = Column(String(20), nullable=False, default=PromptOptimizationStatus.PENDING.value) - config = Column(JSON, nullable=True) - reflection_trace = Column(JSON, nullable=True) - metric_history = Column(JSON, nullable=True) - - num_iterations = Column(Integer, nullable=True) - num_metric_calls = Column(Integer, nullable=True) - - celery_task_id = Column(String, nullable=True, index=True) - error_message = Column(Text, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - candidates = relationship("PromptOptimizationCandidate", back_populates="optimization_run", cascade="all, delete-orphan") - - -class PromptOptimizationCandidate(Base): - """A candidate prompt generated during an optimization run.""" - __tablename__ = "prompt_optimization_candidates" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - optimization_run_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_runs.id", ondelete="CASCADE"), nullable=False, index=True) - # Workspace isolation: mirrors the parent PromptOptimizationRun's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - prompt_text = Column(Text, nullable=False) - score = Column(Float, nullable=True) - metric_breakdown = Column(JSON, nullable=True) - reflection_summary = Column(Text, nullable=True) - - parent_candidate_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_candidates.id"), nullable=True) - - is_accepted = Column(Boolean, nullable=False, default=False) - pushed_to_provider_at = Column(DateTime(timezone=True), nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - optimization_run = relationship("PromptOptimizationRun", back_populates="candidates") - - -class TelephonyIntegration(Base): - """Per-organization telephony provider credentials and configuration. - - Multiple rows per (organization_id, provider) are allowed so that an - organization can keep several Plivo / Exotel accounts side-by-side. - A partial unique index in migration 028 enforces at most one row with - is_default = TRUE per (org, provider); resolution falls back to that - default row when the caller does not pin a specific credential. - """ - - __tablename__ = "telephony_integrations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider = Column(String(50), nullable=False, default="plivo") - name = Column(String(255), nullable=True) # Optional friendly name to disambiguate multiple credentials - - auth_id = Column(String(255), nullable=False) - auth_token = Column(String(512), nullable=False) - - verify_app_uuid = Column(String(255), nullable=True) - voice_app_id = Column(String(255), nullable=True) - sip_domain = Column(String(255), nullable=True) - masking_config = Column(JSON, nullable=True) - - is_active = Column(Boolean, default=True, nullable=False) - is_default = Column(Boolean, default=False, nullable=False) - last_tested_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyPhoneNumber(Base): - """Inventory of telephony phone numbers owned by an organization.""" - - __tablename__ = "telephony_phone_numbers" - __table_args__ = ( - UniqueConstraint("organization_id", "phone_number", name="uq_telephony_number_org_phone"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - telephony_integration_id = Column( - UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=True, index=True - ) - - phone_number = Column(String(20), nullable=False, index=True) - country_iso2 = Column(String(2), nullable=True) - region = Column(String(100), nullable=True) - number_type = Column(String(20), nullable=True) - capabilities = Column(JSON, nullable=True) - provider_app_id = Column(String(255), nullable=True) - - is_masking_pool = Column(Boolean, default=False, nullable=False) - inbound_enabled = Column(Boolean, default=True, nullable=False) - outbound_enabled = Column(Boolean, default=True, nullable=False) - source = Column(String(20), nullable=False, default="imported") - agent_id = Column( - UUID(as_uuid=True), - ForeignKey( - "agents.id", - ondelete="SET NULL", - use_alter=True, - name="fk_telephony_phone_numbers_agent_id", - ), - nullable=True, - index=True, - ) - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyDialTarget(Base): - """Org-scoped saved destination numbers for outbound test calls.""" - - __tablename__ = "telephony_dial_targets" - __table_args__ = ( - UniqueConstraint("organization_id", "phone_number", name="uq_telephony_dial_target_org_phone"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - phone_number = Column(String(20), nullable=False, index=True) - label = Column(String(255), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyVerifySession(Base): - """Tracks voice OTP verification sessions via telephony provider.""" - - __tablename__ = "telephony_verify_sessions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider_session_uuid = Column(String(255), nullable=False, unique=True, index=True) - recipient_number = Column(String(20), nullable=False) - channel = Column(String(10), nullable=False, default="voice") - status = Column(String(20), nullable=False, default="pending") - initiated_by = Column(String(255), nullable=True) - verify_app_uuid = Column(String(255), nullable=True) - verified_at = Column(DateTime(timezone=True), nullable=True) - expires_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyMaskedSession(Base): - """Number-masking session between two parties through a middle number.""" - - __tablename__ = "telephony_masked_sessions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - telephony_integration_id = Column(UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=False) - masked_number_id = Column( - UUID(as_uuid=True), ForeignKey("telephony_phone_numbers.id"), nullable=False, index=True - ) - masked_number = Column(String(20), nullable=False) - party_a_number = Column(String(20), nullable=False) - party_b_number = Column(String(20), nullable=False) - status = Column(String(20), nullable=False, default="active") - expires_at = Column(DateTime(timezone=True), nullable=True) - ended_at = Column(DateTime(timezone=True), nullable=True) - session_metadata = Column("metadata", JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class CallImportSchema(Base): - """Reusable Input Parameter schema for the call-uploads flow. - - A schema is workspace-scoped: users define a named bundle of typed - Input Parameters once (e.g. "Standard Voice QA" with conversation_id + - recording_url + transcript + agent_name) and then map those parameters - to CSV/Excel headers each time they upload a new batch. - - Every schema MUST contain exactly one parameter with - ``type='conversation_id'`` and ``is_required=True`` - that's the - mandatory identity field every imported row needs. The invariant is - enforced in app code on create/update (no DB-level CHECK because the - parent + children are written across two tables in one transaction). - """ - - __tablename__ = "call_import_schemas" - __table_args__ = ( - # Case-insensitive uniqueness is enforced via the matching partial - # index on ``LOWER(name)`` in the migration; this constraint here - # would be case-sensitive and is intentionally omitted to avoid - # confusing the user. - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - created_by_user_id = Column( - UUID(as_uuid=True), - ForeignKey("users.id", ondelete="SET NULL"), - nullable=True, - ) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - parameters = relationship( - "CallImportSchemaParameter", - back_populates="schema", - cascade="all, delete-orphan", - order_by="CallImportSchemaParameter.ordering", - ) - - -class CallImportSchemaParameter(Base): - """A single typed parameter inside a :class:`CallImportSchema`. - - ``type`` is one of the strings tracked by - :data:`app.models.enums.CallImportParameterType`. ``conversation_id`` - is reserved for the mandatory identity parameter every schema must - contain. - """ - - __tablename__ = "call_import_schema_parameters" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - schema_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_schemas.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - type = Column(String(32), nullable=False) - description = Column(Text, nullable=True) - is_required = Column(Boolean, nullable=False, default=False) - # Stable ordering so the UI renders parameters in the order the - # schema author defined them (matters when conversation_id is pinned - # first and the user re-orders the rest). - ordering = Column(Integer, nullable=False, default=0) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - schema = relationship("CallImportSchema", back_populates="parameters") - - -class CallImport(Base): - """Batch record for a CSV-driven call import job.""" - - __tablename__ = "call_imports" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every imported batch belongs to a workspace - # within its org. The /upload endpoint stamps it from the active - # workspace header (or the org's Default if absent). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - created_by_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) - - # Telephony provider key (e.g. ``'exotel'``, ``'plivo'``). In the - # legacy one-shot ``POST /upload`` endpoint this is supplied with the - # file; in the three-stage flow (UPLOAD -> MAP -> IMPORT) the value - # isn't known until the IMPORT stage, so the column is nullable for - # ``uploaded`` / ``mapped`` batches. - provider = Column(String(50), nullable=True, default="exotel") - # Pin a specific telephony credential for this batch so the worker - # downloads recordings using *that* row instead of the org default. - # NULL preserves legacy behavior (resolve by provider + default). - telephony_integration_id = Column( - UUID(as_uuid=True), - ForeignKey("telephony_integrations.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - original_filename = Column(String(512), nullable=True) - # When the source file was a multi-sheet Excel workbook, this records - # the worksheet the rows came from (one batch per sheet). NULL for CSV - # uploads since CSV has no sheet concept. - sheet_name = Column(String(255), nullable=True) - - # --- Source-file staging (UPLOAD stage) --------------------------- - # The raw CSV / Excel file is stored in S3 between stages so the - # user can come back later to MAP and IMPORT without re-uploading. - # ``source_s3_key`` is NULL on legacy batches that were imported via - # the one-shot endpoint (those batches stay read-only post-import). - source_s3_key = Column(Text, nullable=True) - source_format = Column(String(16), nullable=True) - source_size_bytes = Column(BigInteger, nullable=True) - source_content_type = Column(String(255), nullable=True) - - # Snapshot of the file's sheets + headers captured at UPLOAD time - # so the MAP UI doesn't need to re-fetch the source bytes from S3. - # Shape: ``[{"name": str, "headers": [str, ...], "row_count": int}, ...]``. - available_sheets = Column(JSON, nullable=True) - - # User's explicit "drop these columns" decision captured at MAP - # time. Was validation-only and ephemeral in the legacy flow; now - # persisted so the IMPORT stage can re-parse the file with the same - # mapping/skip intent. - skipped_columns = Column(JSON, nullable=False, default=list) - # Rows skipped at parse time (missing/invalid conversation_id or URL). - # Shape: ``[{"source_row": int, "reason": str, "message": str}, ...]``. - source_row_skips = Column(JSON, nullable=False, default=list) - - # Free-text high-level segregation label. Powers the "Dataset" filter - # at the top of the imports page; multiple imports can share a value. - dataset = Column(String(255), nullable=True, index=True) - - # Reusable Input Parameter schema this batch was uploaded against. - # NULL on legacy batches uploaded before the schema-driven flow - # shipped; those still render via ``column_mapping`` + ``extra_columns`` - # + ``custom_column_mapping`` below. - schema_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_schemas.id", ondelete="RESTRICT"), - nullable=True, - index=True, - ) - # New schema-driven mapping: ``{schema_parameter_name: csv_header}``. - # Populated for new uploads; empty dict on legacy batches. - parameter_mapping = Column(JSON, nullable=False, default=dict) - - # Legacy free-form mapping (pre-schema-flow). Kept on the model so - # batches that were uploaded before the schema feature shipped still - # render correctly on the detail page; new uploads stop writing here. - # Keys: external_call_id (required), transcript, recording_url. - # (DB column ``external_call_id`` is now ``conversation_id``; this - # JSON key stays as-is for historical batches.) - # Values: original CSV header strings (preserve user casing for export). - column_mapping = Column(JSON, nullable=False, default=dict) - # Ordered list of additional CSV header strings the uploader wants - # preserved verbatim into the evaluation export CSV. - extra_columns = Column(JSON, nullable=False, default=list) - # User-defined ``{custom_field_name: csv_header}`` mappings on top of - # the three system fields above. Cells from the mapped CSV columns are - # preserved per row (keyed by the CSV header in ``raw_columns``) and - # surface in the evaluation export under the uploader-chosen name. - custom_column_mapping = Column(JSON, nullable=False, default=dict) - - total_rows = Column(Integer, nullable=False, default=0) - completed_rows = Column(Integer, nullable=False, default=0) - failed_rows = Column(Integer, nullable=False, default=0) - - status = Column( - Enum(CallImportStatus, values_callable=get_enum_values), - nullable=False, - default=CallImportStatus.PENDING, - index=True, - ) - error_message = Column(Text, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - rows = relationship( - "CallImportRow", - back_populates="call_import", - cascade="all, delete-orphan", - order_by="CallImportRow.row_index", - ) - tags = relationship( - "CallImportTag", - secondary="call_import_tag_assignments", - backref="call_imports", - lazy="selectin", - ) - evaluations = relationship( - "CallImportEvaluation", - back_populates="call_import", - cascade="all, delete-orphan", - ) - - -class CallImportShardSlice(Base): - """Registry row: which shard stores a slice of rows for an import.""" - - __tablename__ = "call_import_shard_slices" - - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - primary_key=True, - ) - slice_id = Column(Integer, primary_key=True) - shard_id = Column(String(64), nullable=False, index=True) - row_index_min = Column(Integer, nullable=False) - row_index_max = Column(Integer, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - -class CallImportRow(Base): - """A single row within a CallImport batch (one CSV line / one external call).""" - - __tablename__ = "call_import_rows" - __table_args__ = ( - UniqueConstraint("call_import_id", "row_index", name="uq_call_import_row_index"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - row_index = Column(Integer, nullable=False) - # Was historically named ``external_call_id``; renamed to - # ``conversation_id`` so the new schema-driven upload flow can refer - # to it by a single canonical name across the schema definition, - # exports, and downstream evaluation tables. - conversation_id = Column(String(255), nullable=False, index=True) - # Supplied via CSV for Exotel credentialed imports (required per row). - # Nullable in the schema for legacy rows imported before recording_url - # was mandatory on every Exotel upload. - recording_url = Column(Text, nullable=True) - # Date-only call recording date supplied by the import schema. Used - # for historical report comparisons without timezone/time ambiguity. - recording_date = Column(Date, nullable=True, index=True) - # The "production" transcript: the value supplied via the CSV - # upload mapping. Never overwritten by the diarisation worker — - # the worker writes its output into ``diarised_transcript`` so - # the user keeps both versions side by side. - transcript = Column(Text, nullable=True) - # Snapshot of the original CSV row keyed by the user's headers so the - # evaluation export can reproduce every column the uploader supplied - # (mapped + extra). NULL on legacy rows imported before this column. - raw_columns = Column(JSON, nullable=True) - - # Where the value in ``transcript`` came from. ``csv`` = supplied via - # the upload mapping, ``edited`` = manually changed in the UI. NULL - # on rows that have never had a production transcript. - # (Worker-produced transcripts now live in ``diarised_transcript`` - # and are tracked via ``diarised_transcript_*`` metadata below.) - transcript_source = Column(String(20), nullable=True) - # Provider/model recorded by the (legacy) post-hoc transcription - # worker. New worker runs leave these NULL and write into the - # ``diarised_transcript_*`` columns instead; kept on the model for - # backwards compatibility with pre-split rows that still carry the - # original transcription metadata here. - transcript_provider = Column(String(50), nullable=True) - transcript_model = Column(String(100), nullable=True) - # Lifecycle status for the legacy transcription workflow itself, - # independent of the row's recording-fetch ``status``. ``idle`` = - # no transcribe task has touched this column. New diarisation runs - # update ``diarised_transcript_status`` instead. - transcript_status = Column( - String(20), - nullable=False, - default="idle", - ) - transcript_error = Column(Text, nullable=True) - transcribed_at = Column(DateTime(timezone=True), nullable=True) - - # The "diarised" transcript: produced by the post-hoc - # transcription/diarisation worker. Stored separately so a manual - # diarisation run never clobbers the production transcript above. - # Evaluations can be configured to score against either column - # (see ``CallImportEvaluation.transcript_source``). - diarised_transcript = Column(Text, nullable=True) - # Provider/model the diarisation worker used. Surfaced in the UI - # as "Diarised via deepgram/nova-2" next to the diarised - # transcript section. - diarised_transcript_provider = Column(String(50), nullable=True) - diarised_transcript_model = Column(String(100), nullable=True) - # Lifecycle status for the diarisation workflow. - # ``idle`` = no diarisation task has run; ``pending``/``running`` = - # a Celery task is queued or in flight; ``completed``/``failed`` = - # terminal. Independent of ``transcript_status`` so the two - # transcripts can be in different lifecycle states. - diarised_transcript_status = Column( - String(20), - nullable=False, - default="idle", - server_default="idle", - ) - diarised_transcript_error = Column(Text, nullable=True) - diarised_at = Column(DateTime(timezone=True), nullable=True) - - # Structured speaker turns produced by the diarisation worker — - # ``[{ "speaker": "agent"|"user"|"speaker_3", "text": "...", - # "start": float, "end": float, "raw_speaker": "Speaker 1" }, ...]`` - # The plain-text ``diarised_transcript`` above is a rendered view - # of this list (``: `` per line). When the worker - # cannot recover structured turns (no pyannote token / single- - # speaker recording / provider that doesn't surface segments) this - # column stays NULL and the plain-text path is still populated. - diarised_segments = Column(JSON, nullable=True) - # When True the ``agent`` <-> ``user`` mapping inside - # ``diarised_segments`` is inverted at render / export time. The - # worker writes the canonical mapping using the "first speaker is - # the agent" heuristic; reviewers can flip the toggle from the row - # detail panel without re-running diarisation. - diarised_speaker_swap = Column( - Boolean, - nullable=False, - default=False, - server_default="false", - ) - # LLM that turned the STT plain-text output into structured - # ``diarised_segments``. The legacy diarisation worker used - # pyannote and left these NULL; the current path always runs an - # LLM with the operator-supplied (or default) ``diarised_prompt`` - # below, and records exactly which model + prompt produced each - # row so reviewers can reproduce a specific run. - diarised_llm_provider = Column(String(50), nullable=True) - diarised_llm_model = Column(String(100), nullable=True) - diarised_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) - diarised_prompt = Column(Text, nullable=True) - # Which diarisation pipeline produced this row's turns. - # * ``"stt_llm"`` (default) — two-stage: STT then LLM diariser. - # ``diarised_transcript_provider``/``_model`` describe the STT - # side; ``diarised_llm_provider``/``_model`` the LLM side. - # * ``"llm_only"`` — single-stage: audio fed straight to a - # multimodal LLM. ``diarised_transcript_provider`` is stamped - # with the sentinel ``"llm_only"``; the real model is on - # ``diarised_llm_*``. - # Persisting it on the row (not just the run) lets the row detail - # panel render the right "Diarised via …" label even for ad-hoc - # standalone transcribes (no parent evaluation). - transcribe_mode = Column( - String(20), - nullable=False, - default="stt_llm", - server_default="stt_llm", - ) - - status = Column( - Enum(CallImportRowStatus, values_callable=get_enum_values), - nullable=False, - default=CallImportRowStatus.PENDING, - index=True, - ) - - recording_s3_key = Column(String(1024), nullable=True) - recording_content_type = Column(String(128), nullable=True) - recording_size_bytes = Column(Integer, nullable=True) - - error_message = Column(Text, nullable=True) - attempts = Column(Integer, nullable=False, default=0) - celery_task_id = Column(String(255), nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - call_import = relationship("CallImport", back_populates="rows") - - -@event.listens_for(CallImportRow, "before_insert") -def _call_import_row_fill_workspace_id(_mapper, connection, target): - """Denormalize workspace_id from the parent import when omitted.""" - if target.workspace_id is not None or target.call_import_id is None: - return - workspace_id = connection.execute( - select(CallImport.workspace_id).where( - CallImport.id == target.call_import_id - ) - ).scalar_one_or_none() - if workspace_id is not None: - target.workspace_id = workspace_id - - -class CallImportTag(Base): - """User-defined tag that can be attached to one or more call imports. - - Tags coexist with the free-text ``CallImport.dataset`` column: dataset - is the primary high-level segregation, tags are an optional secondary - classification (an import can have many tags). - """ - - __tablename__ = "call_import_tags" - __table_args__ = ( - UniqueConstraint("organization_id", "name", name="uq_call_import_tag_org_name"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - name = Column(String(255), nullable=False) - color = Column(String(32), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - -class CallImportTagAssignment(Base): - """Many-to-many join table between CallImport and CallImportTag.""" - - __tablename__ = "call_import_tag_assignments" - - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - primary_key=True, - ) - tag_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_tags.id", ondelete="CASCADE"), - primary_key=True, - index=True, - ) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - -class CallImportEvaluation(Base): - """Parent record for an evaluation run over a CallImport batch. - - A user picks a subset of org ``Metric`` rows and triggers an evaluation; - we fan out one ``CallImportEvaluationRow`` per source row and roll up - counters as workers finish. Status mirrors ``CallImportStatus`` plus a - ``RUNNING`` value so the UI can distinguish "queued" from "in flight". - """ - - __tablename__ = "call_import_evaluations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id"), - nullable=False, - index=True, - ) - # Workspace isolation: mirrors the parent CallImport's workspace. - # Denormalized for fast filter-by-workspace listings without a join. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - created_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id"), nullable=True - ) - - # Optional user-supplied label for this run. Lets the UI surface - # something more meaningful than the UUID prefix (e.g. "March QA pass"). - name = Column(String(255), nullable=True) - - # JSON list of Metric UUID strings selected for this run. Stored as text - # in JSON so we don't have to deal with PG arrays of UUIDs / cascade - # delete policies when metrics are removed; the loader filters for - # still-existing org metrics at run time. - selected_metric_ids = Column(JSON, nullable=False, default=list) - # Hierarchy grouping snapshot: ``{parent_id_str: [child_id_str, ...]}``. - # Captures which children belong to which parent for THIS run so the UI - # / aggregator can reconstruct the tree even when the user selected - # only a subset of children, or after metrics are deleted / renamed. - # NULL on legacy rows means "no hierarchy" → fall back to flat - # ``selected_metric_ids`` semantics. - selected_metric_groups = Column(JSON, nullable=True) - # User-driven merges of LLM-discovered candidate sub-labels for - # ``allow_discovery`` parents. Shape: - # ``{"": {"": "", ...}}``. - # Populated via ``POST .../discovered-labels/merge``; consulted by - # the discovered-labels aggregator, the flow graph builder, and the - # worker so that rows finishing AFTER a merge cannot reintroduce - # the merged-away slug. Empty dict on fresh rows. - discovered_label_aliases = Column( - JSON, nullable=False, default=dict, server_default="{}" - ) - - # Per-run opt-in for top-level metric discovery. When True, the LLM - # is asked to propose brand-new top-level metrics (boolean / rating / - # category) observed in the transcripts in addition to scoring the - # ``selected_metric_ids`` for the row. Candidates surface in a - # "Discovered metrics" panel on the evaluation's Flow tab and can - # be promoted into real standalone ``Metric`` rows via - # ``POST /metrics/from-discovered``. Defaults to False so existing - # evaluation creation payloads keep their previous behaviour. - discover_new_metrics = Column( - Boolean, nullable=False, default=False, server_default="false" - ) - # Flat slug-to-slug redirect map for user merges + tombstones of - # discovered top-level metric candidates. Mirrors - # ``discovered_label_aliases`` but is NOT nested per parent — - # top-level metric discovery is not scoped to any parent. Shape:: - # - # {"": "", ...} - # - # An empty-string value tombstones the slug so workers finishing - # later can't re-introduce it. - discovered_metric_aliases = Column( - JSON, nullable=False, default=dict, server_default="{}" - ) - - # Run-level LLM config picked from the Run Evaluation modal. NULL on - # legacy rows means "use the historical OpenAI/gpt-4o default" — the - # worker checks for this and falls back accordingly. ``llm_credential_id`` - # pins a specific AIProvider row when the org has multiple credentials - # for the same provider. - llm_provider = Column(String(50), nullable=True) - llm_model = Column(String(100), nullable=True) - llm_credential_id = Column( - UUID(as_uuid=True), - ForeignKey("aiproviders.id", ondelete="SET NULL"), - nullable=True, - ) - llm_config = Column(JSON, nullable=True) - # Optional per-metric LLM override: - # ``{"": {"provider": "...", "model": "...", "credential_id": "..."}}``. - # Each entry overrides the run-level default for that metric only; - # missing keys = use run-level default. Stored as JSON so the UI can - # round-trip arbitrary {provider, model} pairs without migrations. - metric_llm_overrides = Column(JSON, nullable=True) - - # When ``auto_transcribe`` was set on the create payload, record the - # STT provider/model used so the UI can show "Auto-transcribed via - # deepgram/nova-2" on the evaluation header. ``stt_credential_id`` is - # untyped (no FK) because STT keys may live in either ``aiproviders`` - # (OpenAI) or ``integrations`` (Deepgram, ElevenLabs) — the - # transcription service handles the lookup. - stt_provider = Column(String(50), nullable=True) - stt_model = Column(String(100), nullable=True) - stt_credential_id = Column(UUID(as_uuid=True), nullable=True) - - # Run-level LLM diariser config. Used when the create-run / - # retry-run paths chain a ``transcribe_call_import_row_task`` - # because the row is missing a diarised transcript. Persisted on - # the run so a retry uses the same diariser the original create - # call picked (unless the retry payload explicitly overrides). - diarisation_llm_provider = Column(String(50), nullable=True) - diarisation_llm_model = Column(String(100), nullable=True) - diarisation_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) - diarisation_prompt = Column(Text, nullable=True) - # Mode the run was *created* with for its auto-transcribe step. - # Retry chains read this to decide whether to enqueue an STT+LLM - # transcribe or a single-stage multimodal LLM transcribe — without - # it we'd have to infer the mode from "stt_provider is NULL", which - # would silently break legacy rows that simply never configured - # auto-transcribe. See migration 041 for the column DDL. - transcribe_mode = Column( - String(20), - nullable=False, - default="stt_llm", - server_default="stt_llm", - ) - - # Which of the two transcripts on each ``CallImportRow`` this run - # scored against. ``'production'`` reads ``CallImportRow.transcript`` - # (the CSV-supplied value); ``'diarised'`` reads - # ``CallImportRow.diarised_transcript`` (the worker output). When - # the user ticks both checkboxes in the Run Evaluation modal we - # create two ``CallImportEvaluation`` rows — one per source — so - # the two scorings can be compared side-by-side. Defaults to - # ``'production'`` so legacy runs (which always read the single - # historical ``transcript`` column) keep their semantics. - transcript_source = Column( - String(20), - nullable=False, - default="production", - server_default="production", - ) - - # Cached LLM-generated TLDR rendered above the Visualizations charts. - # Populated lazily by ``POST /evaluations/{eval_id}/insights`` so we - # never auto-burn LLM tokens on page load. Shape:: - # {"narrative": str, "patterns": [str, ...], - # "generated_at": iso8601, "generated_at_completed_rows": int, - # "provider": str, "model": str} - # NULL on rows that have never been summarised. - tldr_summary = Column(JSON, nullable=True) - - # Cached LLM-generated user insights for External Audit PDF section 03. - # Populated by a background Celery job triggered alongside TLDR generation. - # Shape: EvaluationUserInsightsState JSON (status, insights[], progress, …). - user_insights = Column(JSON, nullable=True) - - # Cached per-metric failure clustering for internal diagnostics PDF/UI. - # Shape: EvaluationMetricClustersState JSON (status, groups[], …). - metric_clusters = Column(JSON, nullable=True) - - # Cached LLM-generated prompt improvement suggestions keyed to an - # imported agent (PromptPartial tagged __imported_agent__). - # Shape: EvaluationPromptImprovementsState JSON. - prompt_improvements = Column(JSON, nullable=True) - - # Cached LLM explanations for week-over-week metric deltas keyed by - # baseline evaluation id + completed row counts. - period_delta_explanations = Column(JSON, nullable=True) - - status = Column(String(20), nullable=False, default="pending", index=True) - - total_rows = Column(Integer, nullable=False, default=0) - completed_rows = Column(Integer, nullable=False, default=0) - failed_rows = Column(Integer, nullable=False, default=0) - # Flexprice pass-level delta billing watermark: rows already emitted - # on ``call_import.evaluation_completed`` for this evaluation run. - billed_completed_rows = Column( - Integer, nullable=False, default=0, server_default="0" - ) - error_message = Column(Text, nullable=True) - celery_group_id = Column(String(255), nullable=True) - - started_at = Column(DateTime(timezone=True), nullable=True) - finished_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - call_import = relationship("CallImport", back_populates="evaluations") - row_results = relationship( - "CallImportEvaluationRow", - back_populates="evaluation", - cascade="all, delete-orphan", - ) - - -class CallImportEvaluationRow(Base): - """Per-source-row scoring output for a CallImportEvaluation parent.""" - - __tablename__ = "call_import_evaluation_rows" - __table_args__ = ( - UniqueConstraint( - "evaluation_id", "call_import_row_id", name="uq_call_import_evaluation_row" - ), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - call_import_row_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_rows.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - status = Column(String(20), nullable=False, default="pending", index=True) - # Same shape as EvaluatorResult.metric_scores: {metric_id_str: {value, type, metric_name, ...}} - metric_scores = Column(JSON, nullable=False, default=dict) - error_message = Column(Text, nullable=True) - celery_task_id = Column(String(255), nullable=True) - - started_at = Column(DateTime(timezone=True), nullable=True) - finished_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - evaluation = relationship("CallImportEvaluation", back_populates="row_results") - source_row = relationship("CallImportRow") - - -@event.listens_for(CallImportEvaluationRow, "before_insert") -def _call_import_evaluation_row_fill_workspace_id(_mapper, connection, target): - """Denormalize workspace_id from the parent evaluation when omitted.""" - if target.workspace_id is not None or target.evaluation_id is None: - return - workspace_id = connection.execute( - select(CallImportEvaluation.workspace_id).where( - CallImportEvaluation.id == target.evaluation_id - ) - ).scalar_one_or_none() - if workspace_id is not None: - target.workspace_id = workspace_id - - -class CallImportEvaluationReportSnapshot(Base): - """Persisted PDF-report aggregate used for period-over-period deltas.""" - - __tablename__ = "call_import_evaluation_report_snapshots" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - period_label = Column(String(64), nullable=True, index=True) - period_start = Column(Date, nullable=True, index=True) - period_end = Column(Date, nullable=True, index=True) - report_config = Column(JSON, nullable=False, default=dict, server_default="{}") - selected_metric_ids = Column(JSON, nullable=False, default=list, server_default="[]") - metric_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") - insight_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") - narrative = Column(JSON, nullable=True) - total_calls = Column(Integer, nullable=False, default=0) - selected_metric_count = Column(Integer, nullable=False, default=0) - total_metric_count = Column(Integer, nullable=False, default=0) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - -# --------------------------------------------------------------------------- -# Judge Alignment (AlignEval-style hybrid integration) -# -# Three tables back the "Judge Alignment" surface: -# - JudgeDataset: a labeled dataset materialised from one of three sources -# (voice transcripts, existing Metric/Evaluator outputs, -# or a generic CSV upload). Holds the dataset's source -# config + which fields play the role of input/output. -# - JudgeSample: one row in a dataset (input/output pair plus an -# optional binary pass/fail human label). -# - JudgeRun: a single run of an LLM-judge (existing Evaluator) over -# a subset of samples, with computed alignment metrics -# (precision/recall/F1/Cohen's kappa) and per-sample -# predictions. Optionally links to a GEPA optimization -# run when the user kicks off prompt tuning from a -# dataset. -# --------------------------------------------------------------------------- - - -class JudgeDataset(Base): - """Container for binary-labeled samples used to calibrate an LLM-judge.""" - - __tablename__ = "judge_datasets" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - # Workspace isolation: every judge dataset belongs to a workspace - # within its org. Samples and runs inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - name = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - - # One of: "transcript", "metric_output", "csv" - source_type = Column(String(32), nullable=False, index=True) - # Source-specific config. Examples: - # transcript: {"transcription_ids": [...]} or {"agent_id": "..."} - # metric_output: {"metric_id": "...", "evaluator_id": "..."} - # csv: {"s3_key": "...", "filename": "..."} - source_config = Column(JSON, nullable=False, default=dict) - - # Field roles - which textual content is "input" vs "output" for the judge. - # For voice transcripts both default to the transcript text but can be - # tightened (e.g. agent-only turns vs full conversation). - input_field = Column(String(64), nullable=False, default="input") - output_field = Column(String(64), nullable=False, default="output") - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - samples = relationship( - "JudgeSample", - back_populates="dataset", - cascade="all, delete-orphan", - order_by="JudgeSample.created_at", - ) - runs = relationship( - "JudgeRun", - back_populates="dataset", - cascade="all, delete-orphan", - order_by="JudgeRun.created_at.desc()", - ) - - -class JudgeSample(Base): - """One labelable input/output pair within a JudgeDataset.""" - - __tablename__ = "judge_samples" - __table_args__ = ( - UniqueConstraint("dataset_id", "external_id", name="uq_judge_samples_dataset_external"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - dataset_id = Column( - UUID(as_uuid=True), - ForeignKey("judge_datasets.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - # Workspace isolation: mirrors the parent JudgeDataset's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Stable identifier within the source (e.g. transcription UUID, CSV row id). - # Used to dedupe re-imports and link back to the originating record. - external_id = Column(String(128), nullable=True, index=True) - - input_text = Column(Text, nullable=False) - output_text = Column(Text, nullable=False) - - # Binary human label: "pass" | "fail" | null (unlabeled). - # Stored as string (rather than enum) so it stays trivially extendable. - label = Column(String(16), nullable=True, index=True) - labeled_by = Column(String(255), nullable=True) - labeled_at = Column(DateTime(timezone=True), nullable=True) - - # Source-specific context (e.g. agent_id, original metric value, csv row). - extra = Column(JSON, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - dataset = relationship("JudgeDataset", back_populates="samples") - - -class JudgeRun(Base): - """One execution of an LLM-judge against a JudgeDataset, with alignment metrics.""" - - __tablename__ = "judge_runs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - dataset_id = Column( - UUID(as_uuid=True), - ForeignKey("judge_datasets.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - # Workspace isolation: mirrors the parent JudgeDataset's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Reuses the existing Evaluator row (its custom_prompt + llm_provider + llm_model - # define the judge under test). Nullable so a run may target an inline prompt - # in the future without inflating the Evaluator table. - evaluator_id = Column( - UUID(as_uuid=True), ForeignKey("evaluators.id", ondelete="SET NULL"), nullable=True, index=True - ) - - # Which subset was scored: "all" | "dev" | "test" - split = Column(String(16), nullable=False, default="all") - - # Snapshot of the model used (so a later Evaluator edit doesn't rewrite history). - llm_provider = Column(String(64), nullable=True) - llm_model = Column(String(128), nullable=True) - - # Computed alignment metrics: - # {"precision": float, "recall": float, "f1": float, "kappa": float, - # "tp": int, "fp": int, "tn": int, "fn": int, "n": int} - metrics = Column(JSON, nullable=True) - - # Per-sample predictions, keyed by sample_id (UUID string): - # {sample_id: {"prediction": "pass"|"fail", "explanation": str, "raw": str}} - predictions = Column(JSON, nullable=True) - - # Run lifecycle. - status = Column(String(20), nullable=False, default="pending", index=True) - error_message = Column(Text, nullable=True) - celery_task_id = Column(String, nullable=True, index=True) - - # Optional link to a GEPA optimization run kicked off from this dataset. - gepa_optimization_id = Column( - UUID(as_uuid=True), - ForeignKey("prompt_optimization_runs.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - dataset = relationship("JudgeDataset", back_populates="runs") +"""SQLAlchemy database models.""" + +from sqlalchemy import ( + BigInteger, + Boolean, + Column, + Date, + DateTime, + DDL, + Enum, + event, + Float, + ForeignKey, + Integer, + JSON, + String, + Text, + UniqueConstraint, + select, + text, +) +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import relationship +from sqlalchemy.sql import func +import uuid +import enum +from app.models.enums import ( + EvaluationType, EvaluationStatus, EvaluatorResultStatus, RoleEnum, InvitationStatus, + LanguageEnum, CallTypeEnum, CallMediumEnum, GenderEnum, AccentEnum, BackgroundNoiseEnum, + IntegrationPlatform, ModelProvider, VoiceBundleType, TestAgentConversationStatus, + MetricType, MetricCategory, MetricTrigger, CallRecordingStatus, AlertMetricType, AlertAggregation, + AlertOperator, AlertNotifyFrequency, AlertStatus, AlertHistoryStatus, CronJobStatus, + PromptOptimizationStatus, CallImportStatus, CallImportRowStatus, +) + +def get_enum_values(enum_class): + """Helper to get values from enum class for SQLAlchemy.""" + return [e.value for e in enum_class] + +from app.database import Base + + +# Enums moved to enums.py + + +class Organization(Base): + """Organization model for multi-tenancy.""" + + __tablename__ = "organizations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + name = Column(String(255), nullable=False) + voice_playground_threshold_overrides = Column(JSON, nullable=True) + # AlignEval-style judge alignment thresholds. + # Shape: {"min_labels_to_evaluate": int, "min_labels_to_optimize": int} + # Falls back to system defaults (20 / 50) when null. + judge_alignment_settings = Column(JSON, nullable=True) + # Per-org LLM gateway overrides (enabled, gateway_type, base_url, keys). + llm_gateway_settings = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + # Relationships + api_keys = relationship("APIKey", back_populates="organization") + members = relationship("OrganizationMember", back_populates="organization", cascade="all, delete-orphan") + invitations = relationship("Invitation", back_populates="organization", cascade="all, delete-orphan") + workspaces = relationship( + "Workspace", + back_populates="organization", + cascade="all, delete-orphan", + ) + workspace_roles = relationship( + "WorkspaceRole", + back_populates="organization", + cascade="all, delete-orphan", + ) + + +class Workspace(Base): + """Workspace - in-org isolation boundary for call imports and metrics. + + Every organization has at least one workspace (``is_default = True``, + seeded by migration 033). Users pick an "active workspace" in the UI; + list endpoints filter by it so users only see calls/metrics from the + project they're currently working in. Access is governed by + ``workspace_members`` and org-scoped ``workspace_roles`` (capability + bundles); org admins implicitly access all workspaces. + """ + + __tablename__ = "workspaces" + __table_args__ = ( + UniqueConstraint("organization_id", "slug", name="uq_workspaces_org_slug"), + ) + + # ``server_default`` is required so that raw-SQL INSERTs (e.g. the + # per-org Default seed in migration 033) can omit ``id`` and let the + # database fill it in. Without it, ``create_all`` produces a column + # with NOT NULL but no DEFAULT, and the migration crashes with + # ``null value in column "id"``. + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + server_default=text("gen_random_uuid()"), + ) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + slug = Column(String(255), nullable=False) + # At most one default per org. Enforced on Postgres by the partial + # unique index attached via the after_create event below; on + # SQLite (test runs) we rely on the route-level _check_slug_unique + # check + the Default-workspace conftest fixture instead, because + # SQLite doesn't support partial indexes the same way. + is_default = Column(Boolean, nullable=False, default=False, server_default="false") + # Reusable PDF/report branding metadata scoped to this workspace. Images + # live in S3. Shape: {"heading": str|null, "images": [{id, s3_key, + # content_type, filename, size_bytes, updated_at}, ...]}. + report_branding = Column(JSON, nullable=True) + created_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + organization = relationship("Organization", back_populates="workspaces") + members = relationship( + "WorkspaceMember", + back_populates="workspace", + cascade="all, delete-orphan", + ) + + +class WorkspaceRole(Base): + """Org-scoped workspace role (system or custom) as a capability bundle.""" + + __tablename__ = "workspace_roles" + __table_args__ = ( + UniqueConstraint("organization_id", "name", name="uq_workspace_roles_org_name"), + ) + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + server_default=text("gen_random_uuid()"), + ) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + capabilities = Column(JSON, nullable=False, default=list) + is_system = Column(Boolean, nullable=False, default=False, server_default="false") + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + organization = relationship("Organization", back_populates="workspace_roles") + members = relationship("WorkspaceMember", back_populates="role") + + +class WorkspaceMember(Base): + """User membership in a workspace with an assigned workspace role.""" + + __tablename__ = "workspace_members" + __table_args__ = ( + UniqueConstraint("workspace_id", "user_id", name="uq_workspace_members_ws_user"), + ) + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + server_default=text("gen_random_uuid()"), + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + user_id = Column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + role_id = Column( + UUID(as_uuid=True), + ForeignKey("workspace_roles.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + added_by_user_id = Column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + workspace = relationship("Workspace", back_populates="members") + user = relationship("User", foreign_keys=[user_id]) + role = relationship("WorkspaceRole", back_populates="members") + added_by = relationship("User", foreign_keys=[added_by_user_id]) + + +# Partial unique index: "at most one default workspace per org". This +# is attached as an after_create event (rather than declared in +# ``__table_args__``) because SQLAlchemy's ``Index(..., +# postgresql_where=...)`` silently degrades to a *full* unique index on +# SQLite - which then forbids any second workspace per org and breaks +# the test suite. ``execute_if(dialect="postgresql")`` makes this DDL +# a no-op on SQLite while still emitting it on Postgres (prod, CI). +event.listen( + Workspace.__table__, + "after_create", + DDL( + "CREATE UNIQUE INDEX IF NOT EXISTS uq_workspaces_org_default " + "ON workspaces (organization_id) WHERE is_default" + ).execute_if(dialect="postgresql"), +) + + +class User(Base): + """User model for authentication and profile management.""" + + __tablename__ = "users" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + email = Column(String(255), unique=True, nullable=False, index=True) + name = Column(String(255), nullable=True) + first_name = Column(String(255), nullable=True) + last_name = Column(String(255), nullable=True) + password_hash = Column(String(255), nullable=True) # Nullable for users created via invitation + external_id = Column(String(255), unique=True, nullable=True, index=True) + auth_provider = Column(String(50), nullable=True) + mfa_enabled = Column(Boolean, default=False, nullable=False) + last_login_at = Column(DateTime(timezone=True), nullable=True) + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + # Relationships + organization_memberships = relationship("OrganizationMember", back_populates="user", cascade="all, delete-orphan") + api_keys = relationship("APIKey", back_populates="user") + invitations = relationship("Invitation", back_populates="invited_user", foreign_keys="Invitation.invited_user_id") + refresh_tokens = relationship("RefreshToken", back_populates="user", cascade="all, delete-orphan") + + +class RefreshToken(Base): + """Opaque refresh token for extending local-password sessions.""" + + __tablename__ = "refresh_tokens" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True) + token_hash = Column(String(64), unique=True, nullable=False, index=True) + expires_at = Column(DateTime(timezone=True), nullable=False) + revoked_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + user = relationship("User", back_populates="refresh_tokens") + + +class OrganizationMember(Base): + """Organization membership with role.""" + + __tablename__ = "organization_members" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False, index=True) + role = Column(String, nullable=False, default=RoleEnum.READER.value) + + # User preferences for this organization + default_agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) + + joined_at = Column(DateTime(timezone=True), server_default=func.now()) + + # Unique constraint: one membership per user per organization + __table_args__ = ( + UniqueConstraint('organization_id', 'user_id', name='uq_org_user'), + ) + + # Relationships + organization = relationship("Organization", back_populates="members") + user = relationship("User", back_populates="organization_memberships") + default_agent = relationship("Agent", foreign_keys=[default_agent_id]) + + +class Invitation(Base): + """Invitation model for inviting users to organizations.""" + + __tablename__ = "invitations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + invited_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Null if user doesn't exist yet + invited_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False) + email = Column(String(255), nullable=False) # Email of invited user + role = Column(String, nullable=False, default=RoleEnum.READER.value) + status = Column(String, nullable=False, default=InvitationStatus.PENDING.value) + + + + token = Column(String(255), unique=True, nullable=False, index=True) # Invitation token + expires_at = Column(DateTime(timezone=True), nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + accepted_at = Column(DateTime(timezone=True), nullable=True) + + # Relationships + organization = relationship("Organization", back_populates="invitations") + invited_user = relationship("User", foreign_keys=[invited_user_id], back_populates="invitations") + invited_by = relationship("User", foreign_keys=[invited_by_id]) + + +class APIKey(Base): + """API Key model for authentication.""" + + __tablename__ = "api_keys" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + key = Column(String(255), unique=True, nullable=False, index=True) + name = Column(String(255), nullable=True) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Optional: link to user + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + last_used = Column(DateTime(timezone=True), nullable=True) + + # Relationships + organization = relationship("Organization", back_populates="api_keys") + user = relationship("User", back_populates="api_keys") + + +class AudioFile(Base): + """Audio file model.""" + + __tablename__ = "audio_files" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + filename = Column(String(255), nullable=False) + file_path = Column(String(512), nullable=False) + file_size = Column(Integer, nullable=False) # Size in bytes + duration = Column(Float, nullable=True) # Duration in seconds + sample_rate = Column(Integer, nullable=True) + channels = Column(Integer, nullable=True) + format = Column(String(10), nullable=False) # wav, mp3, flac, etc. + uploaded_at = Column(DateTime(timezone=True), server_default=func.now()) + + # Relationships + evaluations = relationship("Evaluation", back_populates="audio_file") + + +class Evaluation(Base): + """Evaluation job model.""" + + __tablename__ = "evaluations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every legacy audio evaluation belongs to a + # workspace within its org. Stamped from the X-Workspace-Id header + # (falling back to the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + audio_id = Column(UUID(as_uuid=True), ForeignKey("audio_files.id"), nullable=False) + reference_text = Column(String, nullable=True) # For WER calculation + evaluation_type = Column(String, nullable=False) + model_name = Column(String(100), nullable=True) + status = Column(String, default=EvaluationStatus.PENDING.value, nullable=False) + + + + metrics_requested = Column(JSON, nullable=True) # List of requested metrics + created_at = Column(DateTime(timezone=True), server_default=func.now()) + started_at = Column(DateTime(timezone=True), nullable=True) + completed_at = Column(DateTime(timezone=True), nullable=True) + error_message = Column(String, nullable=True) + + # Relationships + audio_file = relationship("AudioFile", back_populates="evaluations") + result = relationship("EvaluationResult", back_populates="evaluation", uselist=False) + + +class EvaluationResult(Base): + """Evaluation result model.""" + + __tablename__ = "evaluation_results" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column(UUID(as_uuid=True), ForeignKey("evaluations.id"), nullable=False, unique=True) + # Workspace isolation: mirrors the parent Evaluation's workspace. + # Denormalized for fast filter-by-workspace listings without a join. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + transcript = Column(String, nullable=True) + metrics = Column(JSON, nullable=False) # {"wer": 0.05, "latency_ms": 1250, ...} + raw_output = Column(JSON, nullable=True) # Full model output + processing_time = Column(Float, nullable=True) # Processing time in seconds + model_used = Column(String(100), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + # Relationships + evaluation = relationship("Evaluation", back_populates="result") + + +# ============================================ +# VAIOPS MODELS - Voice AI Ops +# ============================================ + +# Enums moved to enums.py + + +class Agent(Base): + """Test Agent - The voice AI agent being evaluated""" + __tablename__ = "agents" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + agent_id = Column(String(6), unique=True, nullable=True, index=True) # 6-digit ID + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every agent belongs to a workspace within its + # org. Stamped from the X-Workspace-Id header (falling back to the + # org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String, nullable=False) + phone_number = Column(String, nullable=True) # Optional, required only for phone_call + language = Column(String, nullable=False, default=LanguageEnum.ENGLISH.value) + description = Column(String) + provider_prompt = Column(Text, nullable=True) + provider_prompt_synced_at = Column(DateTime(timezone=True), nullable=True) + call_type = Column(String, nullable=False, default=CallTypeEnum.OUTBOUND.value) + call_medium = Column(String, nullable=False, default=CallMediumEnum.PHONE_CALL.value) + telephony_phone_number_id = Column( + UUID(as_uuid=True), + ForeignKey("telephony_phone_numbers.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + + + + # Voice configuration - either voice_bundle_id OR ai_provider_id OR voice_ai_integration_id (mutually exclusive) + voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True, index=True) + ai_provider_id = Column(UUID(as_uuid=True), ForeignKey("aiproviders.id"), nullable=True, index=True) + + # Voice AI agent integration (Retell, Vapi, etc.) + voice_ai_integration_id = Column(UUID(as_uuid=True), ForeignKey("integrations.id"), nullable=True, index=True) + voice_ai_agent_id = Column(String, nullable=True) # Agent ID from the external provider (Retell/Vapi) + prompt_variables = Column(JSON, nullable=True) + silence_hangup_secs = Column(Integer, nullable=False, server_default="15") + + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + created_by = Column(String) + + +class Persona(Base): + """Persona - TTS provider-tied voice identity for testing""" + __tablename__ = "personas" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every persona belongs to a workspace within + # its org. Stamped from the X-Workspace-Id header (falling back to + # the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String, nullable=False) + gender = Column(String, nullable=False, default=GenderEnum.NEUTRAL.value) + tts_provider = Column(String(100), nullable=True) + tts_voice_id = Column(String(255), nullable=True) + tts_voice_name = Column(String(255), nullable=True) + is_custom = Column(Boolean, default=False) + description = Column(Text, nullable=True) + tts_config = Column(JSON, nullable=True) + llm_temperature = Column(Float, nullable=True) + llm_max_tokens = Column(Integer, nullable=True) + response_delay_ms = Column(Integer, nullable=True) + max_turns = Column(Integer, nullable=True) + allow_interruptions = Column(Boolean, nullable=True) + + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + created_by = Column(String) + + +class Scenario(Base): + """Scenario - The conversation scenario/test case""" + __tablename__ = "scenarios" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every scenario belongs to a workspace within + # its org. Stamped from the X-Workspace-Id header (falling back to + # the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) + name = Column(String, nullable=False) + description = Column(String) + required_info = Column(JSON) + + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + created_by = Column(String) + + +# Enums moved to enums.py + + +class Integration(Base): + """Integration model for connecting with external voice AI platforms.""" + __tablename__ = "integrations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + platform = Column(String, nullable=False) + + + + name = Column(String, nullable=True) # Optional friendly name + api_key = Column(String, nullable=False) # Encrypted Private API key for the platform + public_key = Column(String, nullable=True) # Optional Public API key (e.g. for Vapi) + is_active = Column(Boolean, default=True, nullable=False) + # Multiple credentials per (org, platform) are allowed. is_default marks + # the row used when a caller does not explicitly select a credential. + # A partial unique index in migration 028 enforces at most one default + # per (org, platform) at the DB level. + is_default = Column(Boolean, default=False, nullable=False) + # inherit | gateway | direct — per-credential LLM routing override + routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated + + +class ManualTranscription(Base): + """Manual transcription model for storing transcriptions from S3 audio files.""" + + __tablename__ = "manual_transcriptions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + name = Column(String(255), nullable=True) # User-friendly name for the transcription + audio_file_key = Column(String(512), nullable=False) # S3 key or file path + transcript = Column(String, nullable=False) # Full transcript text + speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] + stt_model = Column(String(100), nullable=True) # STT model used (e.g., "whisper-1", "google-speech-v2") + stt_provider = Column(String, nullable=True) # Provider used + + + + language = Column(String(10), nullable=True) # Detected or specified language + processing_time = Column(Float, nullable=True) # Processing time in seconds + raw_output = Column(JSON, nullable=True) # Full model output for reference + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class ConversationEvaluation(Base): + """Conversation evaluation model for evaluating manual transcriptions against agent objectives.""" + + __tablename__ = "conversation_evaluations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + transcription_id = Column(UUID(as_uuid=True), ForeignKey("manual_transcriptions.id"), nullable=False, index=True) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) + + # Evaluation results + objective_achieved = Column(Boolean, nullable=False) # Binary: was the conversation objective achieved? + objective_achieved_reason = Column(String, nullable=True) # Explanation for the binary result + additional_metrics = Column(JSON, nullable=True) # Additional evaluation metrics (e.g., professionalism, clarity, etc.) + overall_score = Column(Float, nullable=True) # Overall score (0.0 to 1.0) + + # LLM metadata + llm_provider = Column(Enum(ModelProvider, native_enum=False), nullable=True) + + llm_model = Column(String(100), nullable=True) + llm_response = Column(JSON, nullable=True) # Full LLM response for reference + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class AIProvider(Base): + """AI Provider - Stores API keys for different AI platforms.""" + __tablename__ = "aiproviders" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider = Column(String, nullable=False) + + + + api_key = Column(String, nullable=False) # Encrypted API key + name = Column(String, nullable=True) # Optional friendly name + # Azure OpenAI resource endpoint (e.g. https://my-resource.openai.azure.com). + # Only used when provider is azure; other providers ignore this column. + endpoint_url = Column(String, nullable=True) + is_active = Column(Boolean, default=True, nullable=False) + # Multiple AIProvider rows per (org, provider) are allowed. is_default + # marks the row resolved when no explicit credential id is selected. + # A partial unique index in migration 028 enforces at most one default. + is_default = Column(Boolean, default=False, nullable=False) + # inherit | gateway | direct — per-credential LLM routing override + routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") + # Bifrost custom model ID used when routing via gateway + gateway_model = Column(String(255), nullable=True) + # inherit | litellm_shim | native_openai — Bifrost API surface override + gateway_interface = Column(String(20), nullable=False, default="inherit", server_default="inherit") + # Optional per-credential Bifrost/gateway base URL override + gateway_base_url = Column(String(512), nullable=True) + # Optional auth header for Bifrost (e.g. x-bf-vk, Authorization, x-api-key) + gateway_auth_header = Column(String(64), nullable=True) + # Env var name whose value is sent as the gateway auth secret + gateway_auth_secret_env = Column(String(128), nullable=True) + # Encrypted inline gateway auth secret (alternative to env var) + gateway_auth_secret = Column(String, nullable=True) + # Arbitrary HTTP headers sent with gateway-routed LiteLLM calls + gateway_extra_headers = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated + + +# Enums moved to enums.py + + +class VoiceBundle(Base): + """VoiceBundle - Composable unit combining STT, LLM, and TTS for voice AI testing, or S2S models.""" + __tablename__ = "voicebundles" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + name = Column(String, nullable=False) + description = Column(String, nullable=True) + + # Bundle type: either STT+LLM+TTS or S2S + # Using String instead of Enum to avoid SQLAlchemy enum conversion issues + # The enum conversion is handled in the Pydantic schemas + bundle_type = Column(String(50), nullable=False, default=VoiceBundleType.STT_LLM_TTS.value) + + # STT Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S + stt_provider = Column(String, nullable=True) + # Optional explicit credential row (aiproviders.id or integrations.id). + # When NULL the credential resolver picks the default row for the + # provider. No FK is set because the target table varies by provider. + stt_credential_id = Column(UUID(as_uuid=True), nullable=True) + + stt_model = Column(String, nullable=True) # e.g., "whisper-1", "google-speech-v2" + + # LLM Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S + llm_provider = Column(String, nullable=True) + llm_credential_id = Column(UUID(as_uuid=True), nullable=True) + + llm_model = Column(String, nullable=True) # e.g., "gpt-4", "claude-3-opus" + llm_temperature = Column(Float, nullable=True, default=0.7) + llm_max_tokens = Column(Integer, nullable=True) + llm_config = Column(JSON, nullable=True) # Additional LLM configuration (extensible) + + # TTS Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S + tts_provider = Column(String, nullable=True) + tts_credential_id = Column(UUID(as_uuid=True), nullable=True) + + tts_model = Column(String, nullable=True) # e.g., "tts-1", "neural-voice" + tts_voice = Column(String, nullable=True) # Voice selection if applicable + tts_config = Column(JSON, nullable=True) # Additional TTS configuration (extensible) + + # S2S Configuration - required for S2S type, optional for STT_LLM_TTS + s2s_provider = Column(String, nullable=True) + s2s_credential_id = Column(UUID(as_uuid=True), nullable=True) + + + + s2s_model = Column(String, nullable=True) # e.g., "gpt-4o-transcribe", speech-to-speech model + s2s_config = Column(JSON, nullable=True) # Additional S2S configuration (extensible) + + # Additional configuration for extensibility + extra_metadata = Column(JSON, nullable=True) # For future extensions (renamed from 'metadata' to avoid SQLAlchemy conflict) + + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +# Enums moved to enums.py + + +class TestAgentConversation(Base): + """Test Agent Conversation - Records conversations between test AI agent and voice AI agent.""" + __tablename__ = "test_agent_conversations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every playground conversation belongs to a + # workspace within its org. Stamped from the X-Workspace-Id header + # (falling back to the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Configuration + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=False) + voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) + + # Conversation data + status = Column(String, nullable=False, default=TestAgentConversationStatus.INITIALIZING.value) + + + + live_transcription = Column(JSON, nullable=True) # Array of conversation turns with timestamps + conversation_audio_key = Column(String, nullable=True) # S3 key for recorded conversation audio + full_transcript = Column(String, nullable=True) # Full conversation transcript + + # Metadata + started_at = Column(DateTime(timezone=True), server_default=func.now()) + ended_at = Column(DateTime(timezone=True), nullable=True) + duration_seconds = Column(Float, nullable=True) + + # Additional metadata + conversation_metadata = Column(JSON, nullable=True) # Additional conversation metadata + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class EvaluatorSuite(Base): + """Evaluator suite — one agent + one persona + N scenario combinations.""" + + __tablename__ = "evaluator_suites" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + name = Column(String, nullable=True) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) + metric_ids = Column(JSON, nullable=True) + llm_provider = Column(String, nullable=True) + llm_model = Column(String, nullable=True) + llm_config = Column(JSON, nullable=True) + tags = Column(JSON, nullable=True) + default_runs_per_combination = Column(Integer, nullable=False, default=1) + round_robin_index = Column(Integer, nullable=False, default=0) + is_active = Column(Boolean, nullable=False, default=False) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class Evaluator(Base): + """Evaluator - Configuration for testing agents with specific persona and scenario combinations, or custom prompt evaluators.""" + __tablename__ = "evaluators" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluator_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every evaluator belongs to a workspace within + # its org. Stamped from the X-Workspace-Id header (falling back to + # the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Display name (required for custom evaluators, optional for standard) + name = Column(String, nullable=True) + + # Parent suite (nullable for legacy/custom evaluators) + suite_id = Column( + UUID(as_uuid=True), + ForeignKey("evaluator_suites.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + + # Standard evaluator configuration (nullable for custom evaluators) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) + + # Custom evaluator prompt (used instead of agent/persona/scenario) + custom_prompt = Column(Text, nullable=True) + + # Custom evaluator metric selection. When set, the worker filters the + # enabled-org metrics down to only these IDs (list of metric UUID strings). + # Standard evaluators leave this NULL and use all enabled agent metrics. + metric_ids = Column(JSON, nullable=True) + + # LLM configuration for evaluation (overrides hardcoded defaults) + llm_provider = Column(String, nullable=True) # e.g. "openai", "anthropic", "google" + llm_model = Column(String, nullable=True) # e.g. "gpt-4.1", "claude-sonnet-4-20250514" + llm_config = Column(JSON, nullable=True) + + # Tags for categorization + tags = Column(JSON, nullable=True) # Array of tag strings + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +# Enums moved to enums.py + + +class Metric(Base): + """Metric - Configuration for evaluation metrics. + + Supports a 2-level hierarchy via ``parent_metric_id``: a "category" + parent metric (e.g. "Call Outcome") owns N child sub-metric labels + (e.g. "happy_completion", "angry_hangup"). ``selection_mode`` is set + only on parents and controls how the LLM scores children together + (``single_choice`` = pick exactly one; ``multi_label`` = independent + yes/no with logical consistency). Children are always boolean. + """ + __tablename__ = "metrics" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: two-shape column. + # + # * ``workspace_id = `` — workspace-scoped metric. Only + # visible inside that workspace (the default behavior; existing + # rows all look like this). + # * ``workspace_id IS NULL`` — org-shared metric. Surfaces in + # every workspace's listing under this org so users don't have + # to recreate the same metric per workspace. + # + # Children always inherit their parent's ``workspace_id`` (including + # NULL) so a category metric's whole subtree shares one scope; the + # add-child / promote-discovered endpoints enforce this. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=True, + index=True, + ) + + # Basic information + name = Column(String, nullable=False) + description = Column(String, nullable=True) + # Free-form illustrative example used to sharpen the LLM judge's + # rubric. Today this is consumed by child sub-labels of a + # categorization parent metric so each label can carry "what does + # this look like in a transcript?" text alongside the rubric in + # ``description``. The column lives on every Metric row for + # forward-compat: a standalone metric could later surface its own + # example without another migration. + example = Column(Text, nullable=True) + + # Configuration + metric_type = Column(String, nullable=False, default=MetricType.RATING.value) + metric_category = Column( + String(30), + nullable=False, + default=MetricCategory.QUALITY.value, + server_default=MetricCategory.QUALITY.value, + ) + trigger = Column(String, nullable=False, default=MetricTrigger.ALWAYS.value) + metric_origin = Column(String(30), nullable=False, default="default") + supported_surfaces = Column(JSON, nullable=False, default=list) # ["agent", "voice_playground", "blind_test"] + enabled_surfaces = Column(JSON, nullable=False, default=list) # subset of supported_surfaces + custom_data_type = Column(String(30), nullable=True) # "boolean" | "enum" | "number_range" + custom_config = Column(JSON, nullable=True) # enum options / number range config + tags = Column(JSON, nullable=True) # ["tone", "latency", ...] + + # Hierarchy: NULL = standalone or parent. When set, this row is a + # child sub-metric of the referenced parent. ON DELETE CASCADE so + # deleting a category removes its children atomically. + parent_metric_id = Column( + UUID(as_uuid=True), + ForeignKey("metrics.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + # Set only on parent rows (``parent_metric_id IS NULL``). Either + # ``single_choice`` or ``multi_label``. NULL = legacy / non-hierarchical + # metric (no children). + selection_mode = Column(String(20), nullable=True) + + # When true on a parent metric (any selection_mode), the LLM is + # invited during call-import evaluation to emit additional + # candidate sub-labels beyond the user-defined children. The + # candidates surface in a "Discovered labels" panel where the user + # manually promotes them into real child Metric rows. For + # ``single_choice`` parents the discovered entries are + # supplemental — the chosen child is still picked from the + # predefined children so the exactly-one-true invariant holds. + # The validator rejects this flag on standalone / child metrics. + allow_discovery = Column( + Boolean, nullable=False, default=False, server_default="false" + ) + + # When True, this metric is a "transcript-compare judge": the + # call-import evaluator feeds BOTH the production transcript + # (``call_import_rows.transcript``, CSV-supplied) and the diarised + # transcript (``call_import_rows.diarised_transcript``, worker- + # produced by the STT/diarisation pipeline) to the LLM as a + # labeled pair instead of feeding one transcript. The parent + # evaluation's ``CallImportEvaluation.transcript_source`` is + # ignored for these metrics — they always read both columns. + # Rows where either transcript is missing are skipped per-metric + # with ``skipped="comparison_missing_transcript"`` so the rest of + # the row's metrics still produce scores. The Pydantic validator + # rejects ``compare_transcripts`` combined with ``parent_metric_id`` + # or ``selection_mode`` (i.e. it can't simultaneously be part of + # a parent/child hierarchy). The call-import worker also + # auto-promotes a metric to comparison mode when its description + # references the production / diarised transcripts in well-known + # phrases (see ``_metric_text_references_production`` in + # ``app.workers.tasks.evaluate_call_import_row``). + compare_transcripts = Column( + Boolean, nullable=False, default=False, server_default="false" + ) + + parent = relationship( + "Metric", + remote_side=[id], + backref="children", + ) + + # When true, the LLM-judge is asked to also return a short free-form + # rationale alongside the value (stored under ``metric_scores[id].rationale``). + # Adds a second " - LLM Rationale" column in the call-import CSV export. + capture_rationale = Column(Boolean, nullable=False, default=False) + + enabled = Column(Boolean, nullable=False, default=True) + + # Metadata + is_default = Column(Boolean, nullable=False, default=False) # Pre-defined metrics + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class EvaluatorResult(Base): + """EvaluatorResult - Results from running an evaluator with transcription and metric evaluations.""" + __tablename__ = "evaluator_results" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + result_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every evaluator result belongs to a workspace + # within its org. Stamped from the active workspace at creation time + # (either the X-Workspace-Id header or the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # References + evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True, index=True) # Optional - can be None for test calls without persona/scenario + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Nullable for custom evaluators + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) # Optional - can be None for test calls + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) # Optional - can be None for test calls + + # Result data + name = Column(String, nullable=True) # Scenario name or test call name (optional) + timestamp = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + duration_seconds = Column(Float, nullable=True) # Call duration + status = Column(String(20), nullable=False, default=EvaluatorResultStatus.QUEUED.value) + + # Audio and transcription + audio_s3_key = Column(String, nullable=True) # S3 key for audio file + transcription = Column(String, nullable=True) # Full transcription + speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] + + # Metric scores - JSON object with metric_id as key and score as value + # Format: {"metric_id_1": {"value": 85, "type": "rating"}, "metric_id_2": {"value": true, "type": "boolean"}} + metric_scores = Column(JSON, nullable=True) + + # Celery task tracking + celery_task_id = Column(String, nullable=True, index=True) # Celery task ID for tracking + + # Error information + error_message = Column(String, nullable=True) + + # Call event tracking (similar to CallRecording) + call_event = Column(String, nullable=True, index=True) # Latest call event (e.g., call_started, call_ended) + provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) + provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" + call_data = Column(JSON, nullable=True) # Full call details from provider (like CallRecording) + + # Data-plane shard routing (payload rows on shard DBs when sharding enabled) + shard_id = Column(String(64), nullable=True, index=True) + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +# Enums moved to enums.py + + +class CallRecordingSource(str, enum.Enum): + """Source of the call recording data.""" + + PLAYGROUND = "playground" + WEBHOOK = "webhook" + + +class CallRecording(Base): + """Call Recording model for tracking voice provider calls.""" + __tablename__ = "call_recordings" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every recording belongs to a workspace within + # its org. For playground-origin rows this is stamped from the active + # workspace at creation time; for webhook-origin rows the worker + # looks up the recording's agent and inherits its workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + call_short_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID + status = Column(Enum(CallRecordingStatus), nullable=False, default=CallRecordingStatus.PENDING, index=True) + call_event = Column(String, nullable=True, index=True) # Latest webhook event (e.g., call_started, call_ended) + source = Column(Enum(CallRecordingSource), nullable=False, default=CallRecordingSource.PLAYGROUND, index=True) + call_data = Column(JSON, nullable=True) # JSON blob for provider response + provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) + provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Reference to our agent + + # Link to EvaluatorResult for metric evaluations + evaluator_result_id = Column( + UUID(as_uuid=True), + ForeignKey("evaluator_results.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + shard_id = Column(String(64), nullable=True, index=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class EvaluatorResultPayload(Base): + """Heavy evaluator result fields stored on data shards when sharding is enabled.""" + + __tablename__ = "evaluator_result_payloads" + + evaluator_result_id = Column(UUID(as_uuid=True), primary_key=True) + workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) + audio_s3_key = Column(String, nullable=True) + transcription = Column(String, nullable=True) + speaker_segments = Column(JSON, nullable=True) + metric_scores = Column(JSON, nullable=True) + call_data = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class CallRecordingPayload(Base): + """Heavy call recording fields stored on data shards when sharding is enabled.""" + + __tablename__ = "call_recording_payloads" + + call_recording_id = Column(UUID(as_uuid=True), primary_key=True) + workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) + call_data = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class Alert(Base): + """Alert model for configuring monitoring alerts.""" + __tablename__ = "alerts" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + + # Basic information + name = Column(String(255), nullable=False) + description = Column(String, nullable=True) + + # Metric condition configuration + metric_type = Column(String, nullable=False, default=AlertMetricType.NUMBER_OF_CALLS.value) + aggregation = Column(String, nullable=False, default=AlertAggregation.SUM.value) + operator = Column(String, nullable=False, default=AlertOperator.GREATER_THAN.value) + threshold_value = Column(Float, nullable=False) + time_window_minutes = Column(Integer, nullable=False, default=60) # Time window for aggregation + + # Agent selection (JSON array of agent UUIDs, null means all agents) + agent_ids = Column(JSON, nullable=True) + + # Notification configuration + notify_frequency = Column(String, nullable=False, default=AlertNotifyFrequency.IMMEDIATE.value) + notify_emails = Column(JSON, nullable=True) # Array of email addresses + notify_webhooks = Column(JSON, nullable=True) # Array of webhook URLs (Slack, etc.) + + # Status + status = Column(String, nullable=False, default=AlertStatus.ACTIVE.value) + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + # Relationships + alert_history = relationship("AlertHistory", back_populates="alert", cascade="all, delete-orphan") + + +class AlertHistory(Base): + """Alert history model for tracking triggered alerts.""" + __tablename__ = "alert_history" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + alert_id = Column(UUID(as_uuid=True), ForeignKey("alerts.id"), nullable=False, index=True) + + # Trigger information + triggered_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + triggered_value = Column(Float, nullable=False) # The actual value that triggered the alert + threshold_value = Column(Float, nullable=False) # The threshold at time of trigger + + # Status tracking + status = Column(String, nullable=False, default=AlertHistoryStatus.TRIGGERED.value) + + # Notification tracking + notified_at = Column(DateTime(timezone=True), nullable=True) + notification_details = Column(JSON, nullable=True) # Details of sent notifications + + # Resolution + acknowledged_at = Column(DateTime(timezone=True), nullable=True) + acknowledged_by = Column(String, nullable=True) + resolved_at = Column(DateTime(timezone=True), nullable=True) + resolved_by = Column(String, nullable=True) + resolution_notes = Column(String, nullable=True) + + # Additional context + context_data = Column(JSON, nullable=True) # Additional data about the trigger + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + # Relationships + alert = relationship("Alert", back_populates="alert_history") + + +class CronJob(Base): + """Cron job model for scheduling automated evaluator runs.""" + __tablename__ = "cron_jobs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + + # Basic information + name = Column(String(255), nullable=False) + cron_expression = Column(String(100), nullable=False) # e.g., "0 9 * * 1-5" + timezone = Column(String(100), nullable=False, default="UTC") + + # Run configuration + max_runs = Column(Integer, nullable=False, default=10) + current_runs = Column(Integer, nullable=False, default=0) + + # Evaluators to trigger (JSON array of evaluator UUIDs) + evaluator_ids = Column(JSON, nullable=False) + + # Status + status = Column(String, nullable=False, default=CronJobStatus.ACTIVE.value) + + # Run tracking + next_run_at = Column(DateTime(timezone=True), nullable=True) + last_run_at = Column(DateTime(timezone=True), nullable=True) + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class TTSComparisonStatus(str, enum.Enum): + PENDING = "pending" + GENERATING = "generating" + EVALUATING = "evaluating" + COMPLETED = "completed" + FAILED = "failed" + + +class TTSSampleStatus(str, enum.Enum): + PENDING = "pending" + GENERATING = "generating" + COMPLETED = "completed" + FAILED = "failed" + + +class TTSReportJobStatus(str, enum.Enum): + PENDING = "pending" + PROCESSING = "processing" + COMPLETED = "completed" + FAILED = "failed" + + +class TTSComparison(Base): + """TTS Comparison session for A/B testing voice providers.""" + __tablename__ = "tts_comparisons" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every voice playground comparison belongs to + # a workspace within its org. Children (samples, report jobs, blind + # test shares) inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + simulation_id = Column(String(6), unique=True, index=True, nullable=True) + + name = Column(String(255), nullable=True) + status = Column(String(50), nullable=False, default=TTSComparisonStatus.PENDING.value) + + # 'benchmark' = traditional TTS A/B benchmark (provider-generated audio). + # 'blind_test_only' = standalone blind test built from existing recordings + # / uploads / past TTS samples; no TTS generation happens. + mode = Column(String(32), nullable=False, default="benchmark") + + provider_a = Column(String(100), nullable=True) + model_a = Column(String(100), nullable=True) + voices_a = Column(JSON, nullable=True) + + provider_b = Column(String(100), nullable=True) + model_b = Column(String(100), nullable=True) + voices_b = Column(JSON, nullable=True) + + sample_texts = Column(JSON, nullable=False) + num_runs = Column(Integer, nullable=False, default=1) + + blind_test_results = Column(JSON, nullable=True) + evaluation_summary = Column(JSON, nullable=True) + + eval_stt_provider = Column(String(100), nullable=True) + eval_stt_model = Column(String(100), nullable=True) + + celery_task_id = Column(String, nullable=True, index=True) + error_message = Column(String, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + samples = relationship("TTSSample", back_populates="comparison", cascade="all, delete-orphan") + + +class TTSSample(Base): + """Individual TTS audio sample within a comparison.""" + __tablename__ = "tts_samples" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: mirrors the parent TTSComparison's workspace. + # Denormalized for fast filter-by-workspace listings without a join. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + provider = Column(String(100), nullable=True) + model = Column(String(100), nullable=True) + voice_id = Column(String(255), nullable=True) + voice_name = Column(String(255), nullable=True) + side = Column(String(1), nullable=True) # "A" or "B" + sample_index = Column(Integer, nullable=False) + run_index = Column(Integer, nullable=False, default=0) + + # 'tts' (default, audio is synthesized by a provider), 'recording' (audio + # is reused from a CallImportRow recording), or 'upload' (audio was + # uploaded by the user). Non-tts samples are marked completed up-front + # by the API and skipped by the generation worker. + source_type = Column(String(32), nullable=False, default="tts") + # When source_type == 'recording', references CallImportRow.id (no FK + # constraint to keep cascading deletes simple if a call import is later + # removed; the audio_s3_key is what's actually used). + source_ref_id = Column(UUID(as_uuid=True), nullable=True) + + text = Column(String, nullable=False) + audio_s3_key = Column(String(512), nullable=True) + duration_seconds = Column(Float, nullable=True) + latency_ms = Column(Float, nullable=True) + ttfb_ms = Column(Float, nullable=True) + + evaluation_metrics = Column(JSON, nullable=True) + status = Column(String(50), nullable=False, default=TTSSampleStatus.PENDING.value) + error_message = Column(String, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + comparison = relationship("TTSComparison", back_populates="samples") + + +class TTSReportJob(Base): + """Asynchronous PDF report generation jobs for Voice Playground.""" + __tablename__ = "tts_report_jobs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: mirrors the parent TTSComparison's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) + + status = Column(String(50), nullable=False, default=TTSReportJobStatus.PENDING.value) + format = Column(String(20), nullable=False, default="pdf") + filename = Column(String(255), nullable=True) + s3_key = Column(String(512), nullable=True) + error_message = Column(String, nullable=True) + celery_task_id = Column(String, nullable=True, index=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + comparison = relationship("TTSComparison") + + +class TTSBlindTestShareStatus(str, enum.Enum): + OPEN = "open" + CLOSED = "closed" + + +class TTSBlindTestShare(Base): + """A publicly sharable blind test for a TTSComparison. + + The share_token is the capability: anyone holding it can open the public + form and submit a response. Each comparison has at most one share row. + """ + __tablename__ = "tts_blind_test_shares" + __table_args__ = ( + UniqueConstraint("comparison_id", name="uq_blind_test_shares_comparison"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + comparison_id = Column( + UUID(as_uuid=True), + ForeignKey("tts_comparisons.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: mirrors the parent TTSComparison's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + share_token = Column(String(64), unique=True, nullable=False, index=True) + + title = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + + # Internal notes visible only to the share creator (e.g. which voice + # corresponds to which side, source notes for standalone blind tests). + # Never exposed via the public blind test payload. + creator_notes = Column(Text, nullable=True) + + # JSON list: [{ "key": str, "label": str, "type": "rating"|"comment", "scale": int? }] + custom_metrics = Column(JSON, nullable=False) + + status = Column(String(20), nullable=False, default=TTSBlindTestShareStatus.OPEN.value) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + closed_at = Column(DateTime(timezone=True), nullable=True) + created_by = Column(String, nullable=True) + + comparison = relationship("TTSComparison") + responses = relationship( + "TTSBlindTestResponse", + back_populates="share", + cascade="all, delete-orphan", + ) + + +class TTSBlindTestResponse(Base): + """A single rater's submission against a TTSBlindTestShare.""" + __tablename__ = "tts_blind_test_responses" + __table_args__ = ( + UniqueConstraint("share_id", "rater_email", name="uq_blind_test_response_share_email"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + share_id = Column( + UUID(as_uuid=True), + ForeignKey("tts_blind_test_shares.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + # Workspace isolation: mirrors the parent TTSBlindTestShare's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + rater_name = Column(String(255), nullable=False) + rater_email = Column(String(320), nullable=False, index=True) + + # JSON list keyed by sample_index. Server stores in TRUE A/B orientation + # (already de-flipped from whatever the rater's UI showed): + # [{ + # "sample_index": int, + # "preferred": "A" | "B", + # "ratings_a": { metric_key: number }, + # "ratings_b": { metric_key: number }, + # "comment": str? + # }] + responses = Column(JSON, nullable=False) + + ip = Column(String(64), nullable=True) + user_agent = Column(String(512), nullable=True) + + submitted_at = Column(DateTime(timezone=True), server_default=func.now()) + + share = relationship("TTSBlindTestShare", back_populates="responses") + + +class PromptPartial(Base): + """Prompt Partial - Reusable prompt templates with version history.""" + __tablename__ = "prompt_partials" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every prompt partial belongs to a workspace + # within its org. Versions inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + description = Column(String, nullable=True) + content = Column(Text, nullable=False) + tags = Column(JSON, nullable=True) + current_version = Column(Integer, nullable=False, default=1) + # Cached LLM-generated flowchart for imported production agent prompts. + # Shape: AgentFlowGraph JSON (nodes[], edges[]). + agent_flowchart = Column(JSON, nullable=True) + agent_flowchart_status = Column(String(20), nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + versions = relationship("PromptPartialVersion", back_populates="prompt_partial", cascade="all, delete-orphan", order_by="PromptPartialVersion.version.desc()") + + +class PromptPartialVersion(Base): + """Version history for a prompt partial.""" + __tablename__ = "prompt_partial_versions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + prompt_partial_id = Column(UUID(as_uuid=True), ForeignKey("prompt_partials.id", ondelete="CASCADE"), nullable=False, index=True) + # Workspace isolation: mirrors the parent PromptPartial's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + version = Column(Integer, nullable=False) + content = Column(Text, nullable=False) + change_summary = Column(String, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + created_by = Column(String, nullable=True) + + prompt_partial = relationship("PromptPartial", back_populates="versions") + + __table_args__ = ( + UniqueConstraint('prompt_partial_id', 'version', name='uq_prompt_partial_version'), + ) + + +class CustomTTSVoice(Base): + """Organization-scoped custom TTS voice metadata.""" + __tablename__ = "custom_tts_voices" + __table_args__ = ( + UniqueConstraint("organization_id", "provider", "voice_id", name="uq_custom_tts_voice_org_provider_voice_id"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider = Column(String(100), nullable=False, index=True) + voice_id = Column(String(255), nullable=False) + name = Column(String(255), nullable=False) + gender = Column(String(50), nullable=True) + accent = Column(String(100), nullable=True) + description = Column(Text, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + +class PromptOptimizationRun(Base): + """A single GEPA prompt optimization run for an agent.""" + __tablename__ = "prompt_optimization_runs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every optimization run belongs to a workspace + # within its org. Candidates inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) + evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True) + voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) + + seed_prompt = Column(Text, nullable=False) + best_prompt = Column(Text, nullable=True) + best_score = Column(Float, nullable=True) + + status = Column(String(20), nullable=False, default=PromptOptimizationStatus.PENDING.value) + config = Column(JSON, nullable=True) + reflection_trace = Column(JSON, nullable=True) + metric_history = Column(JSON, nullable=True) + + num_iterations = Column(Integer, nullable=True) + num_metric_calls = Column(Integer, nullable=True) + + celery_task_id = Column(String, nullable=True, index=True) + error_message = Column(Text, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + candidates = relationship("PromptOptimizationCandidate", back_populates="optimization_run", cascade="all, delete-orphan") + + +class PromptOptimizationCandidate(Base): + """A candidate prompt generated during an optimization run.""" + __tablename__ = "prompt_optimization_candidates" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + optimization_run_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_runs.id", ondelete="CASCADE"), nullable=False, index=True) + # Workspace isolation: mirrors the parent PromptOptimizationRun's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + prompt_text = Column(Text, nullable=False) + score = Column(Float, nullable=True) + metric_breakdown = Column(JSON, nullable=True) + reflection_summary = Column(Text, nullable=True) + + parent_candidate_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_candidates.id"), nullable=True) + + is_accepted = Column(Boolean, nullable=False, default=False) + pushed_to_provider_at = Column(DateTime(timezone=True), nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + optimization_run = relationship("PromptOptimizationRun", back_populates="candidates") + + +class TelephonyIntegration(Base): + """Per-organization telephony provider credentials and configuration. + + Multiple rows per (organization_id, provider) are allowed so that an + organization can keep several Plivo / Exotel accounts side-by-side. + A partial unique index in migration 028 enforces at most one row with + is_default = TRUE per (org, provider); resolution falls back to that + default row when the caller does not pin a specific credential. + """ + + __tablename__ = "telephony_integrations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider = Column(String(50), nullable=False, default="plivo") + name = Column(String(255), nullable=True) # Optional friendly name to disambiguate multiple credentials + + auth_id = Column(String(255), nullable=False) + auth_token = Column(String(512), nullable=False) + + verify_app_uuid = Column(String(255), nullable=True) + voice_app_id = Column(String(255), nullable=True) + sip_domain = Column(String(255), nullable=True) + masking_config = Column(JSON, nullable=True) + + is_active = Column(Boolean, default=True, nullable=False) + is_default = Column(Boolean, default=False, nullable=False) + last_tested_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyPhoneNumber(Base): + """Inventory of telephony phone numbers owned by an organization.""" + + __tablename__ = "telephony_phone_numbers" + __table_args__ = ( + UniqueConstraint("organization_id", "phone_number", name="uq_telephony_number_org_phone"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + telephony_integration_id = Column( + UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=True, index=True + ) + + phone_number = Column(String(20), nullable=False, index=True) + country_iso2 = Column(String(2), nullable=True) + region = Column(String(100), nullable=True) + number_type = Column(String(20), nullable=True) + capabilities = Column(JSON, nullable=True) + provider_app_id = Column(String(255), nullable=True) + + is_masking_pool = Column(Boolean, default=False, nullable=False) + inbound_enabled = Column(Boolean, default=True, nullable=False) + outbound_enabled = Column(Boolean, default=True, nullable=False) + source = Column(String(20), nullable=False, default="imported") + agent_id = Column( + UUID(as_uuid=True), + ForeignKey( + "agents.id", + ondelete="SET NULL", + use_alter=True, + name="fk_telephony_phone_numbers_agent_id", + ), + nullable=True, + index=True, + ) + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyDialTarget(Base): + """Org-scoped saved destination numbers for outbound test calls.""" + + __tablename__ = "telephony_dial_targets" + __table_args__ = ( + UniqueConstraint("organization_id", "phone_number", name="uq_telephony_dial_target_org_phone"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + phone_number = Column(String(20), nullable=False, index=True) + label = Column(String(255), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyVerifySession(Base): + """Tracks voice OTP verification sessions via telephony provider.""" + + __tablename__ = "telephony_verify_sessions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider_session_uuid = Column(String(255), nullable=False, unique=True, index=True) + recipient_number = Column(String(20), nullable=False) + channel = Column(String(10), nullable=False, default="voice") + status = Column(String(20), nullable=False, default="pending") + initiated_by = Column(String(255), nullable=True) + verify_app_uuid = Column(String(255), nullable=True) + verified_at = Column(DateTime(timezone=True), nullable=True) + expires_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyMaskedSession(Base): + """Number-masking session between two parties through a middle number.""" + + __tablename__ = "telephony_masked_sessions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + telephony_integration_id = Column(UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=False) + masked_number_id = Column( + UUID(as_uuid=True), ForeignKey("telephony_phone_numbers.id"), nullable=False, index=True + ) + masked_number = Column(String(20), nullable=False) + party_a_number = Column(String(20), nullable=False) + party_b_number = Column(String(20), nullable=False) + status = Column(String(20), nullable=False, default="active") + expires_at = Column(DateTime(timezone=True), nullable=True) + ended_at = Column(DateTime(timezone=True), nullable=True) + session_metadata = Column("metadata", JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class CallImportSchema(Base): + """Reusable Input Parameter schema for the call-uploads flow. + + A schema is workspace-scoped: users define a named bundle of typed + Input Parameters once (e.g. "Standard Voice QA" with conversation_id + + recording_url + transcript + agent_name) and then map those parameters + to CSV/Excel headers each time they upload a new batch. + + Every schema MUST contain exactly one parameter with + ``type='conversation_id'`` and ``is_required=True`` - that's the + mandatory identity field every imported row needs. The invariant is + enforced in app code on create/update (no DB-level CHECK because the + parent + children are written across two tables in one transaction). + """ + + __tablename__ = "call_import_schemas" + __table_args__ = ( + # Case-insensitive uniqueness is enforced via the matching partial + # index on ``LOWER(name)`` in the migration; this constraint here + # would be case-sensitive and is intentionally omitted to avoid + # confusing the user. + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + created_by_user_id = Column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + parameters = relationship( + "CallImportSchemaParameter", + back_populates="schema", + cascade="all, delete-orphan", + order_by="CallImportSchemaParameter.ordering", + ) + + +class CallImportSchemaParameter(Base): + """A single typed parameter inside a :class:`CallImportSchema`. + + ``type`` is one of the strings tracked by + :data:`app.models.enums.CallImportParameterType`. ``conversation_id`` + is reserved for the mandatory identity parameter every schema must + contain. + """ + + __tablename__ = "call_import_schema_parameters" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + schema_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_schemas.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + type = Column(String(32), nullable=False) + description = Column(Text, nullable=True) + is_required = Column(Boolean, nullable=False, default=False) + # Stable ordering so the UI renders parameters in the order the + # schema author defined them (matters when conversation_id is pinned + # first and the user re-orders the rest). + ordering = Column(Integer, nullable=False, default=0) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + schema = relationship("CallImportSchema", back_populates="parameters") + + +class CallImport(Base): + """Batch record for a CSV-driven call import job.""" + + __tablename__ = "call_imports" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every imported batch belongs to a workspace + # within its org. The /upload endpoint stamps it from the active + # workspace header (or the org's Default if absent). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + created_by_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + last_updated_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + + # Telephony provider key (e.g. ``'exotel'``, ``'plivo'``). In the + # legacy one-shot ``POST /upload`` endpoint this is supplied with the + # file; in the three-stage flow (UPLOAD -> MAP -> IMPORT) the value + # isn't known until the IMPORT stage, so the column is nullable for + # ``uploaded`` / ``mapped`` batches. + provider = Column(String(50), nullable=True, default="exotel") + # Pin a specific telephony credential for this batch so the worker + # downloads recordings using *that* row instead of the org default. + # NULL preserves legacy behavior (resolve by provider + default). + telephony_integration_id = Column( + UUID(as_uuid=True), + ForeignKey("telephony_integrations.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + original_filename = Column(String(512), nullable=True) + # When the source file was a multi-sheet Excel workbook, this records + # the worksheet the rows came from (one batch per sheet). NULL for CSV + # uploads since CSV has no sheet concept. + sheet_name = Column(String(255), nullable=True) + + # --- Source-file staging (UPLOAD stage) --------------------------- + # The raw CSV / Excel file is stored in S3 between stages so the + # user can come back later to MAP and IMPORT without re-uploading. + # ``source_s3_key`` is NULL on legacy batches that were imported via + # the one-shot endpoint (those batches stay read-only post-import). + source_s3_key = Column(Text, nullable=True) + source_format = Column(String(16), nullable=True) + source_size_bytes = Column(BigInteger, nullable=True) + source_content_type = Column(String(255), nullable=True) + + # Snapshot of the file's sheets + headers captured at UPLOAD time + # so the MAP UI doesn't need to re-fetch the source bytes from S3. + # Shape: ``[{"name": str, "headers": [str, ...], "row_count": int}, ...]``. + available_sheets = Column(JSON, nullable=True) + + # User's explicit "drop these columns" decision captured at MAP + # time. Was validation-only and ephemeral in the legacy flow; now + # persisted so the IMPORT stage can re-parse the file with the same + # mapping/skip intent. + skipped_columns = Column(JSON, nullable=False, default=list) + # Rows skipped at parse time (missing/invalid conversation_id or URL). + # Shape: ``[{"source_row": int, "reason": str, "message": str}, ...]``. + source_row_skips = Column(JSON, nullable=False, default=list) + + # Free-text high-level segregation label. Powers the "Dataset" filter + # at the top of the imports page; multiple imports can share a value. + dataset = Column(String(255), nullable=True, index=True) + + # Reusable Input Parameter schema this batch was uploaded against. + # NULL on legacy batches uploaded before the schema-driven flow + # shipped; those still render via ``column_mapping`` + ``extra_columns`` + # + ``custom_column_mapping`` below. + schema_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_schemas.id", ondelete="RESTRICT"), + nullable=True, + index=True, + ) + # New schema-driven mapping: ``{schema_parameter_name: csv_header}``. + # Populated for new uploads; empty dict on legacy batches. + parameter_mapping = Column(JSON, nullable=False, default=dict) + + # Legacy free-form mapping (pre-schema-flow). Kept on the model so + # batches that were uploaded before the schema feature shipped still + # render correctly on the detail page; new uploads stop writing here. + # Keys: external_call_id (required), transcript, recording_url. + # (DB column ``external_call_id`` is now ``conversation_id``; this + # JSON key stays as-is for historical batches.) + # Values: original CSV header strings (preserve user casing for export). + column_mapping = Column(JSON, nullable=False, default=dict) + # Ordered list of additional CSV header strings the uploader wants + # preserved verbatim into the evaluation export CSV. + extra_columns = Column(JSON, nullable=False, default=list) + # User-defined ``{custom_field_name: csv_header}`` mappings on top of + # the three system fields above. Cells from the mapped CSV columns are + # preserved per row (keyed by the CSV header in ``raw_columns``) and + # surface in the evaluation export under the uploader-chosen name. + custom_column_mapping = Column(JSON, nullable=False, default=dict) + + total_rows = Column(Integer, nullable=False, default=0) + completed_rows = Column(Integer, nullable=False, default=0) + failed_rows = Column(Integer, nullable=False, default=0) + + status = Column( + Enum(CallImportStatus, values_callable=get_enum_values), + nullable=False, + default=CallImportStatus.PENDING, + index=True, + ) + error_message = Column(Text, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + rows = relationship( + "CallImportRow", + back_populates="call_import", + cascade="all, delete-orphan", + order_by="CallImportRow.row_index", + ) + tags = relationship( + "CallImportTag", + secondary="call_import_tag_assignments", + backref="call_imports", + lazy="selectin", + ) + evaluations = relationship( + "CallImportEvaluation", + back_populates="call_import", + cascade="all, delete-orphan", + ) + + +class CallImportShardSlice(Base): + """Registry row: which shard stores a slice of rows for an import.""" + + __tablename__ = "call_import_shard_slices" + + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + primary_key=True, + ) + slice_id = Column(Integer, primary_key=True) + shard_id = Column(String(64), nullable=False, index=True) + row_index_min = Column(Integer, nullable=False) + row_index_max = Column(Integer, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class CallImportRow(Base): + """A single row within a CallImport batch (one CSV line / one external call).""" + + __tablename__ = "call_import_rows" + __table_args__ = ( + UniqueConstraint("call_import_id", "row_index", name="uq_call_import_row_index"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + row_index = Column(Integer, nullable=False) + # Was historically named ``external_call_id``; renamed to + # ``conversation_id`` so the new schema-driven upload flow can refer + # to it by a single canonical name across the schema definition, + # exports, and downstream evaluation tables. + conversation_id = Column(String(255), nullable=False, index=True) + # Supplied via CSV for Exotel credentialed imports (required per row). + # Nullable in the schema for legacy rows imported before recording_url + # was mandatory on every Exotel upload. + recording_url = Column(Text, nullable=True) + # Date-only call recording date supplied by the import schema. Used + # for historical report comparisons without timezone/time ambiguity. + recording_date = Column(Date, nullable=True, index=True) + # The "production" transcript: the value supplied via the CSV + # upload mapping. Never overwritten by the diarisation worker — + # the worker writes its output into ``diarised_transcript`` so + # the user keeps both versions side by side. + transcript = Column(Text, nullable=True) + # Snapshot of the original CSV row keyed by the user's headers so the + # evaluation export can reproduce every column the uploader supplied + # (mapped + extra). NULL on legacy rows imported before this column. + raw_columns = Column(JSON, nullable=True) + + # Where the value in ``transcript`` came from. ``csv`` = supplied via + # the upload mapping, ``edited`` = manually changed in the UI. NULL + # on rows that have never had a production transcript. + # (Worker-produced transcripts now live in ``diarised_transcript`` + # and are tracked via ``diarised_transcript_*`` metadata below.) + transcript_source = Column(String(20), nullable=True) + # Provider/model recorded by the (legacy) post-hoc transcription + # worker. New worker runs leave these NULL and write into the + # ``diarised_transcript_*`` columns instead; kept on the model for + # backwards compatibility with pre-split rows that still carry the + # original transcription metadata here. + transcript_provider = Column(String(50), nullable=True) + transcript_model = Column(String(100), nullable=True) + # Lifecycle status for the legacy transcription workflow itself, + # independent of the row's recording-fetch ``status``. ``idle`` = + # no transcribe task has touched this column. New diarisation runs + # update ``diarised_transcript_status`` instead. + transcript_status = Column( + String(20), + nullable=False, + default="idle", + ) + transcript_error = Column(Text, nullable=True) + transcribed_at = Column(DateTime(timezone=True), nullable=True) + + # The "diarised" transcript: produced by the post-hoc + # transcription/diarisation worker. Stored separately so a manual + # diarisation run never clobbers the production transcript above. + # Evaluations can be configured to score against either column + # (see ``CallImportEvaluation.transcript_source``). + diarised_transcript = Column(Text, nullable=True) + # Provider/model the diarisation worker used. Surfaced in the UI + # as "Diarised via deepgram/nova-2" next to the diarised + # transcript section. + diarised_transcript_provider = Column(String(50), nullable=True) + diarised_transcript_model = Column(String(100), nullable=True) + # Lifecycle status for the diarisation workflow. + # ``idle`` = no diarisation task has run; ``pending``/``running`` = + # a Celery task is queued or in flight; ``completed``/``failed`` = + # terminal. Independent of ``transcript_status`` so the two + # transcripts can be in different lifecycle states. + diarised_transcript_status = Column( + String(20), + nullable=False, + default="idle", + server_default="idle", + ) + diarised_transcript_error = Column(Text, nullable=True) + diarised_at = Column(DateTime(timezone=True), nullable=True) + + # Structured speaker turns produced by the diarisation worker — + # ``[{ "speaker": "agent"|"user"|"speaker_3", "text": "...", + # "start": float, "end": float, "raw_speaker": "Speaker 1" }, ...]`` + # The plain-text ``diarised_transcript`` above is a rendered view + # of this list (``: `` per line). When the worker + # cannot recover structured turns (no pyannote token / single- + # speaker recording / provider that doesn't surface segments) this + # column stays NULL and the plain-text path is still populated. + diarised_segments = Column(JSON, nullable=True) + # When True the ``agent`` <-> ``user`` mapping inside + # ``diarised_segments`` is inverted at render / export time. The + # worker writes the canonical mapping using the "first speaker is + # the agent" heuristic; reviewers can flip the toggle from the row + # detail panel without re-running diarisation. + diarised_speaker_swap = Column( + Boolean, + nullable=False, + default=False, + server_default="false", + ) + # LLM that turned the STT plain-text output into structured + # ``diarised_segments``. The legacy diarisation worker used + # pyannote and left these NULL; the current path always runs an + # LLM with the operator-supplied (or default) ``diarised_prompt`` + # below, and records exactly which model + prompt produced each + # row so reviewers can reproduce a specific run. + diarised_llm_provider = Column(String(50), nullable=True) + diarised_llm_model = Column(String(100), nullable=True) + diarised_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) + diarised_prompt = Column(Text, nullable=True) + # Which diarisation pipeline produced this row's turns. + # * ``"stt_llm"`` (default) — two-stage: STT then LLM diariser. + # ``diarised_transcript_provider``/``_model`` describe the STT + # side; ``diarised_llm_provider``/``_model`` the LLM side. + # * ``"llm_only"`` — single-stage: audio fed straight to a + # multimodal LLM. ``diarised_transcript_provider`` is stamped + # with the sentinel ``"llm_only"``; the real model is on + # ``diarised_llm_*``. + # Persisting it on the row (not just the run) lets the row detail + # panel render the right "Diarised via …" label even for ad-hoc + # standalone transcribes (no parent evaluation). + transcribe_mode = Column( + String(20), + nullable=False, + default="stt_llm", + server_default="stt_llm", + ) + + status = Column( + Enum(CallImportRowStatus, values_callable=get_enum_values), + nullable=False, + default=CallImportRowStatus.PENDING, + index=True, + ) + + recording_s3_key = Column(String(1024), nullable=True) + recording_content_type = Column(String(128), nullable=True) + recording_size_bytes = Column(Integer, nullable=True) + + error_message = Column(Text, nullable=True) + attempts = Column(Integer, nullable=False, default=0) + celery_task_id = Column(String(255), nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + call_import = relationship("CallImport", back_populates="rows") + + +@event.listens_for(CallImportRow, "before_insert") +def _call_import_row_fill_workspace_id(_mapper, connection, target): + """Denormalize workspace_id from the parent import when omitted.""" + if target.workspace_id is not None or target.call_import_id is None: + return + workspace_id = connection.execute( + select(CallImport.workspace_id).where( + CallImport.id == target.call_import_id + ) + ).scalar_one_or_none() + if workspace_id is not None: + target.workspace_id = workspace_id + + +class CallImportTag(Base): + """User-defined tag that can be attached to one or more call imports. + + Tags coexist with the free-text ``CallImport.dataset`` column: dataset + is the primary high-level segregation, tags are an optional secondary + classification (an import can have many tags). + """ + + __tablename__ = "call_import_tags" + __table_args__ = ( + UniqueConstraint("organization_id", "name", name="uq_call_import_tag_org_name"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + name = Column(String(255), nullable=False) + color = Column(String(32), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +class CallImportTagAssignment(Base): + """Many-to-many join table between CallImport and CallImportTag.""" + + __tablename__ = "call_import_tag_assignments" + + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + primary_key=True, + ) + tag_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_tags.id", ondelete="CASCADE"), + primary_key=True, + index=True, + ) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class CallImportEvaluation(Base): + """Parent record for an evaluation run over a CallImport batch. + + A user picks a subset of org ``Metric`` rows and triggers an evaluation; + we fan out one ``CallImportEvaluationRow`` per source row and roll up + counters as workers finish. Status mirrors ``CallImportStatus`` plus a + ``RUNNING`` value so the UI can distinguish "queued" from "in flight". + """ + + __tablename__ = "call_import_evaluations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id"), + nullable=False, + index=True, + ) + # Workspace isolation: mirrors the parent CallImport's workspace. + # Denormalized for fast filter-by-workspace listings without a join. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + created_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id"), nullable=True + ) + last_updated_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + + # Optional user-supplied label for this run. Lets the UI surface + # something more meaningful than the UUID prefix (e.g. "March QA pass"). + name = Column(String(255), nullable=True) + + # JSON list of Metric UUID strings selected for this run. Stored as text + # in JSON so we don't have to deal with PG arrays of UUIDs / cascade + # delete policies when metrics are removed; the loader filters for + # still-existing org metrics at run time. + selected_metric_ids = Column(JSON, nullable=False, default=list) + # Hierarchy grouping snapshot: ``{parent_id_str: [child_id_str, ...]}``. + # Captures which children belong to which parent for THIS run so the UI + # / aggregator can reconstruct the tree even when the user selected + # only a subset of children, or after metrics are deleted / renamed. + # NULL on legacy rows means "no hierarchy" → fall back to flat + # ``selected_metric_ids`` semantics. + selected_metric_groups = Column(JSON, nullable=True) + # User-driven merges of LLM-discovered candidate sub-labels for + # ``allow_discovery`` parents. Shape: + # ``{"": {"": "", ...}}``. + # Populated via ``POST .../discovered-labels/merge``; consulted by + # the discovered-labels aggregator, the flow graph builder, and the + # worker so that rows finishing AFTER a merge cannot reintroduce + # the merged-away slug. Empty dict on fresh rows. + discovered_label_aliases = Column( + JSON, nullable=False, default=dict, server_default="{}" + ) + + # Per-run opt-in for top-level metric discovery. When True, the LLM + # is asked to propose brand-new top-level metrics (boolean / rating / + # category) observed in the transcripts in addition to scoring the + # ``selected_metric_ids`` for the row. Candidates surface in a + # "Discovered metrics" panel on the evaluation's Flow tab and can + # be promoted into real standalone ``Metric`` rows via + # ``POST /metrics/from-discovered``. Defaults to False so existing + # evaluation creation payloads keep their previous behaviour. + discover_new_metrics = Column( + Boolean, nullable=False, default=False, server_default="false" + ) + # Flat slug-to-slug redirect map for user merges + tombstones of + # discovered top-level metric candidates. Mirrors + # ``discovered_label_aliases`` but is NOT nested per parent — + # top-level metric discovery is not scoped to any parent. Shape:: + # + # {"": "", ...} + # + # An empty-string value tombstones the slug so workers finishing + # later can't re-introduce it. + discovered_metric_aliases = Column( + JSON, nullable=False, default=dict, server_default="{}" + ) + + # Run-level LLM config picked from the Run Evaluation modal. NULL on + # legacy rows means "use the historical OpenAI/gpt-4o default" — the + # worker checks for this and falls back accordingly. ``llm_credential_id`` + # pins a specific AIProvider row when the org has multiple credentials + # for the same provider. + llm_provider = Column(String(50), nullable=True) + llm_model = Column(String(100), nullable=True) + llm_credential_id = Column( + UUID(as_uuid=True), + ForeignKey("aiproviders.id", ondelete="SET NULL"), + nullable=True, + ) + llm_config = Column(JSON, nullable=True) + # Optional per-metric LLM override: + # ``{"": {"provider": "...", "model": "...", "credential_id": "..."}}``. + # Each entry overrides the run-level default for that metric only; + # missing keys = use run-level default. Stored as JSON so the UI can + # round-trip arbitrary {provider, model} pairs without migrations. + metric_llm_overrides = Column(JSON, nullable=True) + + # When ``auto_transcribe`` was set on the create payload, record the + # STT provider/model used so the UI can show "Auto-transcribed via + # deepgram/nova-2" on the evaluation header. ``stt_credential_id`` is + # untyped (no FK) because STT keys may live in either ``aiproviders`` + # (OpenAI) or ``integrations`` (Deepgram, ElevenLabs) — the + # transcription service handles the lookup. + stt_provider = Column(String(50), nullable=True) + stt_model = Column(String(100), nullable=True) + stt_credential_id = Column(UUID(as_uuid=True), nullable=True) + + # Run-level LLM diariser config. Used when the create-run / + # retry-run paths chain a ``transcribe_call_import_row_task`` + # because the row is missing a diarised transcript. Persisted on + # the run so a retry uses the same diariser the original create + # call picked (unless the retry payload explicitly overrides). + diarisation_llm_provider = Column(String(50), nullable=True) + diarisation_llm_model = Column(String(100), nullable=True) + diarisation_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) + diarisation_prompt = Column(Text, nullable=True) + # Mode the run was *created* with for its auto-transcribe step. + # Retry chains read this to decide whether to enqueue an STT+LLM + # transcribe or a single-stage multimodal LLM transcribe — without + # it we'd have to infer the mode from "stt_provider is NULL", which + # would silently break legacy rows that simply never configured + # auto-transcribe. See migration 041 for the column DDL. + transcribe_mode = Column( + String(20), + nullable=False, + default="stt_llm", + server_default="stt_llm", + ) + + # Which of the two transcripts on each ``CallImportRow`` this run + # scored against. ``'production'`` reads ``CallImportRow.transcript`` + # (the CSV-supplied value); ``'diarised'`` reads + # ``CallImportRow.diarised_transcript`` (the worker output). When + # the user ticks both checkboxes in the Run Evaluation modal we + # create two ``CallImportEvaluation`` rows — one per source — so + # the two scorings can be compared side-by-side. Defaults to + # ``'production'`` so legacy runs (which always read the single + # historical ``transcript`` column) keep their semantics. + transcript_source = Column( + String(20), + nullable=False, + default="production", + server_default="production", + ) + + # Cached LLM-generated TLDR rendered above the Visualizations charts. + # Populated lazily by ``POST /evaluations/{eval_id}/insights`` so we + # never auto-burn LLM tokens on page load. Shape:: + # {"narrative": str, "patterns": [str, ...], + # "generated_at": iso8601, "generated_at_completed_rows": int, + # "provider": str, "model": str} + # NULL on rows that have never been summarised. + tldr_summary = Column(JSON, nullable=True) + + # Cached LLM-generated user insights for External Audit PDF section 03. + # Populated by a background Celery job triggered alongside TLDR generation. + # Shape: EvaluationUserInsightsState JSON (status, insights[], progress, …). + user_insights = Column(JSON, nullable=True) + + # Cached per-metric failure clustering for internal diagnostics PDF/UI. + # Shape: EvaluationMetricClustersState JSON (status, groups[], …). + metric_clusters = Column(JSON, nullable=True) + + # Cached LLM-generated prompt improvement suggestions keyed to an + # imported agent (PromptPartial tagged __imported_agent__). + # Shape: EvaluationPromptImprovementsState JSON. + prompt_improvements = Column(JSON, nullable=True) + + # Cached LLM explanations for week-over-week metric deltas keyed by + # baseline evaluation id + completed row counts. + period_delta_explanations = Column(JSON, nullable=True) + + status = Column(String(20), nullable=False, default="pending", index=True) + + total_rows = Column(Integer, nullable=False, default=0) + completed_rows = Column(Integer, nullable=False, default=0) + failed_rows = Column(Integer, nullable=False, default=0) + # Flexprice pass-level delta billing watermark: rows already emitted + # on ``call_import.evaluation_completed`` for this evaluation run. + billed_completed_rows = Column( + Integer, nullable=False, default=0, server_default="0" + ) + error_message = Column(Text, nullable=True) + celery_group_id = Column(String(255), nullable=True) + + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + call_import = relationship("CallImport", back_populates="evaluations") + row_results = relationship( + "CallImportEvaluationRow", + back_populates="evaluation", + cascade="all, delete-orphan", + ) + + +class CallImportEvaluationRow(Base): + """Per-source-row scoring output for a CallImportEvaluation parent.""" + + __tablename__ = "call_import_evaluation_rows" + __table_args__ = ( + UniqueConstraint( + "evaluation_id", "call_import_row_id", name="uq_call_import_evaluation_row" + ), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_import_row_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_rows.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + status = Column(String(20), nullable=False, default="pending", index=True) + # Same shape as EvaluatorResult.metric_scores: {metric_id_str: {value, type, metric_name, ...}} + metric_scores = Column(JSON, nullable=False, default=dict) + error_message = Column(Text, nullable=True) + celery_task_id = Column(String(255), nullable=True) + + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + evaluation = relationship("CallImportEvaluation", back_populates="row_results") + source_row = relationship("CallImportRow") + + +@event.listens_for(CallImportEvaluationRow, "before_insert") +def _call_import_evaluation_row_fill_workspace_id(_mapper, connection, target): + """Denormalize workspace_id from the parent evaluation when omitted.""" + if target.workspace_id is not None or target.evaluation_id is None: + return + workspace_id = connection.execute( + select(CallImportEvaluation.workspace_id).where( + CallImportEvaluation.id == target.evaluation_id + ) + ).scalar_one_or_none() + if workspace_id is not None: + target.workspace_id = workspace_id + + +class CallImportEvaluationReportSnapshot(Base): + """Persisted PDF-report aggregate used for period-over-period deltas.""" + + __tablename__ = "call_import_evaluation_report_snapshots" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + period_label = Column(String(64), nullable=True, index=True) + period_start = Column(Date, nullable=True, index=True) + period_end = Column(Date, nullable=True, index=True) + report_config = Column(JSON, nullable=False, default=dict, server_default="{}") + selected_metric_ids = Column(JSON, nullable=False, default=list, server_default="[]") + metric_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") + insight_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") + narrative = Column(JSON, nullable=True) + total_calls = Column(Integer, nullable=False, default=0) + selected_metric_count = Column(Integer, nullable=False, default=0) + total_metric_count = Column(Integer, nullable=False, default=0) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +# --------------------------------------------------------------------------- +# Judge Alignment (AlignEval-style hybrid integration) +# +# Three tables back the "Judge Alignment" surface: +# - JudgeDataset: a labeled dataset materialised from one of three sources +# (voice transcripts, existing Metric/Evaluator outputs, +# or a generic CSV upload). Holds the dataset's source +# config + which fields play the role of input/output. +# - JudgeSample: one row in a dataset (input/output pair plus an +# optional binary pass/fail human label). +# - JudgeRun: a single run of an LLM-judge (existing Evaluator) over +# a subset of samples, with computed alignment metrics +# (precision/recall/F1/Cohen's kappa) and per-sample +# predictions. Optionally links to a GEPA optimization +# run when the user kicks off prompt tuning from a +# dataset. +# --------------------------------------------------------------------------- + + +class JudgeDataset(Base): + """Container for binary-labeled samples used to calibrate an LLM-judge.""" + + __tablename__ = "judge_datasets" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + # Workspace isolation: every judge dataset belongs to a workspace + # within its org. Samples and runs inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + name = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + + # One of: "transcript", "metric_output", "csv" + source_type = Column(String(32), nullable=False, index=True) + # Source-specific config. Examples: + # transcript: {"transcription_ids": [...]} or {"agent_id": "..."} + # metric_output: {"metric_id": "...", "evaluator_id": "..."} + # csv: {"s3_key": "...", "filename": "..."} + source_config = Column(JSON, nullable=False, default=dict) + + # Field roles - which textual content is "input" vs "output" for the judge. + # For voice transcripts both default to the transcript text but can be + # tightened (e.g. agent-only turns vs full conversation). + input_field = Column(String(64), nullable=False, default="input") + output_field = Column(String(64), nullable=False, default="output") + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + samples = relationship( + "JudgeSample", + back_populates="dataset", + cascade="all, delete-orphan", + order_by="JudgeSample.created_at", + ) + runs = relationship( + "JudgeRun", + back_populates="dataset", + cascade="all, delete-orphan", + order_by="JudgeRun.created_at.desc()", + ) + + +class JudgeSample(Base): + """One labelable input/output pair within a JudgeDataset.""" + + __tablename__ = "judge_samples" + __table_args__ = ( + UniqueConstraint("dataset_id", "external_id", name="uq_judge_samples_dataset_external"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + dataset_id = Column( + UUID(as_uuid=True), + ForeignKey("judge_datasets.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + # Workspace isolation: mirrors the parent JudgeDataset's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Stable identifier within the source (e.g. transcription UUID, CSV row id). + # Used to dedupe re-imports and link back to the originating record. + external_id = Column(String(128), nullable=True, index=True) + + input_text = Column(Text, nullable=False) + output_text = Column(Text, nullable=False) + + # Binary human label: "pass" | "fail" | null (unlabeled). + # Stored as string (rather than enum) so it stays trivially extendable. + label = Column(String(16), nullable=True, index=True) + labeled_by = Column(String(255), nullable=True) + labeled_at = Column(DateTime(timezone=True), nullable=True) + + # Source-specific context (e.g. agent_id, original metric value, csv row). + extra = Column(JSON, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + dataset = relationship("JudgeDataset", back_populates="samples") + + +class JudgeRun(Base): + """One execution of an LLM-judge against a JudgeDataset, with alignment metrics.""" + + __tablename__ = "judge_runs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + dataset_id = Column( + UUID(as_uuid=True), + ForeignKey("judge_datasets.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + # Workspace isolation: mirrors the parent JudgeDataset's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Reuses the existing Evaluator row (its custom_prompt + llm_provider + llm_model + # define the judge under test). Nullable so a run may target an inline prompt + # in the future without inflating the Evaluator table. + evaluator_id = Column( + UUID(as_uuid=True), ForeignKey("evaluators.id", ondelete="SET NULL"), nullable=True, index=True + ) + + # Which subset was scored: "all" | "dev" | "test" + split = Column(String(16), nullable=False, default="all") + + # Snapshot of the model used (so a later Evaluator edit doesn't rewrite history). + llm_provider = Column(String(64), nullable=True) + llm_model = Column(String(128), nullable=True) + + # Computed alignment metrics: + # {"precision": float, "recall": float, "f1": float, "kappa": float, + # "tp": int, "fp": int, "tn": int, "fn": int, "n": int} + metrics = Column(JSON, nullable=True) + + # Per-sample predictions, keyed by sample_id (UUID string): + # {sample_id: {"prediction": "pass"|"fail", "explanation": str, "raw": str}} + predictions = Column(JSON, nullable=True) + + # Run lifecycle. + status = Column(String(20), nullable=False, default="pending", index=True) + error_message = Column(Text, nullable=True) + celery_task_id = Column(String, nullable=True, index=True) + + # Optional link to a GEPA optimization run kicked off from this dataset. + gepa_optimization_id = Column( + UUID(as_uuid=True), + ForeignKey("prompt_optimization_runs.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + dataset = relationship("JudgeDataset", back_populates="runs") diff --git a/app/models/schemas.py b/app/models/schemas.py index 78dd14f9..de5c203e 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -3168,6 +3168,8 @@ class CallImportResponse(BaseModel): error_message: Optional[str] = None created_at: datetime updated_at: datetime + created_by_email: Optional[str] = None + last_updated_by_email: Optional[str] = None model_config = ConfigDict(from_attributes=True) @@ -3981,6 +3983,8 @@ class CallImportEvaluationResponse(BaseModel): finished_at: Optional[datetime] = None created_at: datetime updated_at: datetime + created_by_email: Optional[str] = None + last_updated_by_email: Optional[str] = None # Cached LLM-generated TLDR for the Visualizations tab. Lazily # populated by ``POST /evaluations/{eval_id}/insights``; ``None`` # for runs the user has not summarised yet. ``is_stale`` on the diff --git a/app/services/call_imports/audit.py b/app/services/call_imports/audit.py new file mode 100644 index 00000000..6479ef18 --- /dev/null +++ b/app/services/call_imports/audit.py @@ -0,0 +1,99 @@ +"""Actor stamping and email resolution for call import audit fields.""" + +from __future__ import annotations + +from typing import Dict, Iterable, Optional, Set, Tuple +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.core.auth.principal import Principal +from app.models.database import CallImport, CallImportEvaluation, User + + +def stamp_call_import_actor( + call_import: CallImport, + principal: Principal, + *, + creating: bool = False, +) -> None: + if creating and principal.user_id is not None: + call_import.created_by_user_id = principal.user_id + if principal.user_id is not None: + call_import.last_updated_by_user_id = principal.user_id + + +def stamp_evaluation_actor( + evaluation: CallImportEvaluation, + principal: Principal, + *, + creating: bool = False, +) -> None: + if creating and principal.user_id is not None: + evaluation.created_by_user_id = principal.user_id + if principal.user_id is not None: + evaluation.last_updated_by_user_id = principal.user_id + + +def user_ids_from_call_imports(imports: Iterable[CallImport]) -> Set[UUID]: + ids: Set[UUID] = set() + for row in imports: + if row.created_by_user_id is not None: + ids.add(row.created_by_user_id) + if row.last_updated_by_user_id is not None: + ids.add(row.last_updated_by_user_id) + return ids + + +def user_ids_from_evaluations( + evaluations: Iterable[CallImportEvaluation], +) -> Set[UUID]: + ids: Set[UUID] = set() + for row in evaluations: + if row.created_by_user_id is not None: + ids.add(row.created_by_user_id) + if row.last_updated_by_user_id is not None: + ids.add(row.last_updated_by_user_id) + return ids + + +def emails_for_user_ids(db: Session, user_ids: Iterable[UUID]) -> Dict[UUID, str]: + unique = {uid for uid in user_ids if uid is not None} + if not unique: + return {} + rows = db.query(User.id, User.email).filter(User.id.in_(unique)).all() + return {row.id: row.email for row in rows if row.email} + + +def actor_emails_for_call_import( + call_import: CallImport, + email_by_id: Dict[UUID, str], +) -> Tuple[Optional[str], Optional[str]]: + created = ( + email_by_id.get(call_import.created_by_user_id) + if call_import.created_by_user_id + else None + ) + updated = ( + email_by_id.get(call_import.last_updated_by_user_id) + if call_import.last_updated_by_user_id + else None + ) + return created, updated + + +def actor_emails_for_evaluation( + evaluation: CallImportEvaluation, + email_by_id: Dict[UUID, str], +) -> Tuple[Optional[str], Optional[str]]: + created = ( + email_by_id.get(evaluation.created_by_user_id) + if evaluation.created_by_user_id + else None + ) + updated = ( + email_by_id.get(evaluation.last_updated_by_user_id) + if evaluation.last_updated_by_user_id + else None + ) + return created, updated diff --git a/env.example b/env.example index ff2f226a..c434c850 100644 --- a/env.example +++ b/env.example @@ -41,7 +41,7 @@ RATE_LIMIT_PER_MINUTE=60 # Comma-separated or JSON list. See config.yml.example for full docs. # OSS default: api_key,local_password # Enterprise SSO: api_key,external_oidc (Okta / Azure AD / Google / Cognito / Auth0 / ...) -AUTH_PROVIDERS=api_key,local_password +AUTH_PROVIDERS=["api_key","local_password"] # Local password (HS256 Bearer tokens signed with SECRET_KEY) AUTH_LOCAL_TOKEN_TTL_MINUTES=15 diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index 91fbc9c5..39c0fed7 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -1,2132 +1,2136 @@ -// API Types matching the backend schemas - -export type { LLMGenerationConfig } from '../config/llmGenerationParams' -import type { LLMGenerationConfig } from '../config/llmGenerationParams' - -export enum EvaluationType { - ASR = 'asr', - TTS = 'tts', -} - -export enum EvaluationStatus { - PENDING = 'pending', - PROCESSING = 'processing', - COMPLETED = 'completed', - FAILED = 'failed', - CANCELLED = 'cancelled', -} - -export interface AudioFile { - id: string - filename: string - format: string - file_size: number - duration?: number | null - sample_rate?: number | null - channels?: number | null - uploaded_at: string -} - -export interface Evaluation { - id: string - audio_id: string - reference_text?: string | null - evaluation_type: EvaluationType - model_name?: string | null - status: EvaluationStatus - metrics_requested?: string[] | null - created_at: string - started_at?: string | null - completed_at?: string | null - error_message?: string | null -} - -export interface DashboardSummary { - evaluations: { - total: number - completed: number - pending: number - failed: number - } - resources: { - agents: number - personas: number - scenarios: number - integrations: number - voice_bundles: number - } - setup_progress: { - has_integration: boolean - has_voice_bundle: boolean - has_agent: boolean - has_evaluation: boolean - } - metrics: { - total: number - enabled: number - } - call_imports: { - total: number - } - call_import_evaluations: { - total: number - completed: number - running: number - failed: number - } - recent_evaluations: Evaluation[] -} - -export interface ModelConfigEntry { - provider: string - model_type: string - description?: string - featured?: boolean - featured_rank?: number - highlights?: string[] -} - -export interface EvaluationCreate { - audio_id: string - reference_text?: string | null - evaluation_type: EvaluationType - model_name?: string | null - metrics?: string[] -} - -export interface EvaluationResult { - evaluation_id: string - status: EvaluationStatus - transcript?: string | null - metrics: Record - processing_time?: number | null - model_used?: string | null - created_at: string -} - -export interface BatchEvaluationResult { - processed_files: number - failed_files: number - aggregated_metrics?: Record | null - individual_results: EvaluationResult[] -} - -/** Voice agent evaluator run (evaluator_results table). */ -export type EvaluatorResultStatus = - | 'queued' - | 'call_initiating' - | 'call_connecting' - | 'call_in_progress' - | 'call_ended' - | 'transcribing' - | 'evaluating' - | 'fetching_details' - | 'completed' - | 'failed' - -export interface EvaluatorResultMetricScore { - value: unknown - type: string - metric_name: string - parent_metric_id?: string | null -} - -export interface EvaluatorResultRow { - id: string - result_id: string - name: string | null - evaluator_id: string | null - agent_id?: string | null - persona_id?: string | null - scenario_id?: string | null - suite_id?: string | null - timestamp: string - duration_seconds: number | null - status: EvaluatorResultStatus - metric_scores: Record | null - error_message: string | null - agent?: { id: string; name: string } | null - scenario?: { id: string; name: string } | null -} - -export interface EvaluatorResultListResponse { - items: EvaluatorResultRow[] - total: number -} - -export interface EvaluatorResultCounts { - total: number - completed: number - failed: number - in_progress: number - last_run_at?: string | null -} - -export interface EvaluatorResultsScenarioSummary { - scenario_id: string - scenario_name: string - counts: EvaluatorResultCounts -} - -export interface EvaluatorResultsSuiteSummary { - suite_id: string - suite_name?: string | null - agent_id: string - persona_id?: string | null - counts: EvaluatorResultCounts - scenarios?: EvaluatorResultsScenarioSummary[] | null -} - -export interface EvaluatorResultsAgentSummary { - agent_id: string - agent_name: string - counts: EvaluatorResultCounts - suites?: EvaluatorResultsSuiteSummary[] | null -} - -export interface EvaluatorResultsOverviewResponse { - workspace_counts: EvaluatorResultCounts - agents: EvaluatorResultsAgentSummary[] - unassigned: { - counts: EvaluatorResultCounts - recent_result_ids: string[] - } -} - -export interface ListEvaluatorResultsParams { - skip?: number - limit?: number - evaluatorId?: string - agentId?: string - suiteId?: string - scenarioId?: string - status?: 'completed' | 'failed' | 'in_progress' - unassignedOnly?: boolean - playground?: boolean - testAgentsOnly?: boolean -} - -export interface APIKey { - id: string - key: string - name?: string | null - is_active: boolean - created_at: string - last_used?: string | null - message?: string -} - -export interface MessageResponse { - message: string -} - -// IAM & User Types -export enum Role { - READER = 'reader', - WRITER = 'writer', - ADMIN = 'admin', -} - -export enum InvitationStatus { - PENDING = 'pending', - ACCEPTED = 'accepted', - DECLINED = 'declined', - EXPIRED = 'expired', -} - -export interface User { - id: string - email: string - name?: string | null - is_active: boolean - created_at: string -} - -export interface OrganizationMember { - id: string - user_id: string - organization_id: string - role: Role - joined_at: string - user: User -} - -export interface Invitation { - id: string - organization_id: string - email: string - role: Role - status: InvitationStatus - expires_at: string - created_at: string - organization_name?: string | null -} - -export interface InvitationCreate { - email: string - role: Role -} - -export interface RoleUpdate { - role: Role -} - -export interface Profile { - id: string - email: string - name?: string | null - first_name?: string | null - last_name?: string | null - created_at: string - organizations: Array<{ - id: string - name: string - role: string - joined_at: string - }> -} - -export interface UserUpdate { - name?: string | null - first_name?: string | null - last_name?: string | null - email?: string | null -} - -export interface UserPreferences { - theme?: string - notifications_enabled?: boolean - email_notifications?: boolean - default_language?: string - [key: string]: any -} - -export interface UserPreferencesUpdate { - theme?: string - notifications_enabled?: boolean - email_notifications?: boolean - default_language?: string - [key: string]: any -} - -// Integration Types -export enum IntegrationPlatform { - RETELL = 'retell', - VAPI = 'vapi', - CARTESIA = 'cartesia', - ELEVENLABS = 'elevenlabs', - DEEPGRAM = 'deepgram', - MURF = 'murf', - SARVAM = 'sarvam', - VOICEMAKER = 'voicemaker', - SMALLEST = 'smallest', -} - -export enum TelephonyProvider { - PLIVO = 'plivo', - EXOTEL = 'exotel', - VOBIZ = 'vobiz', -} - -export type CredentialRoutingMode = 'inherit' | 'gateway' | 'direct' -export type GatewayInterfaceMode = 'inherit' | 'litellm_shim' | 'native_openai' - -export type EffectiveCredentialRouting = - | 'inherit' - | 'direct' - | 'gateway' - | 'bifrost' - | 'litellm_proxy' - -export interface Integration { - id: string - organization_id: string - platform: IntegrationPlatform - name?: string | null - public_key?: string | null - is_active: boolean - /** True if this row is the default credential for (org, platform). */ - is_default?: boolean - routing_mode?: CredentialRoutingMode - effective_routing?: EffectiveCredentialRouting - created_at: string - updated_at: string - last_tested_at?: string | null -} - -export interface IntegrationCreate { - platform: IntegrationPlatform - api_key: string - public_key?: string - name?: string | null - routing_mode?: CredentialRoutingMode - /** Mark the new credential as the default for (org, platform). */ - is_default?: boolean -} - -// VoiceBundle Types -export enum ModelProvider { - OPENAI = 'openai', - ANTHROPIC = 'anthropic', - GOOGLE = 'google', - XAI = 'xai', - FIREWORKS = 'fireworks', - COHERE = 'cohere', - MISTRAL = 'mistral', - META = 'meta', - TOGETHER = 'together', - PERPLEXITY = 'perplexity', - AZURE = 'azure', - AWS = 'aws', - DEEPGRAM = 'deepgram', - CARTESIA = 'cartesia', - ELEVENLABS = 'elevenlabs', - MURF = 'murf', - CUSTOM = 'custom', - SARVAM = 'sarvam', - VOICEMAKER = 'voicemaker', - SMALLEST = 'smallest', -} - -// AI Provider Types -export interface AIProvider { - id: string - provider: ModelProvider - api_key?: string | null - name?: string | null - endpoint_url?: string | null - is_active: boolean - /** True if this row is the default credential for (org, provider). */ - is_default?: boolean - routing_mode?: CredentialRoutingMode - gateway_model?: string | null - gateway_interface?: GatewayInterfaceMode - gateway_base_url?: string | null - gateway_auth_header?: string | null - gateway_auth_secret_env?: string | null - has_gateway_auth_secret?: boolean - gateway_extra_headers?: Record | null - /** True when provider secrets are resolved by the Bifrost gateway. */ - gateway_managed?: boolean - effective_routing?: EffectiveCredentialRouting - effective_gateway_interface?: 'litellm_shim' | 'native_openai' - created_at: string - updated_at: string - last_tested_at?: string | null -} - -export interface AIProviderCreate { - provider: ModelProvider - api_key?: string | null - name?: string | null - endpoint_url?: string | null - routing_mode?: CredentialRoutingMode - gateway_model?: string | null - gateway_interface?: GatewayInterfaceMode - gateway_base_url?: string | null - gateway_auth_header?: string | null - gateway_auth_secret_env?: string | null - gateway_auth_secret?: string | null - gateway_extra_headers?: Record | null - /** Mark the new credential as the default for (org, provider). */ - is_default?: boolean -} - -export interface AIProviderUpdate { - api_key?: string | null - name?: string | null - endpoint_url?: string | null - is_active?: boolean - routing_mode?: CredentialRoutingMode - gateway_model?: string | null - gateway_interface?: GatewayInterfaceMode - gateway_base_url?: string | null - gateway_auth_header?: string | null - gateway_auth_secret_env?: string | null - gateway_auth_secret?: string | null - clear_gateway_auth_secret?: boolean - gateway_extra_headers?: Record | null -} - -export enum VoiceBundleType { - STT_LLM_TTS = 'stt_llm_tts', - S2S = 's2s', -} - -export interface VoiceBundle { - id: string - name: string - description?: string | null - bundle_type: VoiceBundleType - stt_provider?: ModelProvider | null - stt_model?: string | null - /** - * Optional explicit AIProvider/Integration row id for STT. When null the - * runtime resolver picks the default credential for stt_provider. - */ - stt_credential_id?: string | null - llm_provider?: ModelProvider | null - llm_model?: string | null - llm_temperature?: number | null - llm_max_tokens?: number | null - llm_config?: Record | null - llm_credential_id?: string | null - tts_provider?: ModelProvider | null - tts_model?: string | null - tts_voice?: string | null - tts_config?: Record | null - tts_credential_id?: string | null - s2s_provider?: ModelProvider | null - s2s_model?: string | null - s2s_config?: Record | null - s2s_credential_id?: string | null - extra_metadata?: Record | null - is_active: boolean - created_at: string - updated_at: string - created_by?: string | null -} - -export interface VoiceBundleCreate { - name: string - description?: string | null - bundle_type?: VoiceBundleType - stt_provider?: ModelProvider | null - stt_model?: string | null - stt_credential_id?: string | null - llm_provider?: ModelProvider | null - llm_model?: string | null - llm_temperature?: number | null - llm_max_tokens?: number | null - llm_config?: Record | null - llm_credential_id?: string | null - tts_provider?: ModelProvider | null - tts_model?: string | null - tts_voice?: string | null - tts_config?: Record | null - tts_credential_id?: string | null - s2s_provider?: ModelProvider | null - s2s_model?: string | null - s2s_config?: Record | null - s2s_credential_id?: string | null - extra_metadata?: Record | null -} - -// Test Agent Types -export interface AgentPhoneAssignmentConflict { - agent_id: string - agent_name: string - phone_number: string -} - -export interface AgentPhoneAssignmentCheckResponse { - available: boolean - phone_number?: string | null - conflict?: AgentPhoneAssignmentConflict | null -} - -export interface TestAgent { - id: string - agent_id?: string | null - name: string - phone_number?: string | null - telephony_phone_number_id?: string | null - language: string - description: string | null - prompt_variables?: Record | null - silence_hangup_secs?: number - call_type: string - call_medium: string - voice_bundle_id?: string | null - voice_ai_integration_id?: string | null - voice_ai_agent_id?: string | null - provider_prompt?: string | null - provider_prompt_synced_at?: string | null - created_at: string - updated_at: string -} - -// Test Agent Conversation Types -export interface TestAgentConversation { - id: string - organization_id: string - agent_id: string - persona_id: string - scenario_id: string - voice_bundle_id: string - status: string - live_transcription?: Array<{ - speaker: string - text: string - timestamp: number - audio_segment_key?: string - }> | null - conversation_audio_key?: string | null - full_transcript?: string | null - started_at: string - ended_at?: string | null - duration_seconds?: number | null - conversation_metadata?: Record | null - created_at: string - updated_at: string - created_by?: string | null -} - -export interface TestAgentConversationCreate { - agent_id: string - persona_id: string - scenario_id: string - voice_bundle_id: string - conversation_metadata?: Record | null -} - -export interface TestAgentConversationUpdate { - status?: string | null - live_transcription?: Array> | null - full_transcript?: string | null - conversation_metadata?: Record | null -} - -export interface VoiceBundleUpdate { - name?: string - description?: string | null - stt_provider?: ModelProvider - stt_model?: string - stt_credential_id?: string | null - llm_provider?: ModelProvider - llm_model?: string - llm_temperature?: number | null - llm_max_tokens?: number | null - llm_config?: Record | null - llm_credential_id?: string | null - tts_provider?: ModelProvider - tts_model?: string - tts_voice?: string | null - tts_config?: Record | null - tts_credential_id?: string | null - s2s_provider?: ModelProvider | null - s2s_model?: string | null - s2s_config?: Record | null - s2s_credential_id?: string | null - extra_metadata?: Record | null - is_active?: boolean -} - -// Data Sources Types -export interface S3ConnectionTest { - bucket_name: string - region?: string - access_key_id: string - secret_access_key: string - endpoint_url?: string | null -} - -export interface S3ConnectionTestResponse { - success: boolean - message: string - bucket_name?: string | null -} - -export interface S3FileInfo { - key: string - filename: string - size: number - last_modified: string -} - -export interface S3FolderInfo { - name: string - path: string -} - -export interface S3ListFilesResponse { - files: S3FileInfo[] - total: number - prefix?: string | null -} - -export interface S3BrowseResponse { - folders: S3FolderInfo[] - files: S3FileInfo[] - current_path: string - organization_id: string -} - -export interface S3Status { - enabled: boolean - provider?: 's3' | 'gcs' | string - error?: string | null -} - -// Alert Types -export enum AlertMetricType { - NUMBER_OF_CALLS = 'number_of_calls', - CALL_DURATION = 'call_duration', - ERROR_RATE = 'error_rate', - SUCCESS_RATE = 'success_rate', - LATENCY = 'latency', - CUSTOM = 'custom', -} - -export enum AlertAggregation { - SUM = 'sum', - AVG = 'avg', - COUNT = 'count', - MIN = 'min', - MAX = 'max', -} - -export enum AlertOperator { - GREATER_THAN = '>', - LESS_THAN = '<', - GREATER_THAN_OR_EQUAL = '>=', - LESS_THAN_OR_EQUAL = '<=', - EQUAL = '=', - NOT_EQUAL = '!=', -} - -export enum AlertNotifyFrequency { - IMMEDIATE = 'immediate', - HOURLY = 'hourly', - DAILY = 'daily', - WEEKLY = 'weekly', -} - -export enum AlertStatus { - ACTIVE = 'active', - PAUSED = 'paused', - DISABLED = 'disabled', -} - -export enum AlertHistoryStatus { - TRIGGERED = 'triggered', - NOTIFIED = 'notified', - ACKNOWLEDGED = 'acknowledged', - RESOLVED = 'resolved', -} - -export interface Alert { - id: string - organization_id: string - name: string - description?: string | null - metric_type: AlertMetricType - aggregation: AlertAggregation - operator: AlertOperator - threshold_value: number - time_window_minutes: number - agent_ids?: string[] | null - notify_frequency: AlertNotifyFrequency - notify_emails?: string[] | null - notify_webhooks?: string[] | null - status: AlertStatus - created_at: string - updated_at: string - created_by?: string | null -} - -export interface AlertCreate { - name: string - description?: string | null - metric_type?: AlertMetricType - aggregation?: AlertAggregation - operator?: AlertOperator - threshold_value: number - time_window_minutes?: number - agent_ids?: string[] | null - notify_frequency?: AlertNotifyFrequency - notify_emails?: string[] - notify_webhooks?: string[] -} - -export interface AlertUpdate { - name?: string - description?: string | null - metric_type?: AlertMetricType - aggregation?: AlertAggregation - operator?: AlertOperator - threshold_value?: number - time_window_minutes?: number - agent_ids?: string[] | null - notify_frequency?: AlertNotifyFrequency - notify_emails?: string[] - notify_webhooks?: string[] - status?: AlertStatus -} - -export interface AlertHistoryItem { - id: string - organization_id: string - alert_id: string - triggered_at: string - triggered_value: number - threshold_value: number - status: AlertHistoryStatus - notified_at?: string | null - notification_details?: Record | null - acknowledged_at?: string | null - acknowledged_by?: string | null - resolved_at?: string | null - resolved_by?: string | null - resolution_notes?: string | null - context_data?: Record | null - created_at: string - updated_at: string - alert?: Alert -} - - -// Cron Job Types -export enum CronJobStatus { - ACTIVE = 'active', - PAUSED = 'paused', - COMPLETED = 'completed', -} - -export interface CronJob { - id: string - organization_id: string - name: string - cron_expression: string - timezone: string - max_runs: number - current_runs: number - evaluator_ids: string[] - status: CronJobStatus - next_run_at?: string | null - last_run_at?: string | null - created_at: string - updated_at: string - created_by?: string | null -} - -export interface CronJobCreate { - name: string - cron_expression: string - timezone: string - max_runs: number - evaluator_ids: string[] -} - -export interface CronJobUpdate { - name?: string - cron_expression?: string - timezone?: string - max_runs?: number - evaluator_ids?: string[] - status?: CronJobStatus -} - -// --- Call Imports --- - -/** - * Lifecycle for a call-import batch. - * - * - ``uploaded`` : file landed in S3, no mapping yet. - * - ``mapped`` : user picked a schema + sheet + column mapping; no - * rows materialised yet, no worker enqueued. - * - ``processing`` : rows materialised + workers enqueued. - * - ``pending`` : transient state used by the legacy one-shot - * ``POST /upload`` endpoint just before transitioning - * to ``processing``. - */ -export type CallImportStatus = - | 'pending' - | 'uploaded' - | 'mapped' - | 'processing' - | 'completed' - | 'partial' - | 'failed' - | 'deleting' - -export type CallImportRowStatus = - | 'pending' - | 'processing' - | 'completed' - | 'failed' - -/** Where the value in `transcript` came from. */ -export type CallImportTranscriptSource = - | 'csv' - | 'transcribed' - | 'edited' - | null -/** Lifecycle status for the post-hoc transcription workflow itself. */ -export type CallImportTranscriptStatus = - | 'idle' - | 'pending' - | 'running' - | 'completed' - | 'failed' - | null - -/** - * Which transcript an evaluation run scored against. - * - `production`: the CSV-supplied value on `CallImportRow.transcript`. - * - `diarised`: the worker-produced value on `CallImportRow.diarised_transcript`. - */ -export type CallImportEvaluationTranscriptSource = 'production' | 'diarised' - -/** - * One contiguous turn inside ``CallImportRow.diarised_segments``. - * - * The diarisation worker rewrites each pyannote ``Speaker N`` label - * into ``agent`` / ``user`` (first speaker = agent heuristic). Anything - * beyond two distinct speakers keeps a generic ``speaker_N`` label so - * multi-party recordings still render every voice. - */ -export interface CallImportDiarisedSegment { - speaker: string - text: string - start: number - end: number - /** Original pyannote label (``Speaker 1`` / ``Speaker 2`` / ...). */ - raw_speaker: string -} - -export interface CallImportRow { - id: string - row_index: number - /** Mandatory identifier per row. Renamed from ``external_call_id``. */ - conversation_id: string - recording_url: string | null - recording_date: string | null - /** Production transcript — the value supplied via the CSV upload. */ - transcript: string | null - /** Provenance of the stored production transcript (csv = CSV upload, edited = manual edit). */ - transcript_source: CallImportTranscriptSource - /** Legacy: provider recorded by the original transcription worker before the split. */ - transcript_provider: string | null - transcript_model: string | null - transcript_status: CallImportTranscriptStatus - transcript_error: string | null - transcribed_at: string | null - /** Diarised transcript — produced by the post-hoc diarisation worker. */ - diarised_transcript: string | null - /** Provider used by the diarisation worker (e.g. "deepgram"). */ - diarised_transcript_provider: string | null - diarised_transcript_model: string | null - diarised_transcript_status: CallImportTranscriptStatus - diarised_transcript_error: string | null - diarised_at: string | null - /** - * Structured speaker turns produced by the diarisation worker. Each - * entry is a single contiguous turn shaped as - * `{ speaker: 'agent' | 'user' | 'speaker_N', text, start, end, - * raw_speaker }`. ``diarised_transcript`` is a rendered - * `: ` view of this list with - * ``diarised_speaker_swap`` applied. ``null`` on legacy rows that - * were diarised before structured turns were persisted (or when the - * STT provider didn't surface segments). - */ - diarised_segments: CallImportDiarisedSegment[] | null - /** - * When ``true`` the ``agent`` <-> ``user`` mapping inside - * ``diarised_segments`` is inverted in the rendered transcript / - * CSV export. The worker writes the canonical mapping using a - * "first speaker is the agent" heuristic; reviewers can flip the - * toggle from the row detail panel without re-running diarisation. - */ - diarised_speaker_swap: boolean - /** - * LLM that turned the STT plain-text output into structured - * ``diarised_segments``. NULL on legacy rows (pre-LLM-diariser). - */ - diarised_llm_provider: string | null - diarised_llm_model: string | null - /** - * Exact prompt the LLM diariser ran with. Useful for the modal to - * pre-fill its textarea when the operator wants to iterate on a - * previously-diarised row. - */ - diarised_prompt: string | null - /** - * Diarisation pipeline that produced this row's turns. - * - `stt_llm` (default) — two-stage STT then LLM diariser. - * - `llm_only` — single-stage multimodal LLM (audio in). - * Read-only; written by the worker on each diarisation. - */ - transcribe_mode?: 'stt_llm' | 'llm_only' - /** - * Per-row preservation of the mapped source cells. Values land here - * as whatever type the schema parameter coerced them to — - * strings (text / url / conversation_id / recording_url / - * recording_date / transcript / datetime), numbers, booleans, or - * ``null`` for blanks. Always - * coerce with ``String(value)`` before string operations. - */ - raw_columns: Record | null - status: CallImportRowStatus - recording_s3_key: string | null - recording_content_type: string | null - recording_size_bytes: number | null - error_message: string | null - attempts: number - created_at: string - updated_at: string -} - -export interface CallImportTag { - id: string - name: string - color: string | null - created_at: string - updated_at: string -} - -/** - * Parameter type tag on a Call Import schema parameter. - * - * - ``conversation_id``: mandatory identifier (one per schema). - * - ``recording_url``: feeds ``CallImportRow.recording_url``. - * - ``recording_date``: date-only call recording date used for reports. - * - ``transcript``: feeds ``CallImportRow.transcript``. - * - ``text`` / ``number`` / ``boolean`` / ``datetime`` / ``url``: - * generic typed fields preserved per row in ``raw_columns`` and - * surfaced in the evaluation export under the parameter's name. - */ -export type CallImportSchemaParameterType = - | 'conversation_id' - | 'recording_url' - | 'recording_date' - | 'transcript' - | 'text' - | 'number' - | 'boolean' - | 'datetime' - | 'url' - -export interface CallImportSchemaParameter { - id?: string - name: string - type: CallImportSchemaParameterType - description: string | null - is_required: boolean - ordering?: number -} - -export interface CallImportSchema { - id: string - organization_id: string - workspace_id: string - name: string - description: string | null - parameters: CallImportSchemaParameter[] - /** How many CallImport batches reference this schema. */ - usage_count: number - created_at: string - updated_at: string -} - -export interface CallImportSchemaListResponse { - items: CallImportSchema[] - total: number -} - -export interface CallImportSchemaCreate { - name: string - description?: string | null - parameters: Array> -} - -export interface CallImportSchemaUpdate { - name?: string - description?: string | null - parameters?: Array> -} - -/** - * In-org Workspace - the active workspace scopes call imports and - * metrics in the UI. The org's Default workspace is auto-seeded by - * migration 033 and cannot be deleted. - */ -export interface Workspace { - id: string - organization_id: string - name: string - slug: string - is_default: boolean - created_at: string - updated_at: string - role_id?: string | null - role_name?: string | null - capabilities?: string[] -} - -export interface WorkspaceRole { - id: string - organization_id: string - name: string - description?: string | null - capabilities: string[] - is_system: boolean - created_at: string - updated_at: string -} - -export interface WorkspaceMember { - id: string - workspace_id: string - user_id: string - role_id: string - role_name: string - user_email: string - user_name?: string | null - added_by_user_id?: string | null - created_at: string -} - -export interface CapabilityInfo { - key: string - label: string -} - -export interface CapabilityDomain { - key: string - label: string - capabilities: CapabilityInfo[] -} - -export interface WorkspaceRoleCreate { - name: string - description?: string | null - capabilities: string[] -} - -export interface WorkspaceRoleUpdate { - name?: string - description?: string | null - capabilities?: string[] -} - -export interface CallImportSourceRowSkip { - source_row: number - reason: string - message: string -} - -export interface CallImport { - id: string - organization_id: string - /** Workspace this import belongs to. */ - workspace_id: string - /** - * Telephony provider key. ``null`` until the IMPORT stage in the - * staged flow (which is the first step that knows the provider). - * Always populated on post-import batches and on legacy one-shot - * uploads. - */ - provider: string | null - telephony_integration_id: string | null - original_filename: string | null - /** - * For Excel uploads, which worksheet this batch came from. ``null`` - * for CSV uploads (CSV files have no sheet concept) and for any - * imports created before multi-sheet support landed. - */ - sheet_name: string | null - /** Optional free-text dataset label (high-level segregation filter). */ - dataset: string | null - /** Tags currently attached to this import. Empty array if untagged. */ - tags: CallImportTag[] - /** - * Reusable Input Parameter schema the batch was uploaded against. - * NULL on legacy batches uploaded before the schema-driven flow shipped. - */ - schema_id: string | null - /** - * Schema-driven mapping: ``{parameter_name: csv_header}``. Empty on - * legacy batches; check ``column_mapping`` / ``extra_columns`` / - * ``custom_column_mapping`` instead for those. - */ - parameter_mapping: Record - /** Legacy free-form mapping kept for batches uploaded before schemas. */ - column_mapping: Record - /** Legacy extra-column list kept for backwards-compat. */ - extra_columns: string[] - /** Legacy uploader-named columns kept for backwards-compat. */ - custom_column_mapping: Record - /** - * Source headers the uploader explicitly skipped, captured at the - * MAP stage. Empty for legacy one-shot uploads where the value was - * ephemeral. - */ - skipped_columns: string[] - /** - * Source rows skipped at parse time (missing/invalid conversation ID or URL). - */ - source_row_skips?: CallImportSourceRowSkip[] - /** S3 key for the staged source file. ``null`` on legacy batches. */ - source_s3_key: string | null - /** ``'csv'`` / ``'xlsx'`` for staged files, or ``'audio'`` for manual uploads. */ - source_format: string | null - source_size_bytes: number | null - source_content_type: string | null - /** - * Snapshot of the file's sheets + headers captured at UPLOAD time so - * the MAP UI can render without re-fetching the source from S3. - * ``null`` on legacy batches. - */ - available_sheets: CallImportPreviewSheet[] | null - total_rows: number - completed_rows: number - failed_rows: number - status: CallImportStatus - error_message: string | null - created_at: string - updated_at: string -} - -export interface CallImportDetail extends CallImport { - rows: CallImportRow[] - /** - * Total row count *after* applying the optional ``q`` search filter. - * ``null`` when no filter is active — paginate against ``total_rows`` - * in that case. - */ - filtered_total_rows: number | null - /** - * Batch-wide aggregates of ``CallImportRow.diarised_transcript_status``. - * The ``idle`` bucket (rows never touched by the transcribe/diarise - * worker) is implicit: ``total_rows - (pending + running + completed - * + failed)``. Lets the UI render a transcribe-and-diarise progress - * bar without paginating through every row. - */ - diarised_pending_rows: number - diarised_running_rows: number - diarised_completed_rows: number - diarised_failed_rows: number -} - -export interface CallImportListResponse { - items: CallImport[] - total: number - page: number - page_size: number -} - -export interface CallImportUploadResponse { - id: string - total_rows: number - status: CallImportStatus - dataset: string | null - tags: CallImportTag[] - message: string -} - -/** One worksheet (or one CSV file synthesized as a single sheet). */ -export interface CallImportPreviewSheet { - /** Sheet name for xlsx; filename for csv. */ - name: string - /** Column headers from the first non-empty row. */ - headers: string[] - /** Approximate count of data rows (excludes the header row). */ - row_count: number -} - -/** - * Sheets / headers extracted server-side from an uploaded CSV or Excel - * workbook. Drives the modal's column-mapping UI without forcing the - * frontend to parse the file itself. - */ -export interface CallImportPreviewResponse { - /** ``'csv'`` or ``'xlsx'``. */ - format: 'csv' | 'xlsx' - sheets: CallImportPreviewSheet[] -} - -export type MetricSelectionMode = 'single_choice' | 'multi_label' - -export interface CallImportMetricSummary { - id: string - name: string - metric_type: string | null - description: string | null - parent_metric_id?: string | null - selection_mode?: MetricSelectionMode | null - /** Only meaningful on multi_label parents; gates the Discovered - * Labels panel on the Flow tab. Defaults to false. */ - allow_discovery?: boolean -} - -/** Per-metric LLM override (provider+model+optional credential + generation params). */ -export interface CallImportEvaluationLLMOverride { - provider?: string | null - model?: string | null - credential_id?: string | null - llm_config?: LLMGenerationConfig | null -} - -export interface CallImportEvaluation { - id: string - call_import_id: string - organization_id: string - /** User-supplied label for the run; null when not named. */ - name: string | null - selected_metric_ids: string[] - /** parent_id -> [child_id, ...] snapshot captured at run time. */ - selected_metric_groups?: Record | null - metrics: CallImportMetricSummary[] - status: 'pending' | 'running' | 'completed' | 'partial' | 'failed' - total_rows: number - completed_rows: number - failed_rows: number - error_message: string | null - /** Run-level LLM provider chosen by the user (null = legacy default). */ - llm_provider: string | null - llm_model: string | null - llm_credential_id: string | null - llm_config?: LLMGenerationConfig | null - metric_llm_overrides: Record | null - stt_provider: string | null - stt_model: string | null - stt_credential_id: string | null - /** - * Run-level LLM diariser config used when the worker auto-diarises - * rows that are missing a diarised transcript. - */ - diarisation_llm_provider?: string | null - diarisation_llm_model?: string | null - diarisation_llm_credential_id?: string | null - diarisation_prompt?: string | null - /** - * Diarisation pipeline shape this run was created with. - * - `stt_llm` (default) — STT then an LLM diariser over the text. - * - `llm_only` — audio fed directly to a multimodal diariser LLM. - * Surfaced so the retry / re-run UI can preselect the right mode. - */ - transcribe_mode?: 'stt_llm' | 'llm_only' - /** - * Which transcript column this run scored against. - * Defaults to `production` on legacy runs. - */ - transcript_source: CallImportEvaluationTranscriptSource - /** - * Other evaluation ids created in the same Run Evaluation request. - * Populated only on the POST response when the user ticked both - * Production and Diarised. Empty array on all other reads. - */ - sibling_evaluation_ids: string[] - started_at: string | null - finished_at: string | null - created_at: string - updated_at: string - /** - * Cached LLM-generated TLDR rendered above the Visualizations tab. - * Populated lazily via ``POST /evaluations/{id}/insights``; null on - * runs the user has not summarised yet. - */ - tldr_summary?: EvaluationTldrSummary | null - user_insights?: EvaluationUserInsightsState | null - metric_clusters?: EvaluationMetricClustersState | null - /** - * True when the user opted into top-level metric discovery on the - * Run Evaluation modal. Gates the Discovered metrics panel on the - * evaluation detail Flow tab. - */ - discover_new_metrics?: boolean - /** - * Set while a bulk background operation (abort, force-fail, retry) is - * still running. The UI disables other mutating actions until cleared. - */ - bulk_operation?: 'abort' | 'force_fail_pending' | 'retry' | null -} - -/** - * LLM-generated narrative + bullet patterns for a single evaluation - * run. Cached on the evaluation row so re-opening the Visualizations - * tab doesn't auto-burn LLM tokens. ``is_stale`` is computed by the - * backend at read time when ``completed_rows`` has grown since the - * summary was generated. - */ -export interface EvaluationTldrSummary { - narrative: string - patterns: string[] - metric_insights?: Record - generated_at: string - generated_at_completed_rows: number - provider?: string | null - model?: string | null - is_stale: boolean -} - -export interface UserInsightCategory { - label: string - count: number - share_pct: number -} - -export interface UserInsightEvidenceTurn { - speaker: string - text: string -} - -export interface UserInsightEvidence { - conversation_id?: string | null - quote: string - turns?: UserInsightEvidenceTurn[] -} - -export interface EvaluationUserInsightItem { - id: string - title: string - categories: UserInsightCategory[] - observation: string - evidence: UserInsightEvidence -} - -export interface EvaluationUserInsightsState { - status: 'idle' | 'running' | 'completed' | 'failed' - insights: EvaluationUserInsightItem[] - overview?: string | null - generated_at?: string | null - generated_at_completed_rows: number - progress?: { completed_llm_calls: number; total_llm_calls: number } | null - provider?: string | null - model?: string | null - llm_calls_used: number - max_llm_calls?: number | null - error_message?: string | null - is_stale: boolean -} - -export type MetricClusterGapLabel = - | 'LOGIC_GAP' - | 'UNDERSPEC' - | 'EXISTS_NO_TRIGGER' - | 'MISSING' - -export interface MetricSubCluster { - label: string - count: number - share_pct: number -} - -export interface MetricClusterEvidenceTurn { - speaker: string - text: string -} - -export interface MetricClusterEvidence { - conversation_id?: string | null - evaluation_row_id?: string | null - quote: string - turns?: MetricClusterEvidenceTurn[] -} - -export interface MetricCluster { - id: string - label: string - gap_label: MetricClusterGapLabel - level: number - count: number - share_pct: number - sub_clusters: MetricSubCluster[] - observation: string - failure_reason?: string - evidence: MetricClusterEvidence - is_discovered: boolean -} - -export interface MetricClusterGroup { - metric_id: string - metric_name: string - flagged_count: number - failure_reason?: string - clusters: MetricCluster[] -} - -export interface DiscoveredProblemCluster { - id: string - label: string - gap_label: MetricClusterGapLabel - count: number - share_pct: number - observation: string - failure_reason?: string - evidence: MetricClusterEvidence -} - -export interface RcaRepeatedPatternRow { - metric_id: string - metric_name: string - top_rca_patterns: string - evidence_share_pct: number - evidence_calls: number - evidence_cluster_count?: number - failure_reason: string -} - -export interface RcaMetricHotspotRow { - metric_id: string - metric_name: string - description: string - metric_rate_pct: number - flagged_calls: number -} - -export interface RcaPromptAreaRow { - label: string - share_pct: number - gap_label: MetricClusterGapLabel -} - -export interface MetricClustersRcaSummary { - total_clusters: number - total_clustered_instances: number - total_flagged_instances?: number - analysed_calls: number - repeated_patterns: RcaRepeatedPatternRow[] - metric_hotspots: RcaMetricHotspotRow[] - prompt_areas: RcaPromptAreaRow[] -} - -export interface MetricFailurePolicy { - metric_id: string - failure_values: string[] - failure_child_names?: string[] - numeric_rule?: { op: 'lt' | 'lte' | 'gt' | 'gte'; threshold: number } | null -} - -export interface MetricFailurePolicyValueCount { - label: string - count: number -} - -export interface MetricFailurePolicyMetricPreview { - metric_id: string - metric_name: string - metric_type?: string | null - selection_mode?: string | null - is_multi_label_parent: boolean - value_counts: MetricFailurePolicyValueCount[] - child_names: string[] - row_count_by_value: Record - suggested_policy: MetricFailurePolicy - effective_policy: MetricFailurePolicy -} - -export interface MetricFailurePoliciesResponse { - previews: MetricFailurePolicyMetricPreview[] - policies: Record - source: 'inferred' | 'user' - updated_at?: string | null -} - -export interface MetricClusterEligibleRow { - evaluation_row_id: string - conversation_id?: string | null - row_index?: number | null - flagged_metric_names: string[] -} - -export interface MetricClusterEligibleRowsResponse { - items: MetricClusterEligibleRow[] - total: number -} - -export interface EvaluationMetricClustersState { - status: 'idle' | 'running' | 'completed' | 'failed' | 'cancelled' - groups: MetricClusterGroup[] - discovered_problems: DiscoveredProblemCluster[] - overview?: string | null - generated_at?: string | null - generated_at_completed_rows: number - progress?: { completed_llm_calls: number; total_llm_calls: number } | null - provider?: string | null - model?: string | null - llm_calls_used: number - max_llm_calls?: number | null - error_message?: string | null - is_stale: boolean - selected_evaluation_row_ids?: string[] - failure_policies?: Record - failure_policies_source?: 'inferred' | 'user' - failure_policies_updated_at?: string | null - rca_summary?: MetricClustersRcaSummary | null -} - -export interface AgentFlowNode { - id: string - label: string - node_type: 'start' | 'decision' | 'action' | 'terminal' - position_x?: number | null - position_y?: number | null - prompt_excerpt?: string | null - start_offset?: number | null - end_offset?: number | null -} - -export interface AgentFlowEdge { - source: string - target: string - condition?: string | null -} - -export interface AgentFlowGraph { - nodes: AgentFlowNode[] - edges: AgentFlowEdge[] - generated_at?: string | null - provider?: string | null - model?: string | null - layout_saved_at?: string | null - prompt_content_hash?: string | null - mapping_error?: string | null - generation_error?: string | null -} - -export interface ImportedAgent { - id: string - organization_id: string - name: string - description: string | null - content: string - tags: string[] | null - current_version: number - agent_flowchart?: AgentFlowGraph | null - agent_flowchart_status?: string | null - created_at: string - updated_at: string - created_by: string | null -} - -export interface ImportedAgentDetail extends ImportedAgent { - versions: PromptPartialVersion[] -} - -export interface MetricPartialChild { - name: string - description: string - example: string -} - -export interface MetricPartialContent { - schema_version: 1 - metric_kind: 'single' | 'category' - description: string - children?: MetricPartialChild[] -} - -export interface MetricPartial { - id: string - organization_id: string - name: string - description: string | null - content: string - tags: string[] | null - current_version: number - created_at: string - updated_at: string - created_by: string | null -} - -export interface MetricPartialDetail extends MetricPartial { - versions: PromptPartialVersion[] -} - -export interface PromptPartialVersion { - id: string - prompt_partial_id: string - version: number - content: string - change_summary: string | null - created_at: string - created_by: string | null -} - -export interface PromptImprovementSuggestion { - id: string - metric_id: string - metric_name: string - cluster_id: string - cluster_label: string - gap_label: MetricClusterGapLabel - share_pct: number - priority: 'high' | 'medium' | 'low' - change_type?: 'edit' | 'add' - target_section: string - anchor_excerpt?: string - current_gap: string - suggested_text: string - rationale: string - flow_node_id?: string - flow_node_label?: string -} - -export interface EvaluationPromptImprovementsState { - status: 'idle' | 'running' | 'completed' | 'failed' - imported_agent_id?: string | null - imported_agent_name?: string | null - suggestions: PromptImprovementSuggestion[] - overview?: string | null - generated_at?: string | null - generated_at_completed_rows: number - provider?: string | null - model?: string | null - error_message?: string | null - is_stale: boolean -} - -export interface MetricPeriodDelta { - label: string - detail: string - why?: string | null -} - -export interface CallImportEvaluationListResponse { - items: CallImportEvaluation[] - total: number -} - -export interface CallImportEvaluationBaselineCandidate { - evaluation_id: string - name: string - dataset: string - period_label: string | null - period_start: string | null - period_end: string | null - period_display: string - completed_rows: number - created_at: string - is_default: boolean -} - -export interface CallImportEvaluationBaselineCandidatesResponse { - items: CallImportEvaluationBaselineCandidate[] - default_evaluation_id: string | null -} - -export interface CallImportEvaluationRow { - id: string - evaluation_id: string - call_import_row_id: string - row_index: number | null - /** Mandatory identifier from the source batch (renamed from ``external_call_id``). */ - conversation_id: string | null - transcript: string | null - raw_columns: Record | null - recording_url: string | null - recording_date: string | null - /** - * S3 object key for the downloaded recording. Prefer this over - * ``recording_url`` for playback — we resolve it to a presigned URL - * so audio plays from our storage instead of the (often expired) - * provider URL. - */ - recording_s3_key: string | null - diarised_transcript_status?: string | null - diarised_transcript_error?: string | null - status: 'pending' | 'running' | 'completed' | 'failed' | 'skipped' - metric_scores: Record - error_message: string | null - started_at: string | null - finished_at: string | null - created_at: string - updated_at: string -} - -export interface CallImportEvaluationRowListResponse { - items: CallImportEvaluationRow[] - total: number - page: number - page_size: number -} - -// --- Retry (re-enqueue failed rows on an existing evaluation run) --- - -export interface CallImportEvaluationRetryRequest { - /** - * Restrict the retry to a specific subset of evaluation rows. - * When omitted, every row with status='failed' in this run is - * re-enqueued. - */ - eval_row_ids?: string[] - - /** - * Optional LLM overrides. When provided, persisted onto the run so - * future retries default to the new config. ``llm_provider`` and - * ``llm_model`` must be sent together — the backend 400s on - * half-configured input. - */ - llm_provider?: string - llm_model?: string - llm_credential_id?: string | null - - /** - * Optional STT overrides (only meaningful when the run scores the - * diarised transcript). Same paired-field rule as LLM. - */ - stt_provider?: string - stt_model?: string - stt_credential_id?: string | null - - /** - * When true, wipe the diarised transcript on every retried row so - * the (possibly new) STT runs from scratch. Only takes effect for - * diarised runs that have STT config. - */ - transcribe_overwrite?: boolean -} - -export interface CallImportEvaluationRetrySkippedItem { - eval_row_id: string - /** - * Why this row was not re-enqueued. Known values: - * - 'unknown' (id not in this run) - * - 'in_progress' (status is pending/running) - * - 'completed' (already successful) - * - 'source_row_missing' - */ - reason: 'unknown' | 'in_progress' | 'completed' | 'source_row_missing' -} - -export interface CallImportEvaluationRetryResponse { - requeued: number - /** - * Of those, how many were chained through a diarisation task first - * because the diarised transcript was missing. - */ - transcribe_requeued: number - skipped: CallImportEvaluationRetrySkippedItem[] -} - -export interface CallImportEvaluationBulkActionResponse { - accepted: boolean - target_count: number - evaluation_id: string -} - -// --- Diarization / transcription --- - -export interface CallImportTranscribeRequest { - /** - * Diarisation pipeline shape. - * - `stt_llm` (default) — STT produces plain text, then an LLM - * diariser splits it into agent/user turns. STT fields required. - * - `llm_only` — skip STT entirely and feed the audio bytes - * directly to a multimodal `diarization_llm_*` model along with - * `diarization_prompt`. STT fields MUST be omitted in this mode. - */ - mode?: 'stt_llm' | 'llm_only' - /** Required when `mode === 'stt_llm'`; must be null/omitted in `llm_only`. */ - stt_provider?: string | null - /** Required when `mode === 'stt_llm'`; must be null/omitted in `llm_only`. */ - stt_model?: string | null - credential_id?: string | null - language?: string | null - only_missing?: boolean - overwrite_existing?: boolean - row_ids?: string[] - /** - * LLM diariser. In `stt_llm` mode it splits the STT plain-text into - * agent/user turns; in `llm_only` mode it directly receives the - * audio along with `diarization_prompt`. Always required. - */ - diarization_llm_provider: string - diarization_llm_model: string - diarization_llm_credential_id?: string | null - /** - * Operator-supplied system prompt for the diariser LLM. NULL/empty - * means "fall back to the canonical default" (see - * ``getDiarisationDefaultPrompt``). - */ - diarization_prompt?: string | null -} - -export interface CallImportTranscribeResponse { - queued: number - skipped_rows: number - skipped_reason_counts: Record - accepted?: boolean -} - -export interface CallImportRowBulkDeleteResponse { - deleted: number - status?: 'completed' | 'accepted' -} - -export interface CallImportRetryFailedRowsResponse { - requeued: number - enqueue_failed: number - skipped: number -} - -export interface CallImportDiarisationPromptDefaultResponse { - prompt: string -} - -// --- Aggregation / visualization payloads --- - -export interface CallImportMetricHistogramBucket { - x0: number - x1: number - count: number -} - -export interface CallImportMetricValueCount { - label: string - count: number -} - -/** - * One unordered pair-count cell from a multi-label parent's - * co-occurrence matrix. ``a`` and ``b`` are child label names with - * ``a < b`` lexicographically; ``count`` is the number of rows on - * which both labels fired together. - */ -export interface CallImportMetricLabelPair { - a: string - b: string - count: number -} - -export interface CallImportMetricAggregate { - metric_id: string - metric_name: string - metric_type: string | null - metric_category?: 'quality' | 'user_insight' | string - /** - * True when the metric is a multi-label classifier parent. - * ``value_counts`` then lists per-child label tallies and one row - * may contribute to several labels, so the chart layout has to - * ignore the pie toggle (slices wouldn't sum to 100%) and the - * n-badge represents rows scored, not label occurrences. - */ - is_multi_label_parent?: boolean - count: number - skipped_count: number - error_count: number - mean: number | null - median: number | null - p25: number | null - p75: number | null - p95: number | null - min: number | null - max: number | null - stddev: number | null - histogram_buckets: CallImportMetricHistogramBucket[] - value_counts: CallImportMetricValueCount[] - /** - * Pairwise label intersections for multi-label parent metrics. - * Empty for everything else. The Visualizations tab reconstructs - * a square symmetric matrix from these unordered pairs to render - * the co-occurrence heatmap chart type. - */ - co_occurrence?: CallImportMetricLabelPair[] -} - -export interface CallImportEvaluationAggregateResponse { - evaluation_id: string - total_rows: number - completed_rows: number - failed_rows: number - metrics: CallImportMetricAggregate[] - period_deltas?: Record - baseline_evaluation_id?: string | null - failure_policies_source?: 'inferred' | 'user' | null -} - -export interface EvaluatorResultsAggregateResponse { - scope: string - suite_id?: string | null - agent_id?: string | null - scenario_id?: string | null - total_rows: number - completed_rows: number - failed_rows: number - metrics: CallImportMetricAggregate[] -} - -export interface CallImportInsightsRunPoint { - evaluation_id: string - name: string | null - created_at: string - mean: number | null - completed_rows: number -} - -export interface CallImportInsightsMetric { - metric_id: string - metric_name: string - metric_type: string | null - latest: CallImportMetricAggregate | null - trend: CallImportInsightsRunPoint[] -} - -export interface CallImportInsightsResponse { - call_import_id: string - total_rows: number - rows_with_transcript: number - rows_without_transcript: number - transcript_source_counts: Record - evaluation_count: number - metrics: CallImportInsightsMetric[] -} - -// --- Metrics hierarchy + flow visualization --- - -export interface MetricSummary { - id: string - organization_id: string - name: string - description: string | null - metric_type: string - metric_category?: 'quality' | 'user_insight' | string - trigger: string - enabled: boolean - is_default: boolean - metric_origin: string - supported_surfaces: string[] - enabled_surfaces: string[] - custom_data_type: string | null - custom_config: Record | null - tags: string[] | null - capture_rationale: boolean - parent_metric_id: string | null - selection_mode: MetricSelectionMode | null - allow_discovery?: boolean - /** - * When true, this metric is a "transcript-compare judge": at - * call-import evaluation time the worker feeds BOTH the production - * transcript and the diarised transcript to the LLM as a labeled - * pair, and the run's transcript_source toggle is ignored for this - * metric. Mutually exclusive with parent_metric_id and selection_mode - * — comparison metrics stay standalone. - */ - compare_transcripts?: boolean - children?: MetricSummary[] - created_at: string - updated_at: string - created_by: string | null -} - -export interface MetricChildDraft { - name: string - description?: string | null - enabled?: boolean - capture_rationale?: boolean | null - tags?: string[] | null -} - -export interface MetricCreateWithChildrenPayload { - name: string - description?: string | null - selection_mode: MetricSelectionMode - enabled?: boolean - supported_surfaces?: string[] - enabled_surfaces?: string[] - tags?: string[] | null - allow_discovery?: boolean - children: MetricChildDraft[] -} - -export interface MetricFlowNode { - id: string - label: string - count: number - is_terminal: boolean - is_discovered?: boolean -} - -export interface MetricFlowEdge { - source: string - target: string - count: number -} - -export interface MetricFlowResponse { - parent_metric_id: string - parent_metric_name: string - selection_mode: MetricSelectionMode | null - nodes: MetricFlowNode[] - edges: MetricFlowEdge[] - total_rows: number - rows_with_sequence: number -} - -export interface DiscoveredLabel { - key: string - name: string - description?: string | null - sample_rationale?: string | null - /** - * Up to 3 distinct LLM rationales captured for this candidate - * across rows. The Discovered Labels promote flow surfaces the - * first 2 as an ``Examples:`` block on the new sub-metric's - * rubric so the user starts with concrete cases in the prompt. - */ - examples?: string[] - count: number -} - -export interface DiscoveredLabelsResponse { - parent_metric_id: string - items: DiscoveredLabel[] -} - -/** - * One LLM-discovered candidate TOP-LEVEL metric aggregated across all - * rows of an evaluation. Mirrors :class:`DiscoveredLabel` but adds a - * ``suggested_type`` field — the LLM's guess at the best shape for - * the new metric — that the promote modal can pre-fill the type radio - * with. - */ -export interface DiscoveredMetric { - key: string - name: string - description?: string | null - suggested_type: 'boolean' | 'rating' | 'category' - sample_rationale?: string | null - examples?: string[] - count: number -} - -export interface DiscoveredMetricsResponse { - evaluation_id: string - items: DiscoveredMetric[] -} - -export interface ObservabilityCallAgent { - id: string - agent_id?: string | null - name: string -} - -export interface ObservabilityCallData { - startedAt?: string - started_at?: string - endedAt?: string - ended_at?: string - from_phone_number?: string - to_phone_number?: string - endedReason?: string - recording_s3_key?: string - recording_url?: string - duration_seconds?: number - agent_name?: string - _agent_ref?: string | number - direction?: string - messages?: Array<{ role: string; content: string; start_time?: number; end_time?: number }> - live_transcript?: Array<{ role: string; content: string; timestamp?: string; start_time?: number }> - metadata?: Record - call_short_id?: string -} - -export interface ObservabilityCall { - id: string - call_short_id: string - status?: string | null - call_event?: string | null - is_live?: boolean - direction?: string | null - source?: string | null - provider_platform?: string | null - provider_call_id?: string | null - agent_id?: string | null - agent?: ObservabilityCallAgent | null - created_at?: string | null - updated_at?: string | null - call_data?: ObservabilityCallData | null - live_transcript?: Array<{ role: string; content: string; timestamp?: string }> -} +// API Types matching the backend schemas + +export type { LLMGenerationConfig } from '../config/llmGenerationParams' +import type { LLMGenerationConfig } from '../config/llmGenerationParams' + +export enum EvaluationType { + ASR = 'asr', + TTS = 'tts', +} + +export enum EvaluationStatus { + PENDING = 'pending', + PROCESSING = 'processing', + COMPLETED = 'completed', + FAILED = 'failed', + CANCELLED = 'cancelled', +} + +export interface AudioFile { + id: string + filename: string + format: string + file_size: number + duration?: number | null + sample_rate?: number | null + channels?: number | null + uploaded_at: string +} + +export interface Evaluation { + id: string + audio_id: string + reference_text?: string | null + evaluation_type: EvaluationType + model_name?: string | null + status: EvaluationStatus + metrics_requested?: string[] | null + created_at: string + started_at?: string | null + completed_at?: string | null + error_message?: string | null +} + +export interface DashboardSummary { + evaluations: { + total: number + completed: number + pending: number + failed: number + } + resources: { + agents: number + personas: number + scenarios: number + integrations: number + voice_bundles: number + } + setup_progress: { + has_integration: boolean + has_voice_bundle: boolean + has_agent: boolean + has_evaluation: boolean + } + metrics: { + total: number + enabled: number + } + call_imports: { + total: number + } + call_import_evaluations: { + total: number + completed: number + running: number + failed: number + } + recent_evaluations: Evaluation[] +} + +export interface ModelConfigEntry { + provider: string + model_type: string + description?: string + featured?: boolean + featured_rank?: number + highlights?: string[] +} + +export interface EvaluationCreate { + audio_id: string + reference_text?: string | null + evaluation_type: EvaluationType + model_name?: string | null + metrics?: string[] +} + +export interface EvaluationResult { + evaluation_id: string + status: EvaluationStatus + transcript?: string | null + metrics: Record + processing_time?: number | null + model_used?: string | null + created_at: string +} + +export interface BatchEvaluationResult { + processed_files: number + failed_files: number + aggregated_metrics?: Record | null + individual_results: EvaluationResult[] +} + +/** Voice agent evaluator run (evaluator_results table). */ +export type EvaluatorResultStatus = + | 'queued' + | 'call_initiating' + | 'call_connecting' + | 'call_in_progress' + | 'call_ended' + | 'transcribing' + | 'evaluating' + | 'fetching_details' + | 'completed' + | 'failed' + +export interface EvaluatorResultMetricScore { + value: unknown + type: string + metric_name: string + parent_metric_id?: string | null +} + +export interface EvaluatorResultRow { + id: string + result_id: string + name: string | null + evaluator_id: string | null + agent_id?: string | null + persona_id?: string | null + scenario_id?: string | null + suite_id?: string | null + timestamp: string + duration_seconds: number | null + status: EvaluatorResultStatus + metric_scores: Record | null + error_message: string | null + agent?: { id: string; name: string } | null + scenario?: { id: string; name: string } | null +} + +export interface EvaluatorResultListResponse { + items: EvaluatorResultRow[] + total: number +} + +export interface EvaluatorResultCounts { + total: number + completed: number + failed: number + in_progress: number + last_run_at?: string | null +} + +export interface EvaluatorResultsScenarioSummary { + scenario_id: string + scenario_name: string + counts: EvaluatorResultCounts +} + +export interface EvaluatorResultsSuiteSummary { + suite_id: string + suite_name?: string | null + agent_id: string + persona_id?: string | null + counts: EvaluatorResultCounts + scenarios?: EvaluatorResultsScenarioSummary[] | null +} + +export interface EvaluatorResultsAgentSummary { + agent_id: string + agent_name: string + counts: EvaluatorResultCounts + suites?: EvaluatorResultsSuiteSummary[] | null +} + +export interface EvaluatorResultsOverviewResponse { + workspace_counts: EvaluatorResultCounts + agents: EvaluatorResultsAgentSummary[] + unassigned: { + counts: EvaluatorResultCounts + recent_result_ids: string[] + } +} + +export interface ListEvaluatorResultsParams { + skip?: number + limit?: number + evaluatorId?: string + agentId?: string + suiteId?: string + scenarioId?: string + status?: 'completed' | 'failed' | 'in_progress' + unassignedOnly?: boolean + playground?: boolean + testAgentsOnly?: boolean +} + +export interface APIKey { + id: string + key: string + name?: string | null + is_active: boolean + created_at: string + last_used?: string | null + message?: string +} + +export interface MessageResponse { + message: string +} + +// IAM & User Types +export enum Role { + READER = 'reader', + WRITER = 'writer', + ADMIN = 'admin', +} + +export enum InvitationStatus { + PENDING = 'pending', + ACCEPTED = 'accepted', + DECLINED = 'declined', + EXPIRED = 'expired', +} + +export interface User { + id: string + email: string + name?: string | null + is_active: boolean + created_at: string +} + +export interface OrganizationMember { + id: string + user_id: string + organization_id: string + role: Role + joined_at: string + user: User +} + +export interface Invitation { + id: string + organization_id: string + email: string + role: Role + status: InvitationStatus + expires_at: string + created_at: string + organization_name?: string | null +} + +export interface InvitationCreate { + email: string + role: Role +} + +export interface RoleUpdate { + role: Role +} + +export interface Profile { + id: string + email: string + name?: string | null + first_name?: string | null + last_name?: string | null + created_at: string + organizations: Array<{ + id: string + name: string + role: string + joined_at: string + }> +} + +export interface UserUpdate { + name?: string | null + first_name?: string | null + last_name?: string | null + email?: string | null +} + +export interface UserPreferences { + theme?: string + notifications_enabled?: boolean + email_notifications?: boolean + default_language?: string + [key: string]: any +} + +export interface UserPreferencesUpdate { + theme?: string + notifications_enabled?: boolean + email_notifications?: boolean + default_language?: string + [key: string]: any +} + +// Integration Types +export enum IntegrationPlatform { + RETELL = 'retell', + VAPI = 'vapi', + CARTESIA = 'cartesia', + ELEVENLABS = 'elevenlabs', + DEEPGRAM = 'deepgram', + MURF = 'murf', + SARVAM = 'sarvam', + VOICEMAKER = 'voicemaker', + SMALLEST = 'smallest', +} + +export enum TelephonyProvider { + PLIVO = 'plivo', + EXOTEL = 'exotel', + VOBIZ = 'vobiz', +} + +export type CredentialRoutingMode = 'inherit' | 'gateway' | 'direct' +export type GatewayInterfaceMode = 'inherit' | 'litellm_shim' | 'native_openai' + +export type EffectiveCredentialRouting = + | 'inherit' + | 'direct' + | 'gateway' + | 'bifrost' + | 'litellm_proxy' + +export interface Integration { + id: string + organization_id: string + platform: IntegrationPlatform + name?: string | null + public_key?: string | null + is_active: boolean + /** True if this row is the default credential for (org, platform). */ + is_default?: boolean + routing_mode?: CredentialRoutingMode + effective_routing?: EffectiveCredentialRouting + created_at: string + updated_at: string + last_tested_at?: string | null +} + +export interface IntegrationCreate { + platform: IntegrationPlatform + api_key: string + public_key?: string + name?: string | null + routing_mode?: CredentialRoutingMode + /** Mark the new credential as the default for (org, platform). */ + is_default?: boolean +} + +// VoiceBundle Types +export enum ModelProvider { + OPENAI = 'openai', + ANTHROPIC = 'anthropic', + GOOGLE = 'google', + XAI = 'xai', + FIREWORKS = 'fireworks', + COHERE = 'cohere', + MISTRAL = 'mistral', + META = 'meta', + TOGETHER = 'together', + PERPLEXITY = 'perplexity', + AZURE = 'azure', + AWS = 'aws', + DEEPGRAM = 'deepgram', + CARTESIA = 'cartesia', + ELEVENLABS = 'elevenlabs', + MURF = 'murf', + CUSTOM = 'custom', + SARVAM = 'sarvam', + VOICEMAKER = 'voicemaker', + SMALLEST = 'smallest', +} + +// AI Provider Types +export interface AIProvider { + id: string + provider: ModelProvider + api_key?: string | null + name?: string | null + endpoint_url?: string | null + is_active: boolean + /** True if this row is the default credential for (org, provider). */ + is_default?: boolean + routing_mode?: CredentialRoutingMode + gateway_model?: string | null + gateway_interface?: GatewayInterfaceMode + gateway_base_url?: string | null + gateway_auth_header?: string | null + gateway_auth_secret_env?: string | null + has_gateway_auth_secret?: boolean + gateway_extra_headers?: Record | null + /** True when provider secrets are resolved by the Bifrost gateway. */ + gateway_managed?: boolean + effective_routing?: EffectiveCredentialRouting + effective_gateway_interface?: 'litellm_shim' | 'native_openai' + created_at: string + updated_at: string + last_tested_at?: string | null +} + +export interface AIProviderCreate { + provider: ModelProvider + api_key?: string | null + name?: string | null + endpoint_url?: string | null + routing_mode?: CredentialRoutingMode + gateway_model?: string | null + gateway_interface?: GatewayInterfaceMode + gateway_base_url?: string | null + gateway_auth_header?: string | null + gateway_auth_secret_env?: string | null + gateway_auth_secret?: string | null + gateway_extra_headers?: Record | null + /** Mark the new credential as the default for (org, provider). */ + is_default?: boolean +} + +export interface AIProviderUpdate { + api_key?: string | null + name?: string | null + endpoint_url?: string | null + is_active?: boolean + routing_mode?: CredentialRoutingMode + gateway_model?: string | null + gateway_interface?: GatewayInterfaceMode + gateway_base_url?: string | null + gateway_auth_header?: string | null + gateway_auth_secret_env?: string | null + gateway_auth_secret?: string | null + clear_gateway_auth_secret?: boolean + gateway_extra_headers?: Record | null +} + +export enum VoiceBundleType { + STT_LLM_TTS = 'stt_llm_tts', + S2S = 's2s', +} + +export interface VoiceBundle { + id: string + name: string + description?: string | null + bundle_type: VoiceBundleType + stt_provider?: ModelProvider | null + stt_model?: string | null + /** + * Optional explicit AIProvider/Integration row id for STT. When null the + * runtime resolver picks the default credential for stt_provider. + */ + stt_credential_id?: string | null + llm_provider?: ModelProvider | null + llm_model?: string | null + llm_temperature?: number | null + llm_max_tokens?: number | null + llm_config?: Record | null + llm_credential_id?: string | null + tts_provider?: ModelProvider | null + tts_model?: string | null + tts_voice?: string | null + tts_config?: Record | null + tts_credential_id?: string | null + s2s_provider?: ModelProvider | null + s2s_model?: string | null + s2s_config?: Record | null + s2s_credential_id?: string | null + extra_metadata?: Record | null + is_active: boolean + created_at: string + updated_at: string + created_by?: string | null +} + +export interface VoiceBundleCreate { + name: string + description?: string | null + bundle_type?: VoiceBundleType + stt_provider?: ModelProvider | null + stt_model?: string | null + stt_credential_id?: string | null + llm_provider?: ModelProvider | null + llm_model?: string | null + llm_temperature?: number | null + llm_max_tokens?: number | null + llm_config?: Record | null + llm_credential_id?: string | null + tts_provider?: ModelProvider | null + tts_model?: string | null + tts_voice?: string | null + tts_config?: Record | null + tts_credential_id?: string | null + s2s_provider?: ModelProvider | null + s2s_model?: string | null + s2s_config?: Record | null + s2s_credential_id?: string | null + extra_metadata?: Record | null +} + +// Test Agent Types +export interface AgentPhoneAssignmentConflict { + agent_id: string + agent_name: string + phone_number: string +} + +export interface AgentPhoneAssignmentCheckResponse { + available: boolean + phone_number?: string | null + conflict?: AgentPhoneAssignmentConflict | null +} + +export interface TestAgent { + id: string + agent_id?: string | null + name: string + phone_number?: string | null + telephony_phone_number_id?: string | null + language: string + description: string | null + prompt_variables?: Record | null + silence_hangup_secs?: number + call_type: string + call_medium: string + voice_bundle_id?: string | null + voice_ai_integration_id?: string | null + voice_ai_agent_id?: string | null + provider_prompt?: string | null + provider_prompt_synced_at?: string | null + created_at: string + updated_at: string +} + +// Test Agent Conversation Types +export interface TestAgentConversation { + id: string + organization_id: string + agent_id: string + persona_id: string + scenario_id: string + voice_bundle_id: string + status: string + live_transcription?: Array<{ + speaker: string + text: string + timestamp: number + audio_segment_key?: string + }> | null + conversation_audio_key?: string | null + full_transcript?: string | null + started_at: string + ended_at?: string | null + duration_seconds?: number | null + conversation_metadata?: Record | null + created_at: string + updated_at: string + created_by?: string | null +} + +export interface TestAgentConversationCreate { + agent_id: string + persona_id: string + scenario_id: string + voice_bundle_id: string + conversation_metadata?: Record | null +} + +export interface TestAgentConversationUpdate { + status?: string | null + live_transcription?: Array> | null + full_transcript?: string | null + conversation_metadata?: Record | null +} + +export interface VoiceBundleUpdate { + name?: string + description?: string | null + stt_provider?: ModelProvider + stt_model?: string + stt_credential_id?: string | null + llm_provider?: ModelProvider + llm_model?: string + llm_temperature?: number | null + llm_max_tokens?: number | null + llm_config?: Record | null + llm_credential_id?: string | null + tts_provider?: ModelProvider + tts_model?: string + tts_voice?: string | null + tts_config?: Record | null + tts_credential_id?: string | null + s2s_provider?: ModelProvider | null + s2s_model?: string | null + s2s_config?: Record | null + s2s_credential_id?: string | null + extra_metadata?: Record | null + is_active?: boolean +} + +// Data Sources Types +export interface S3ConnectionTest { + bucket_name: string + region?: string + access_key_id: string + secret_access_key: string + endpoint_url?: string | null +} + +export interface S3ConnectionTestResponse { + success: boolean + message: string + bucket_name?: string | null +} + +export interface S3FileInfo { + key: string + filename: string + size: number + last_modified: string +} + +export interface S3FolderInfo { + name: string + path: string +} + +export interface S3ListFilesResponse { + files: S3FileInfo[] + total: number + prefix?: string | null +} + +export interface S3BrowseResponse { + folders: S3FolderInfo[] + files: S3FileInfo[] + current_path: string + organization_id: string +} + +export interface S3Status { + enabled: boolean + provider?: 's3' | 'gcs' | string + error?: string | null +} + +// Alert Types +export enum AlertMetricType { + NUMBER_OF_CALLS = 'number_of_calls', + CALL_DURATION = 'call_duration', + ERROR_RATE = 'error_rate', + SUCCESS_RATE = 'success_rate', + LATENCY = 'latency', + CUSTOM = 'custom', +} + +export enum AlertAggregation { + SUM = 'sum', + AVG = 'avg', + COUNT = 'count', + MIN = 'min', + MAX = 'max', +} + +export enum AlertOperator { + GREATER_THAN = '>', + LESS_THAN = '<', + GREATER_THAN_OR_EQUAL = '>=', + LESS_THAN_OR_EQUAL = '<=', + EQUAL = '=', + NOT_EQUAL = '!=', +} + +export enum AlertNotifyFrequency { + IMMEDIATE = 'immediate', + HOURLY = 'hourly', + DAILY = 'daily', + WEEKLY = 'weekly', +} + +export enum AlertStatus { + ACTIVE = 'active', + PAUSED = 'paused', + DISABLED = 'disabled', +} + +export enum AlertHistoryStatus { + TRIGGERED = 'triggered', + NOTIFIED = 'notified', + ACKNOWLEDGED = 'acknowledged', + RESOLVED = 'resolved', +} + +export interface Alert { + id: string + organization_id: string + name: string + description?: string | null + metric_type: AlertMetricType + aggregation: AlertAggregation + operator: AlertOperator + threshold_value: number + time_window_minutes: number + agent_ids?: string[] | null + notify_frequency: AlertNotifyFrequency + notify_emails?: string[] | null + notify_webhooks?: string[] | null + status: AlertStatus + created_at: string + updated_at: string + created_by?: string | null +} + +export interface AlertCreate { + name: string + description?: string | null + metric_type?: AlertMetricType + aggregation?: AlertAggregation + operator?: AlertOperator + threshold_value: number + time_window_minutes?: number + agent_ids?: string[] | null + notify_frequency?: AlertNotifyFrequency + notify_emails?: string[] + notify_webhooks?: string[] +} + +export interface AlertUpdate { + name?: string + description?: string | null + metric_type?: AlertMetricType + aggregation?: AlertAggregation + operator?: AlertOperator + threshold_value?: number + time_window_minutes?: number + agent_ids?: string[] | null + notify_frequency?: AlertNotifyFrequency + notify_emails?: string[] + notify_webhooks?: string[] + status?: AlertStatus +} + +export interface AlertHistoryItem { + id: string + organization_id: string + alert_id: string + triggered_at: string + triggered_value: number + threshold_value: number + status: AlertHistoryStatus + notified_at?: string | null + notification_details?: Record | null + acknowledged_at?: string | null + acknowledged_by?: string | null + resolved_at?: string | null + resolved_by?: string | null + resolution_notes?: string | null + context_data?: Record | null + created_at: string + updated_at: string + alert?: Alert +} + + +// Cron Job Types +export enum CronJobStatus { + ACTIVE = 'active', + PAUSED = 'paused', + COMPLETED = 'completed', +} + +export interface CronJob { + id: string + organization_id: string + name: string + cron_expression: string + timezone: string + max_runs: number + current_runs: number + evaluator_ids: string[] + status: CronJobStatus + next_run_at?: string | null + last_run_at?: string | null + created_at: string + updated_at: string + created_by?: string | null +} + +export interface CronJobCreate { + name: string + cron_expression: string + timezone: string + max_runs: number + evaluator_ids: string[] +} + +export interface CronJobUpdate { + name?: string + cron_expression?: string + timezone?: string + max_runs?: number + evaluator_ids?: string[] + status?: CronJobStatus +} + +// --- Call Imports --- + +/** + * Lifecycle for a call-import batch. + * + * - ``uploaded`` : file landed in S3, no mapping yet. + * - ``mapped`` : user picked a schema + sheet + column mapping; no + * rows materialised yet, no worker enqueued. + * - ``processing`` : rows materialised + workers enqueued. + * - ``pending`` : transient state used by the legacy one-shot + * ``POST /upload`` endpoint just before transitioning + * to ``processing``. + */ +export type CallImportStatus = + | 'pending' + | 'uploaded' + | 'mapped' + | 'processing' + | 'completed' + | 'partial' + | 'failed' + | 'deleting' + +export type CallImportRowStatus = + | 'pending' + | 'processing' + | 'completed' + | 'failed' + +/** Where the value in `transcript` came from. */ +export type CallImportTranscriptSource = + | 'csv' + | 'transcribed' + | 'edited' + | null +/** Lifecycle status for the post-hoc transcription workflow itself. */ +export type CallImportTranscriptStatus = + | 'idle' + | 'pending' + | 'running' + | 'completed' + | 'failed' + | null + +/** + * Which transcript an evaluation run scored against. + * - `production`: the CSV-supplied value on `CallImportRow.transcript`. + * - `diarised`: the worker-produced value on `CallImportRow.diarised_transcript`. + */ +export type CallImportEvaluationTranscriptSource = 'production' | 'diarised' + +/** + * One contiguous turn inside ``CallImportRow.diarised_segments``. + * + * The diarisation worker rewrites each pyannote ``Speaker N`` label + * into ``agent`` / ``user`` (first speaker = agent heuristic). Anything + * beyond two distinct speakers keeps a generic ``speaker_N`` label so + * multi-party recordings still render every voice. + */ +export interface CallImportDiarisedSegment { + speaker: string + text: string + start: number + end: number + /** Original pyannote label (``Speaker 1`` / ``Speaker 2`` / ...). */ + raw_speaker: string +} + +export interface CallImportRow { + id: string + row_index: number + /** Mandatory identifier per row. Renamed from ``external_call_id``. */ + conversation_id: string + recording_url: string | null + recording_date: string | null + /** Production transcript — the value supplied via the CSV upload. */ + transcript: string | null + /** Provenance of the stored production transcript (csv = CSV upload, edited = manual edit). */ + transcript_source: CallImportTranscriptSource + /** Legacy: provider recorded by the original transcription worker before the split. */ + transcript_provider: string | null + transcript_model: string | null + transcript_status: CallImportTranscriptStatus + transcript_error: string | null + transcribed_at: string | null + /** Diarised transcript — produced by the post-hoc diarisation worker. */ + diarised_transcript: string | null + /** Provider used by the diarisation worker (e.g. "deepgram"). */ + diarised_transcript_provider: string | null + diarised_transcript_model: string | null + diarised_transcript_status: CallImportTranscriptStatus + diarised_transcript_error: string | null + diarised_at: string | null + /** + * Structured speaker turns produced by the diarisation worker. Each + * entry is a single contiguous turn shaped as + * `{ speaker: 'agent' | 'user' | 'speaker_N', text, start, end, + * raw_speaker }`. ``diarised_transcript`` is a rendered + * `: ` view of this list with + * ``diarised_speaker_swap`` applied. ``null`` on legacy rows that + * were diarised before structured turns were persisted (or when the + * STT provider didn't surface segments). + */ + diarised_segments: CallImportDiarisedSegment[] | null + /** + * When ``true`` the ``agent`` <-> ``user`` mapping inside + * ``diarised_segments`` is inverted in the rendered transcript / + * CSV export. The worker writes the canonical mapping using a + * "first speaker is the agent" heuristic; reviewers can flip the + * toggle from the row detail panel without re-running diarisation. + */ + diarised_speaker_swap: boolean + /** + * LLM that turned the STT plain-text output into structured + * ``diarised_segments``. NULL on legacy rows (pre-LLM-diariser). + */ + diarised_llm_provider: string | null + diarised_llm_model: string | null + /** + * Exact prompt the LLM diariser ran with. Useful for the modal to + * pre-fill its textarea when the operator wants to iterate on a + * previously-diarised row. + */ + diarised_prompt: string | null + /** + * Diarisation pipeline that produced this row's turns. + * - `stt_llm` (default) — two-stage STT then LLM diariser. + * - `llm_only` — single-stage multimodal LLM (audio in). + * Read-only; written by the worker on each diarisation. + */ + transcribe_mode?: 'stt_llm' | 'llm_only' + /** + * Per-row preservation of the mapped source cells. Values land here + * as whatever type the schema parameter coerced them to — + * strings (text / url / conversation_id / recording_url / + * recording_date / transcript / datetime), numbers, booleans, or + * ``null`` for blanks. Always + * coerce with ``String(value)`` before string operations. + */ + raw_columns: Record | null + status: CallImportRowStatus + recording_s3_key: string | null + recording_content_type: string | null + recording_size_bytes: number | null + error_message: string | null + attempts: number + created_at: string + updated_at: string +} + +export interface CallImportTag { + id: string + name: string + color: string | null + created_at: string + updated_at: string +} + +/** + * Parameter type tag on a Call Import schema parameter. + * + * - ``conversation_id``: mandatory identifier (one per schema). + * - ``recording_url``: feeds ``CallImportRow.recording_url``. + * - ``recording_date``: date-only call recording date used for reports. + * - ``transcript``: feeds ``CallImportRow.transcript``. + * - ``text`` / ``number`` / ``boolean`` / ``datetime`` / ``url``: + * generic typed fields preserved per row in ``raw_columns`` and + * surfaced in the evaluation export under the parameter's name. + */ +export type CallImportSchemaParameterType = + | 'conversation_id' + | 'recording_url' + | 'recording_date' + | 'transcript' + | 'text' + | 'number' + | 'boolean' + | 'datetime' + | 'url' + +export interface CallImportSchemaParameter { + id?: string + name: string + type: CallImportSchemaParameterType + description: string | null + is_required: boolean + ordering?: number +} + +export interface CallImportSchema { + id: string + organization_id: string + workspace_id: string + name: string + description: string | null + parameters: CallImportSchemaParameter[] + /** How many CallImport batches reference this schema. */ + usage_count: number + created_at: string + updated_at: string +} + +export interface CallImportSchemaListResponse { + items: CallImportSchema[] + total: number +} + +export interface CallImportSchemaCreate { + name: string + description?: string | null + parameters: Array> +} + +export interface CallImportSchemaUpdate { + name?: string + description?: string | null + parameters?: Array> +} + +/** + * In-org Workspace - the active workspace scopes call imports and + * metrics in the UI. The org's Default workspace is auto-seeded by + * migration 033 and cannot be deleted. + */ +export interface Workspace { + id: string + organization_id: string + name: string + slug: string + is_default: boolean + created_at: string + updated_at: string + role_id?: string | null + role_name?: string | null + capabilities?: string[] +} + +export interface WorkspaceRole { + id: string + organization_id: string + name: string + description?: string | null + capabilities: string[] + is_system: boolean + created_at: string + updated_at: string +} + +export interface WorkspaceMember { + id: string + workspace_id: string + user_id: string + role_id: string + role_name: string + user_email: string + user_name?: string | null + added_by_user_id?: string | null + created_at: string +} + +export interface CapabilityInfo { + key: string + label: string +} + +export interface CapabilityDomain { + key: string + label: string + capabilities: CapabilityInfo[] +} + +export interface WorkspaceRoleCreate { + name: string + description?: string | null + capabilities: string[] +} + +export interface WorkspaceRoleUpdate { + name?: string + description?: string | null + capabilities?: string[] +} + +export interface CallImportSourceRowSkip { + source_row: number + reason: string + message: string +} + +export interface CallImport { + id: string + organization_id: string + /** Workspace this import belongs to. */ + workspace_id: string + /** + * Telephony provider key. ``null`` until the IMPORT stage in the + * staged flow (which is the first step that knows the provider). + * Always populated on post-import batches and on legacy one-shot + * uploads. + */ + provider: string | null + telephony_integration_id: string | null + original_filename: string | null + /** + * For Excel uploads, which worksheet this batch came from. ``null`` + * for CSV uploads (CSV files have no sheet concept) and for any + * imports created before multi-sheet support landed. + */ + sheet_name: string | null + /** Optional free-text dataset label (high-level segregation filter). */ + dataset: string | null + /** Tags currently attached to this import. Empty array if untagged. */ + tags: CallImportTag[] + /** + * Reusable Input Parameter schema the batch was uploaded against. + * NULL on legacy batches uploaded before the schema-driven flow shipped. + */ + schema_id: string | null + /** + * Schema-driven mapping: ``{parameter_name: csv_header}``. Empty on + * legacy batches; check ``column_mapping`` / ``extra_columns`` / + * ``custom_column_mapping`` instead for those. + */ + parameter_mapping: Record + /** Legacy free-form mapping kept for batches uploaded before schemas. */ + column_mapping: Record + /** Legacy extra-column list kept for backwards-compat. */ + extra_columns: string[] + /** Legacy uploader-named columns kept for backwards-compat. */ + custom_column_mapping: Record + /** + * Source headers the uploader explicitly skipped, captured at the + * MAP stage. Empty for legacy one-shot uploads where the value was + * ephemeral. + */ + skipped_columns: string[] + /** + * Source rows skipped at parse time (missing/invalid conversation ID or URL). + */ + source_row_skips?: CallImportSourceRowSkip[] + /** S3 key for the staged source file. ``null`` on legacy batches. */ + source_s3_key: string | null + /** ``'csv'`` / ``'xlsx'`` for staged files, or ``'audio'`` for manual uploads. */ + source_format: string | null + source_size_bytes: number | null + source_content_type: string | null + /** + * Snapshot of the file's sheets + headers captured at UPLOAD time so + * the MAP UI can render without re-fetching the source from S3. + * ``null`` on legacy batches. + */ + available_sheets: CallImportPreviewSheet[] | null + total_rows: number + completed_rows: number + failed_rows: number + status: CallImportStatus + error_message: string | null + created_at: string + updated_at: string + created_by_email?: string | null + last_updated_by_email?: string | null +} + +export interface CallImportDetail extends CallImport { + rows: CallImportRow[] + /** + * Total row count *after* applying the optional ``q`` search filter. + * ``null`` when no filter is active — paginate against ``total_rows`` + * in that case. + */ + filtered_total_rows: number | null + /** + * Batch-wide aggregates of ``CallImportRow.diarised_transcript_status``. + * The ``idle`` bucket (rows never touched by the transcribe/diarise + * worker) is implicit: ``total_rows - (pending + running + completed + * + failed)``. Lets the UI render a transcribe-and-diarise progress + * bar without paginating through every row. + */ + diarised_pending_rows: number + diarised_running_rows: number + diarised_completed_rows: number + diarised_failed_rows: number +} + +export interface CallImportListResponse { + items: CallImport[] + total: number + page: number + page_size: number +} + +export interface CallImportUploadResponse { + id: string + total_rows: number + status: CallImportStatus + dataset: string | null + tags: CallImportTag[] + message: string +} + +/** One worksheet (or one CSV file synthesized as a single sheet). */ +export interface CallImportPreviewSheet { + /** Sheet name for xlsx; filename for csv. */ + name: string + /** Column headers from the first non-empty row. */ + headers: string[] + /** Approximate count of data rows (excludes the header row). */ + row_count: number +} + +/** + * Sheets / headers extracted server-side from an uploaded CSV or Excel + * workbook. Drives the modal's column-mapping UI without forcing the + * frontend to parse the file itself. + */ +export interface CallImportPreviewResponse { + /** ``'csv'`` or ``'xlsx'``. */ + format: 'csv' | 'xlsx' + sheets: CallImportPreviewSheet[] +} + +export type MetricSelectionMode = 'single_choice' | 'multi_label' + +export interface CallImportMetricSummary { + id: string + name: string + metric_type: string | null + description: string | null + parent_metric_id?: string | null + selection_mode?: MetricSelectionMode | null + /** Only meaningful on multi_label parents; gates the Discovered + * Labels panel on the Flow tab. Defaults to false. */ + allow_discovery?: boolean +} + +/** Per-metric LLM override (provider+model+optional credential + generation params). */ +export interface CallImportEvaluationLLMOverride { + provider?: string | null + model?: string | null + credential_id?: string | null + llm_config?: LLMGenerationConfig | null +} + +export interface CallImportEvaluation { + id: string + call_import_id: string + organization_id: string + /** User-supplied label for the run; null when not named. */ + name: string | null + selected_metric_ids: string[] + /** parent_id -> [child_id, ...] snapshot captured at run time. */ + selected_metric_groups?: Record | null + metrics: CallImportMetricSummary[] + status: 'pending' | 'running' | 'completed' | 'partial' | 'failed' + total_rows: number + completed_rows: number + failed_rows: number + error_message: string | null + /** Run-level LLM provider chosen by the user (null = legacy default). */ + llm_provider: string | null + llm_model: string | null + llm_credential_id: string | null + llm_config?: LLMGenerationConfig | null + metric_llm_overrides: Record | null + stt_provider: string | null + stt_model: string | null + stt_credential_id: string | null + /** + * Run-level LLM diariser config used when the worker auto-diarises + * rows that are missing a diarised transcript. + */ + diarisation_llm_provider?: string | null + diarisation_llm_model?: string | null + diarisation_llm_credential_id?: string | null + diarisation_prompt?: string | null + /** + * Diarisation pipeline shape this run was created with. + * - `stt_llm` (default) — STT then an LLM diariser over the text. + * - `llm_only` — audio fed directly to a multimodal diariser LLM. + * Surfaced so the retry / re-run UI can preselect the right mode. + */ + transcribe_mode?: 'stt_llm' | 'llm_only' + /** + * Which transcript column this run scored against. + * Defaults to `production` on legacy runs. + */ + transcript_source: CallImportEvaluationTranscriptSource + /** + * Other evaluation ids created in the same Run Evaluation request. + * Populated only on the POST response when the user ticked both + * Production and Diarised. Empty array on all other reads. + */ + sibling_evaluation_ids: string[] + started_at: string | null + finished_at: string | null + created_at: string + updated_at: string + created_by_email?: string | null + last_updated_by_email?: string | null + /** + * Cached LLM-generated TLDR rendered above the Visualizations tab. + * Populated lazily via ``POST /evaluations/{id}/insights``; null on + * runs the user has not summarised yet. + */ + tldr_summary?: EvaluationTldrSummary | null + user_insights?: EvaluationUserInsightsState | null + metric_clusters?: EvaluationMetricClustersState | null + /** + * True when the user opted into top-level metric discovery on the + * Run Evaluation modal. Gates the Discovered metrics panel on the + * evaluation detail Flow tab. + */ + discover_new_metrics?: boolean + /** + * Set while a bulk background operation (abort, force-fail, retry) is + * still running. The UI disables other mutating actions until cleared. + */ + bulk_operation?: 'abort' | 'force_fail_pending' | 'retry' | null +} + +/** + * LLM-generated narrative + bullet patterns for a single evaluation + * run. Cached on the evaluation row so re-opening the Visualizations + * tab doesn't auto-burn LLM tokens. ``is_stale`` is computed by the + * backend at read time when ``completed_rows`` has grown since the + * summary was generated. + */ +export interface EvaluationTldrSummary { + narrative: string + patterns: string[] + metric_insights?: Record + generated_at: string + generated_at_completed_rows: number + provider?: string | null + model?: string | null + is_stale: boolean +} + +export interface UserInsightCategory { + label: string + count: number + share_pct: number +} + +export interface UserInsightEvidenceTurn { + speaker: string + text: string +} + +export interface UserInsightEvidence { + conversation_id?: string | null + quote: string + turns?: UserInsightEvidenceTurn[] +} + +export interface EvaluationUserInsightItem { + id: string + title: string + categories: UserInsightCategory[] + observation: string + evidence: UserInsightEvidence +} + +export interface EvaluationUserInsightsState { + status: 'idle' | 'running' | 'completed' | 'failed' + insights: EvaluationUserInsightItem[] + overview?: string | null + generated_at?: string | null + generated_at_completed_rows: number + progress?: { completed_llm_calls: number; total_llm_calls: number } | null + provider?: string | null + model?: string | null + llm_calls_used: number + max_llm_calls?: number | null + error_message?: string | null + is_stale: boolean +} + +export type MetricClusterGapLabel = + | 'LOGIC_GAP' + | 'UNDERSPEC' + | 'EXISTS_NO_TRIGGER' + | 'MISSING' + +export interface MetricSubCluster { + label: string + count: number + share_pct: number +} + +export interface MetricClusterEvidenceTurn { + speaker: string + text: string +} + +export interface MetricClusterEvidence { + conversation_id?: string | null + evaluation_row_id?: string | null + quote: string + turns?: MetricClusterEvidenceTurn[] +} + +export interface MetricCluster { + id: string + label: string + gap_label: MetricClusterGapLabel + level: number + count: number + share_pct: number + sub_clusters: MetricSubCluster[] + observation: string + failure_reason?: string + evidence: MetricClusterEvidence + is_discovered: boolean +} + +export interface MetricClusterGroup { + metric_id: string + metric_name: string + flagged_count: number + failure_reason?: string + clusters: MetricCluster[] +} + +export interface DiscoveredProblemCluster { + id: string + label: string + gap_label: MetricClusterGapLabel + count: number + share_pct: number + observation: string + failure_reason?: string + evidence: MetricClusterEvidence +} + +export interface RcaRepeatedPatternRow { + metric_id: string + metric_name: string + top_rca_patterns: string + evidence_share_pct: number + evidence_calls: number + evidence_cluster_count?: number + failure_reason: string +} + +export interface RcaMetricHotspotRow { + metric_id: string + metric_name: string + description: string + metric_rate_pct: number + flagged_calls: number +} + +export interface RcaPromptAreaRow { + label: string + share_pct: number + gap_label: MetricClusterGapLabel +} + +export interface MetricClustersRcaSummary { + total_clusters: number + total_clustered_instances: number + total_flagged_instances?: number + analysed_calls: number + repeated_patterns: RcaRepeatedPatternRow[] + metric_hotspots: RcaMetricHotspotRow[] + prompt_areas: RcaPromptAreaRow[] +} + +export interface MetricFailurePolicy { + metric_id: string + failure_values: string[] + failure_child_names?: string[] + numeric_rule?: { op: 'lt' | 'lte' | 'gt' | 'gte'; threshold: number } | null +} + +export interface MetricFailurePolicyValueCount { + label: string + count: number +} + +export interface MetricFailurePolicyMetricPreview { + metric_id: string + metric_name: string + metric_type?: string | null + selection_mode?: string | null + is_multi_label_parent: boolean + value_counts: MetricFailurePolicyValueCount[] + child_names: string[] + row_count_by_value: Record + suggested_policy: MetricFailurePolicy + effective_policy: MetricFailurePolicy +} + +export interface MetricFailurePoliciesResponse { + previews: MetricFailurePolicyMetricPreview[] + policies: Record + source: 'inferred' | 'user' + updated_at?: string | null +} + +export interface MetricClusterEligibleRow { + evaluation_row_id: string + conversation_id?: string | null + row_index?: number | null + flagged_metric_names: string[] +} + +export interface MetricClusterEligibleRowsResponse { + items: MetricClusterEligibleRow[] + total: number +} + +export interface EvaluationMetricClustersState { + status: 'idle' | 'running' | 'completed' | 'failed' | 'cancelled' + groups: MetricClusterGroup[] + discovered_problems: DiscoveredProblemCluster[] + overview?: string | null + generated_at?: string | null + generated_at_completed_rows: number + progress?: { completed_llm_calls: number; total_llm_calls: number } | null + provider?: string | null + model?: string | null + llm_calls_used: number + max_llm_calls?: number | null + error_message?: string | null + is_stale: boolean + selected_evaluation_row_ids?: string[] + failure_policies?: Record + failure_policies_source?: 'inferred' | 'user' + failure_policies_updated_at?: string | null + rca_summary?: MetricClustersRcaSummary | null +} + +export interface AgentFlowNode { + id: string + label: string + node_type: 'start' | 'decision' | 'action' | 'terminal' + position_x?: number | null + position_y?: number | null + prompt_excerpt?: string | null + start_offset?: number | null + end_offset?: number | null +} + +export interface AgentFlowEdge { + source: string + target: string + condition?: string | null +} + +export interface AgentFlowGraph { + nodes: AgentFlowNode[] + edges: AgentFlowEdge[] + generated_at?: string | null + provider?: string | null + model?: string | null + layout_saved_at?: string | null + prompt_content_hash?: string | null + mapping_error?: string | null + generation_error?: string | null +} + +export interface ImportedAgent { + id: string + organization_id: string + name: string + description: string | null + content: string + tags: string[] | null + current_version: number + agent_flowchart?: AgentFlowGraph | null + agent_flowchart_status?: string | null + created_at: string + updated_at: string + created_by: string | null +} + +export interface ImportedAgentDetail extends ImportedAgent { + versions: PromptPartialVersion[] +} + +export interface MetricPartialChild { + name: string + description: string + example: string +} + +export interface MetricPartialContent { + schema_version: 1 + metric_kind: 'single' | 'category' + description: string + children?: MetricPartialChild[] +} + +export interface MetricPartial { + id: string + organization_id: string + name: string + description: string | null + content: string + tags: string[] | null + current_version: number + created_at: string + updated_at: string + created_by: string | null +} + +export interface MetricPartialDetail extends MetricPartial { + versions: PromptPartialVersion[] +} + +export interface PromptPartialVersion { + id: string + prompt_partial_id: string + version: number + content: string + change_summary: string | null + created_at: string + created_by: string | null +} + +export interface PromptImprovementSuggestion { + id: string + metric_id: string + metric_name: string + cluster_id: string + cluster_label: string + gap_label: MetricClusterGapLabel + share_pct: number + priority: 'high' | 'medium' | 'low' + change_type?: 'edit' | 'add' + target_section: string + anchor_excerpt?: string + current_gap: string + suggested_text: string + rationale: string + flow_node_id?: string + flow_node_label?: string +} + +export interface EvaluationPromptImprovementsState { + status: 'idle' | 'running' | 'completed' | 'failed' + imported_agent_id?: string | null + imported_agent_name?: string | null + suggestions: PromptImprovementSuggestion[] + overview?: string | null + generated_at?: string | null + generated_at_completed_rows: number + provider?: string | null + model?: string | null + error_message?: string | null + is_stale: boolean +} + +export interface MetricPeriodDelta { + label: string + detail: string + why?: string | null +} + +export interface CallImportEvaluationListResponse { + items: CallImportEvaluation[] + total: number +} + +export interface CallImportEvaluationBaselineCandidate { + evaluation_id: string + name: string + dataset: string + period_label: string | null + period_start: string | null + period_end: string | null + period_display: string + completed_rows: number + created_at: string + is_default: boolean +} + +export interface CallImportEvaluationBaselineCandidatesResponse { + items: CallImportEvaluationBaselineCandidate[] + default_evaluation_id: string | null +} + +export interface CallImportEvaluationRow { + id: string + evaluation_id: string + call_import_row_id: string + row_index: number | null + /** Mandatory identifier from the source batch (renamed from ``external_call_id``). */ + conversation_id: string | null + transcript: string | null + raw_columns: Record | null + recording_url: string | null + recording_date: string | null + /** + * S3 object key for the downloaded recording. Prefer this over + * ``recording_url`` for playback — we resolve it to a presigned URL + * so audio plays from our storage instead of the (often expired) + * provider URL. + */ + recording_s3_key: string | null + diarised_transcript_status?: string | null + diarised_transcript_error?: string | null + status: 'pending' | 'running' | 'completed' | 'failed' | 'skipped' + metric_scores: Record + error_message: string | null + started_at: string | null + finished_at: string | null + created_at: string + updated_at: string +} + +export interface CallImportEvaluationRowListResponse { + items: CallImportEvaluationRow[] + total: number + page: number + page_size: number +} + +// --- Retry (re-enqueue failed rows on an existing evaluation run) --- + +export interface CallImportEvaluationRetryRequest { + /** + * Restrict the retry to a specific subset of evaluation rows. + * When omitted, every row with status='failed' in this run is + * re-enqueued. + */ + eval_row_ids?: string[] + + /** + * Optional LLM overrides. When provided, persisted onto the run so + * future retries default to the new config. ``llm_provider`` and + * ``llm_model`` must be sent together — the backend 400s on + * half-configured input. + */ + llm_provider?: string + llm_model?: string + llm_credential_id?: string | null + + /** + * Optional STT overrides (only meaningful when the run scores the + * diarised transcript). Same paired-field rule as LLM. + */ + stt_provider?: string + stt_model?: string + stt_credential_id?: string | null + + /** + * When true, wipe the diarised transcript on every retried row so + * the (possibly new) STT runs from scratch. Only takes effect for + * diarised runs that have STT config. + */ + transcribe_overwrite?: boolean +} + +export interface CallImportEvaluationRetrySkippedItem { + eval_row_id: string + /** + * Why this row was not re-enqueued. Known values: + * - 'unknown' (id not in this run) + * - 'in_progress' (status is pending/running) + * - 'completed' (already successful) + * - 'source_row_missing' + */ + reason: 'unknown' | 'in_progress' | 'completed' | 'source_row_missing' +} + +export interface CallImportEvaluationRetryResponse { + requeued: number + /** + * Of those, how many were chained through a diarisation task first + * because the diarised transcript was missing. + */ + transcribe_requeued: number + skipped: CallImportEvaluationRetrySkippedItem[] +} + +export interface CallImportEvaluationBulkActionResponse { + accepted: boolean + target_count: number + evaluation_id: string +} + +// --- Diarization / transcription --- + +export interface CallImportTranscribeRequest { + /** + * Diarisation pipeline shape. + * - `stt_llm` (default) — STT produces plain text, then an LLM + * diariser splits it into agent/user turns. STT fields required. + * - `llm_only` — skip STT entirely and feed the audio bytes + * directly to a multimodal `diarization_llm_*` model along with + * `diarization_prompt`. STT fields MUST be omitted in this mode. + */ + mode?: 'stt_llm' | 'llm_only' + /** Required when `mode === 'stt_llm'`; must be null/omitted in `llm_only`. */ + stt_provider?: string | null + /** Required when `mode === 'stt_llm'`; must be null/omitted in `llm_only`. */ + stt_model?: string | null + credential_id?: string | null + language?: string | null + only_missing?: boolean + overwrite_existing?: boolean + row_ids?: string[] + /** + * LLM diariser. In `stt_llm` mode it splits the STT plain-text into + * agent/user turns; in `llm_only` mode it directly receives the + * audio along with `diarization_prompt`. Always required. + */ + diarization_llm_provider: string + diarization_llm_model: string + diarization_llm_credential_id?: string | null + /** + * Operator-supplied system prompt for the diariser LLM. NULL/empty + * means "fall back to the canonical default" (see + * ``getDiarisationDefaultPrompt``). + */ + diarization_prompt?: string | null +} + +export interface CallImportTranscribeResponse { + queued: number + skipped_rows: number + skipped_reason_counts: Record + accepted?: boolean +} + +export interface CallImportRowBulkDeleteResponse { + deleted: number + status?: 'completed' | 'accepted' +} + +export interface CallImportRetryFailedRowsResponse { + requeued: number + enqueue_failed: number + skipped: number +} + +export interface CallImportDiarisationPromptDefaultResponse { + prompt: string +} + +// --- Aggregation / visualization payloads --- + +export interface CallImportMetricHistogramBucket { + x0: number + x1: number + count: number +} + +export interface CallImportMetricValueCount { + label: string + count: number +} + +/** + * One unordered pair-count cell from a multi-label parent's + * co-occurrence matrix. ``a`` and ``b`` are child label names with + * ``a < b`` lexicographically; ``count`` is the number of rows on + * which both labels fired together. + */ +export interface CallImportMetricLabelPair { + a: string + b: string + count: number +} + +export interface CallImportMetricAggregate { + metric_id: string + metric_name: string + metric_type: string | null + metric_category?: 'quality' | 'user_insight' | string + /** + * True when the metric is a multi-label classifier parent. + * ``value_counts`` then lists per-child label tallies and one row + * may contribute to several labels, so the chart layout has to + * ignore the pie toggle (slices wouldn't sum to 100%) and the + * n-badge represents rows scored, not label occurrences. + */ + is_multi_label_parent?: boolean + count: number + skipped_count: number + error_count: number + mean: number | null + median: number | null + p25: number | null + p75: number | null + p95: number | null + min: number | null + max: number | null + stddev: number | null + histogram_buckets: CallImportMetricHistogramBucket[] + value_counts: CallImportMetricValueCount[] + /** + * Pairwise label intersections for multi-label parent metrics. + * Empty for everything else. The Visualizations tab reconstructs + * a square symmetric matrix from these unordered pairs to render + * the co-occurrence heatmap chart type. + */ + co_occurrence?: CallImportMetricLabelPair[] +} + +export interface CallImportEvaluationAggregateResponse { + evaluation_id: string + total_rows: number + completed_rows: number + failed_rows: number + metrics: CallImportMetricAggregate[] + period_deltas?: Record + baseline_evaluation_id?: string | null + failure_policies_source?: 'inferred' | 'user' | null +} + +export interface EvaluatorResultsAggregateResponse { + scope: string + suite_id?: string | null + agent_id?: string | null + scenario_id?: string | null + total_rows: number + completed_rows: number + failed_rows: number + metrics: CallImportMetricAggregate[] +} + +export interface CallImportInsightsRunPoint { + evaluation_id: string + name: string | null + created_at: string + mean: number | null + completed_rows: number +} + +export interface CallImportInsightsMetric { + metric_id: string + metric_name: string + metric_type: string | null + latest: CallImportMetricAggregate | null + trend: CallImportInsightsRunPoint[] +} + +export interface CallImportInsightsResponse { + call_import_id: string + total_rows: number + rows_with_transcript: number + rows_without_transcript: number + transcript_source_counts: Record + evaluation_count: number + metrics: CallImportInsightsMetric[] +} + +// --- Metrics hierarchy + flow visualization --- + +export interface MetricSummary { + id: string + organization_id: string + name: string + description: string | null + metric_type: string + metric_category?: 'quality' | 'user_insight' | string + trigger: string + enabled: boolean + is_default: boolean + metric_origin: string + supported_surfaces: string[] + enabled_surfaces: string[] + custom_data_type: string | null + custom_config: Record | null + tags: string[] | null + capture_rationale: boolean + parent_metric_id: string | null + selection_mode: MetricSelectionMode | null + allow_discovery?: boolean + /** + * When true, this metric is a "transcript-compare judge": at + * call-import evaluation time the worker feeds BOTH the production + * transcript and the diarised transcript to the LLM as a labeled + * pair, and the run's transcript_source toggle is ignored for this + * metric. Mutually exclusive with parent_metric_id and selection_mode + * — comparison metrics stay standalone. + */ + compare_transcripts?: boolean + children?: MetricSummary[] + created_at: string + updated_at: string + created_by: string | null +} + +export interface MetricChildDraft { + name: string + description?: string | null + enabled?: boolean + capture_rationale?: boolean | null + tags?: string[] | null +} + +export interface MetricCreateWithChildrenPayload { + name: string + description?: string | null + selection_mode: MetricSelectionMode + enabled?: boolean + supported_surfaces?: string[] + enabled_surfaces?: string[] + tags?: string[] | null + allow_discovery?: boolean + children: MetricChildDraft[] +} + +export interface MetricFlowNode { + id: string + label: string + count: number + is_terminal: boolean + is_discovered?: boolean +} + +export interface MetricFlowEdge { + source: string + target: string + count: number +} + +export interface MetricFlowResponse { + parent_metric_id: string + parent_metric_name: string + selection_mode: MetricSelectionMode | null + nodes: MetricFlowNode[] + edges: MetricFlowEdge[] + total_rows: number + rows_with_sequence: number +} + +export interface DiscoveredLabel { + key: string + name: string + description?: string | null + sample_rationale?: string | null + /** + * Up to 3 distinct LLM rationales captured for this candidate + * across rows. The Discovered Labels promote flow surfaces the + * first 2 as an ``Examples:`` block on the new sub-metric's + * rubric so the user starts with concrete cases in the prompt. + */ + examples?: string[] + count: number +} + +export interface DiscoveredLabelsResponse { + parent_metric_id: string + items: DiscoveredLabel[] +} + +/** + * One LLM-discovered candidate TOP-LEVEL metric aggregated across all + * rows of an evaluation. Mirrors :class:`DiscoveredLabel` but adds a + * ``suggested_type`` field — the LLM's guess at the best shape for + * the new metric — that the promote modal can pre-fill the type radio + * with. + */ +export interface DiscoveredMetric { + key: string + name: string + description?: string | null + suggested_type: 'boolean' | 'rating' | 'category' + sample_rationale?: string | null + examples?: string[] + count: number +} + +export interface DiscoveredMetricsResponse { + evaluation_id: string + items: DiscoveredMetric[] +} + +export interface ObservabilityCallAgent { + id: string + agent_id?: string | null + name: string +} + +export interface ObservabilityCallData { + startedAt?: string + started_at?: string + endedAt?: string + ended_at?: string + from_phone_number?: string + to_phone_number?: string + endedReason?: string + recording_s3_key?: string + recording_url?: string + duration_seconds?: number + agent_name?: string + _agent_ref?: string | number + direction?: string + messages?: Array<{ role: string; content: string; start_time?: number; end_time?: number }> + live_transcript?: Array<{ role: string; content: string; timestamp?: string; start_time?: number }> + metadata?: Record + call_short_id?: string +} + +export interface ObservabilityCall { + id: string + call_short_id: string + status?: string | null + call_event?: string | null + is_live?: boolean + direction?: string | null + source?: string | null + provider_platform?: string | null + provider_call_id?: string | null + agent_id?: string | null + agent?: ObservabilityCallAgent | null + created_at?: string | null + updated_at?: string | null + call_data?: ObservabilityCallData | null + live_transcript?: Array<{ role: string; content: string; timestamp?: string }> +} diff --git a/tests/test_api/test_call_import_audit.py b/tests/test_api/test_call_import_audit.py new file mode 100644 index 00000000..60592ff0 --- /dev/null +++ b/tests/test_api/test_call_import_audit.py @@ -0,0 +1,78 @@ +"""Audit fields (created_by / last_updated_by email) on call imports.""" + +from uuid import uuid4 + +from app.models.database import CallImport, Workspace +from app.models.enums import CallImportStatus + + +def _ensure_default_workspace(db_session, org_id): + ws = ( + db_session.query(Workspace) + .filter(Workspace.organization_id == org_id, Workspace.is_default.is_(True)) + .first() + ) + if ws is None: + ws = Workspace( + organization_id=org_id, name="Default", slug="default", is_default=True + ) + db_session.add(ws) + db_session.commit() + return ws + + +def test_update_call_import_metadata_stamps_actor_emails( + authenticated_client, db_session, org_id, seed_org +): + workspace = _ensure_default_workspace(db_session, org_id) + call_import = CallImport( + id=uuid4(), + organization_id=org_id, + workspace_id=workspace.id, + provider="exotel", + original_filename="batch.csv", + total_rows=0, + completed_rows=0, + failed_rows=0, + status=CallImportStatus.COMPLETED, + dataset="before", + ) + db_session.add(call_import) + db_session.commit() + + response = authenticated_client.patch( + f"/api/v1/call-imports/{call_import.id}", + json={"dataset": "after"}, + ) + assert response.status_code == 200, response.text + body = response.json() + assert body["dataset"] == "after" + assert body["created_by_email"] is None + assert body["last_updated_by_email"] == "owner@example.com" + + +def test_list_call_imports_includes_actor_emails( + authenticated_client, db_session, org_id, seed_org +): + workspace = _ensure_default_workspace(db_session, org_id) + call_import = CallImport( + id=uuid4(), + organization_id=org_id, + workspace_id=workspace.id, + provider=None, + original_filename="listed.csv", + total_rows=0, + completed_rows=0, + failed_rows=0, + status=CallImportStatus.UPLOADED, + ) + db_session.add(call_import) + db_session.commit() + + listing = authenticated_client.get("/api/v1/call-imports") + assert listing.status_code == 200, listing.text + items = listing.json()["items"] + match = [item for item in items if item["id"] == str(call_import.id)] + assert len(match) == 1 + assert match[0]["created_by_email"] is None + assert match[0]["last_updated_by_email"] is None diff --git a/tests/test_api/test_call_import_evaluations.py b/tests/test_api/test_call_import_evaluations.py index faf3ff9d..43d9230b 100644 --- a/tests/test_api/test_call_import_evaluations.py +++ b/tests/test_api/test_call_import_evaluations.py @@ -1152,3 +1152,43 @@ def test_evaluation_retry_can_override_telephony_credentials( db_session.refresh(call_import) assert call_import.telephony_integration_id == right_integration.id assert call_import.provider == "exotel" + + +def test_create_evaluation_sets_actor_emails( + authenticated_client, db_session, org_id, seed_org +): + metric = _make_metric(db_session, org_id) + call_import, _rows = _make_call_import(db_session, org_id, rows=2) + + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations", + json=_eval_body([metric.id]), + ) + assert response.status_code == 202, response.text + body = response.json() + assert body["created_by_email"] == "owner@example.com" + assert body["last_updated_by_email"] == "owner@example.com" + + +def test_update_evaluation_name_stamps_last_updated_by_email( + authenticated_client, db_session, org_id, seed_org +): + metric = _make_metric(db_session, org_id) + call_import, _rows = _make_call_import(db_session, org_id, rows=1) + + created = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations", + json=_eval_body([metric.id]), + ) + assert created.status_code == 202, created.text + eval_id = created.json()["id"] + + patched = authenticated_client.patch( + f"/api/v1/call-imports/{call_import.id}/evaluations/{eval_id}", + json={"name": "Renamed run"}, + ) + assert patched.status_code == 200, patched.text + body = patched.json() + assert body["name"] == "Renamed run" + assert body["created_by_email"] == "owner@example.com" + assert body["last_updated_by_email"] == "owner@example.com" From 3115b13f8653173ae810970f52410efcfe583ac3 Mon Sep 17 00:00:00 2001 From: M Sami Date: Fri, 7 Aug 2026 19:46:46 +0530 Subject: [PATCH 03/32] feat: implement actor stamping for call import evaluations and user insights --- app/api/v1/routes/call_import_evaluations.py | 16 +++++++ ...st_call_import_evaluation_user_insights.py | 26 +++++++++++ ...st_call_import_evaluations_mapped_async.py | 45 +++++++++++++++++++ .../test_call_import_metric_clusters_rows.py | 40 +++++++++++++++++ 4 files changed, 127 insertions(+) diff --git a/app/api/v1/routes/call_import_evaluations.py b/app/api/v1/routes/call_import_evaluations.py index 95222d6d..12069420 100644 --- a/app/api/v1/routes/call_import_evaluations.py +++ b/app/api/v1/routes/call_import_evaluations.py @@ -35,6 +35,7 @@ from app.services.call_imports.audit import ( actor_emails_for_evaluation, emails_for_user_ids, + stamp_call_import_actor, stamp_evaluation_actor, user_ids_from_evaluations, ) @@ -1032,6 +1033,7 @@ async def create_call_import_evaluation( call_import.failed_rows = 0 call_import.error_message = None call_import.status = CallImportStatus.PROCESSING + stamp_call_import_actor(call_import, principal) db.commit() db.refresh(call_import) starting_from_mapped = True @@ -5139,6 +5141,9 @@ async def generate_call_import_evaluation_insights( summary = EvaluationTldrSummary.model_validate(task_result) db.refresh(evaluation) + stamp_evaluation_actor(evaluation, principal) + db.commit() + db.refresh(evaluation) from app.services.ai.llm_resolver import get_llm_provider_and_model @@ -5153,6 +5158,7 @@ async def generate_call_import_evaluation_insights( force=body.regenerate, max_llm_calls=body.max_llm_calls, db=db, + principal=principal, ) return summary @@ -5210,6 +5216,7 @@ def _enqueue_user_insights_job( force: bool = False, max_llm_calls: Optional[int] = None, db: Optional[Session] = None, + principal: Optional[Principal] = None, ) -> None: """Enqueue background user-insights generation unless already running.""" current = _user_insights_payload(evaluation) @@ -5241,6 +5248,8 @@ def _enqueue_user_insights_job( "error_message": None, } if db is not None: + if principal is not None: + stamp_evaluation_actor(evaluation, principal) flag_modified(evaluation, "user_insights") db.commit() @@ -5348,6 +5357,7 @@ async def generate_call_import_evaluation_user_insights( force=body.force or body.regenerate, max_llm_calls=body.max_llm_calls, db=db, + principal=principal, ) db.refresh(evaluation) @@ -5590,6 +5600,7 @@ def _enqueue_metric_clusters_job( selected_evaluation_row_ids: Optional[List[str]] = None, failure_policies: Optional[Dict[str, MetricFailurePolicy]] = None, db: Optional[Session] = None, + principal: Optional[Principal] = None, ) -> None: current = _metric_clusters_payload(evaluation) if current is not None and current.status == "running" and not force: @@ -5668,6 +5679,8 @@ def _enqueue_metric_clusters_job( **policy_blob, } if db is not None: + if principal is not None: + stamp_evaluation_actor(evaluation, principal) flag_modified(evaluation, "metric_clusters") db.commit() @@ -5689,6 +5702,8 @@ def _enqueue_metric_clusters_job( if db is not None and isinstance(evaluation.metric_clusters, dict): evaluation.metric_clusters["celery_task_id"] = async_result.id flag_modified(evaluation, "metric_clusters") + if principal is not None: + stamp_evaluation_actor(evaluation, principal) db.commit() @@ -6092,6 +6107,7 @@ async def generate_call_import_evaluation_metric_clusters( selected_evaluation_row_ids=selected_row_ids, failure_policies=merged_policies, db=db, + principal=principal, ) db.refresh(evaluation) diff --git a/tests/test_api/test_call_import_evaluation_user_insights.py b/tests/test_api/test_call_import_evaluation_user_insights.py index 1fd76744..4d9a1336 100644 --- a/tests/test_api/test_call_import_evaluation_user_insights.py +++ b/tests/test_api/test_call_import_evaluation_user_insights.py @@ -191,6 +191,32 @@ def test_post_user_insights_enqueues_task( assert response.json()["max_llm_calls"] == 100 +def test_post_user_insights_stamps_last_updated_by_email( + authenticated_client, + db_session, + org_id, + seed_org, + make_ai_provider, + stub_user_insights_worker, +): + make_ai_provider(provider="openai", is_active=True) + call_import, evaluation = _seed_evaluation(db_session, org_id) + evaluation.last_updated_by_user_id = None + db_session.commit() + + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations/{evaluation.id}/user-insights", + json={"regenerate": True, "force": True}, + ) + assert response.status_code == 200, response.text + + detail = authenticated_client.get( + f"/api/v1/call-imports/{call_import.id}/evaluations/{evaluation.id}" + ) + assert detail.status_code == 200, detail.text + assert detail.json()["last_updated_by_email"] == "owner@example.com" + + def test_post_user_insights_requires_completed_rows( authenticated_client, db_session, org_id, seed_org ): diff --git a/tests/test_api/test_call_import_evaluations_mapped_async.py b/tests/test_api/test_call_import_evaluations_mapped_async.py index ac3035f9..f072dd68 100644 --- a/tests/test_api/test_call_import_evaluations_mapped_async.py +++ b/tests/test_api/test_call_import_evaluations_mapped_async.py @@ -117,3 +117,48 @@ def test_create_evaluation_from_mapped_enqueues_async_materialization( .first() ) assert refreshed_import.status == CallImportStatus.PROCESSING + + +def test_create_evaluation_from_mapped_stamps_parent_last_updated_by( + authenticated_client, + db_session, + org_id, + seed_org, + monkeypatch, +): + from tests.test_api.test_call_import_evaluations import ( + _eval_body, + _make_metric, + ) + + monkeypatch.setattr( + "app.api.v1.routes.call_imports._ensure_blob_storage_enabled", + lambda: None, + ) + + metric = _make_metric(db_session, org_id) + workspace = metric.workspace_id + call_import = _make_mapped_call_import(db_session, org_id, workspace) + call_import.last_updated_by_user_id = None + db_session.commit() + + delay_mock = MagicMock(return_value=MagicMock(id="async-task")) + fake_bulk_ops = types.ModuleType("app.workers.tasks.call_import_bulk_ops") + fake_bulk_ops.materialize_mapped_call_import_evaluation_task = MagicMock( + delay=delay_mock, + ) + monkeypatch.setitem( + sys.modules, + "app.workers.tasks.call_import_bulk_ops", + fake_bulk_ops, + ) + + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations", + json=_eval_body([metric.id]), + ) + assert response.status_code == 202, response.text + + detail = authenticated_client.get(f"/api/v1/call-imports/{call_import.id}") + assert detail.status_code == 200, detail.text + assert detail.json()["last_updated_by_email"] == "owner@example.com" diff --git a/tests/test_api/test_call_import_metric_clusters_rows.py b/tests/test_api/test_call_import_metric_clusters_rows.py index a1d1bd31..6595b5ab 100644 --- a/tests/test_api/test_call_import_metric_clusters_rows.py +++ b/tests/test_api/test_call_import_metric_clusters_rows.py @@ -147,6 +147,46 @@ def fake_apply_async(*, kwargs=None, **_kw): assert len(captured["evaluation_row_ids"]) == 2 +def test_generate_metric_clusters_stamps_last_updated_by_email( + authenticated_client, + db_session, + org_id, + seed_org, + make_ai_provider, + monkeypatch, +): + make_ai_provider(provider="openai", is_active=True) + call_import, evaluation, _ = _seed_eval_with_rows( + db_session, + org_id, + rows=[ + {"conversation_id": "c0", "status": "completed", "score_value": 0.2}, + ], + ) + evaluation.last_updated_by_user_id = None + db_session.commit() + + def fake_apply_async(*, kwargs=None, **_kw): + return types.SimpleNamespace(id="cluster-task-1") + + monkeypatch.setattr( + "app.workers.tasks.generate_evaluation_metric_clusters.generate_evaluation_metric_clusters_task.apply_async", + fake_apply_async, + ) + + response = authenticated_client.post( + f"/api/v1/call-imports/{call_import.id}/evaluations/{evaluation.id}/metric-clusters", + json={"row_limit": 1}, + ) + assert response.status_code == 200, response.text + + detail = authenticated_client.get( + f"/api/v1/call-imports/{call_import.id}/evaluations/{evaluation.id}" + ) + assert detail.status_code == 200, detail.text + assert detail.json()["last_updated_by_email"] == "owner@example.com" + + def test_cancel_preserves_selected_row_ids_in_state( authenticated_client, db_session, org_id, seed_org, make_ai_provider, monkeypatch ): From 681fe3719682e6169deddefe17fc26f16ed4c722 Mon Sep 17 00:00:00 2001 From: M Sami Date: Fri, 7 Aug 2026 20:09:48 +0530 Subject: [PATCH 04/32] feat: enhance user insights with improved call import evaluations --- .gitignore | 1 - app/config.py | 31 ++++++++-- config.docker.yml | 137 --------------------------------------------- docker-compose.yml | 8 +++ env.example | 4 ++ 5 files changed, 39 insertions(+), 142 deletions(-) delete mode 100644 config.docker.yml diff --git a/.gitignore b/.gitignore index 3fb88623..106532ab 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,6 @@ env/ ENV/ .venv config.yml -config.docker.yml # uv .python-version uv.lock diff --git a/app/config.py b/app/config.py index 94b51f18..0eec6b2a 100644 --- a/app/config.py +++ b/app/config.py @@ -1,6 +1,8 @@ """Configuration management using Pydantic settings.""" import json +import os +import re import yaml from pathlib import Path from typing import Annotated, Any, Dict, List, Optional, Union @@ -412,6 +414,19 @@ def apply_service_mode(mode: str) -> None: settings.SERVICE_MODE = normalized +_ENV_REF_PATTERN = re.compile(r"^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$") + + +def _expand_env_ref(value: Any) -> Any: + """Replace ``${VAR}`` with ``os.environ[VAR]`` when loading YAML secrets.""" + if not isinstance(value, str): + return value + match = _ENV_REF_PATTERN.match(value.strip()) + if not match: + return value + return os.environ.get(match.group(1), "") + + def load_config_from_file(config_path: str) -> None: """Load configuration from a YAML file and update global settings.""" import yaml @@ -577,9 +592,13 @@ def load_config_from_file(config_path: str) -> None: if "region" in s3_config: settings.S3_REGION = s3_config["region"] if "access_key_id" in s3_config: - settings.S3_ACCESS_KEY_ID = s3_config["access_key_id"] + resolved = _expand_env_ref(s3_config["access_key_id"]) + if resolved: + settings.S3_ACCESS_KEY_ID = resolved if "secret_access_key" in s3_config: - settings.S3_SECRET_ACCESS_KEY = s3_config["secret_access_key"] + resolved = _expand_env_ref(s3_config["secret_access_key"]) + if resolved: + settings.S3_SECRET_ACCESS_KEY = resolved if "endpoint_url" in s3_config: settings.S3_ENDPOINT_URL = s3_config["endpoint_url"] if "prefix" in s3_config: @@ -730,9 +749,13 @@ def load_config_from_file(config_path: str) -> None: if "region" in loki_s3: settings.LOKI_S3_REGION = loki_s3["region"] if "access_key_id" in loki_s3: - settings.LOKI_S3_ACCESS_KEY_ID = loki_s3["access_key_id"] + resolved = _expand_env_ref(loki_s3["access_key_id"]) + if resolved: + settings.LOKI_S3_ACCESS_KEY_ID = resolved if "secret_access_key" in loki_s3: - settings.LOKI_S3_SECRET_ACCESS_KEY = loki_s3["secret_access_key"] + resolved = _expand_env_ref(loki_s3["secret_access_key"]) + if resolved: + settings.LOKI_S3_SECRET_ACCESS_KEY = resolved if "prefix" in loki_s3: settings.LOKI_S3_PREFIX = loki_s3["prefix"] if "plivo" in config_data: diff --git a/config.docker.yml b/config.docker.yml deleted file mode 100644 index 6b84d505..00000000 --- a/config.docker.yml +++ /dev/null @@ -1,137 +0,0 @@ -# EfficientAI Docker Configuration File -# This config uses Docker service names for networking - -# Application Settings -app: - name: "EfficientAI Voice AI Evaluation Platform" - version: "0.1.0" - debug: true - secret_key: "your-secret-key-here-change-in-production" - -# Server Settings -server: - host: "0.0.0.0" - port: 8000 - -# Operational endpoints (/metrics). /health is always open for load balancers. -# Add VPC CIDRs here if Prometheus scrapes /metrics from inside the VPC. -operational: - public: false - trusted_ips: - - "10.0.0.0/8" - -# Database Configuration (Docker service name) -database: - url: "postgresql://efficientai:password@db:5432/efficientai" - -# Redis Configuration (Docker service name) -redis: - url: "redis://redis:6379/0" - -# Celery Configuration (Docker service names) -celery: - broker_url: "redis://redis:6379/0" - result_backend: "redis://redis:6379/0" - -# File Storage -storage: - upload_dir: "/app/uploads" - max_file_size_mb: 500 - allowed_audio_formats: - - "wav" - - "mp3" - - "flac" - - "m4a" - -# S3 Configuration (for data sources integration) -s3: - enabled: true - bucket_name: "voiceai-evals-test" - region: "us-east-1" - access_key_id: "${S3_ACCESS_KEY_ID}" - secret_access_key: "${S3_SECRET_ACCESS_KEY}" - endpoint_url: null - prefix: "audio/" - -# Speaker Diarization (pyannote.audio) -# Requires accepting model terms at https://huggingface.co/pyannote/speaker-diarization-3.1 -diarization: - huggingface_token: # Set your HuggingFace token here (e.g., "hf_xxxxx") - num_speakers: 2 # Force exact speaker count (2 = agent + customer). Set to null for auto-detect. - -# CORS Settings -cors: - origins: - - "http://localhost:3000" - - "http://localhost:8000" - -# API Settings -api: - prefix: "/api/v1" - key_header: "X-API-Key" - rate_limit_per_minute: 60 - -# Observability (Loki log aggregation) -# storage: "filesystem" (default, local Docker volume) or "s3" (durable, for production) -# multi_tenant: false (default, single tenant) or true (per-org log isolation) -observability: - enabled: false - loki: - enabled: false - url: "http://loki:3100" - storage: filesystem - multi_tenant: false - platform_tenant: "platform" - # s3: - # bucket_name: "my-logs-bucket" - # region: "us-east-1" - # access_key_id: "" - # secret_access_key: "" - # prefix: "logs/" - -auth: - providers: - - api_key - - local_password - # - external_oidc # requires EFFICIENTAI_LICENSE feature: oidc_sso - - local_password: - # Lifetime of the Bearer tokens minted at POST /auth/login (in minutes). - # Keep this short; clients re-authenticate silently. - token_ttl_minutes: 720 # 12 hours - # Turn this off in Cloud SaaS to block self-serve signup. - allow_signup: true - - # External OIDC (enterprise): bring your own IdP. Works with any - # OIDC-compliant provider. See README.md > Authentication & Deployment - # Recipes for copy-paste recipes for Okta, Azure AD, Google Workspace - # and AWS Cognito. - # - # oidc: - # issuer: "https://example.okta.com" # REQUIRED - # audience: "efficientai" # expected `aud` claim - # client_id: "0oa..." # SPA client id - # jwks_uri: "https://example.okta.com/oauth2/v1/keys" # optional, derived from issuer - # default_org_name: "Example Inc" # used when no org claim is present - # org_claim_path: ["https://efficientai.com/org"] # dotted path into the JWT to find the org name - -# Judge Alignment (AlignEval-style hybrid integration). -# Per-org thresholds and judge model selection are UI-driven; only the -# operator kill switch + CSV upload limit live here. -judge_alignment: - enabled: true - csv_max_rows: 5000 - -# LLM gateway (optional). Per-org overrides in Integrations UI. -# llm_gateway: -# enabled: false -# type: bifrost # bifrost | litellm_proxy -# base_url: "http://localhost:8080/litellm" -# virtual_key: null -# master_key: null -# passthrough_provider_keys: true - -# Enterprise License (JWT signed with RS256). Unlocks gated features like -# oidc_sso, mfa_enforce, audit_export, voice_playground, gepa_optimization. -# license: -# key: "eyJhbGciOi..." diff --git a/docker-compose.yml b/docker-compose.yml index 066b7832..552884f3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,6 +47,8 @@ services: context: . dockerfile: docker/Dockerfile.api container_name: efficientai_api + env_file: + - .env environment: DATABASE_URL: postgresql://${POSTGRES_USER:-efficientai}:${POSTGRES_PASSWORD:-password}@db:5432/${POSTGRES_DB:-efficientai} REDIS_URL: redis://redis:6379/0 @@ -95,6 +97,8 @@ services: args: INSTALL_EXTRAS: "qualitative-voice" container_name: efficientai_media + env_file: + - .env environment: DATABASE_URL: postgresql://${POSTGRES_USER:-efficientai}:${POSTGRES_PASSWORD:-password}@db:5432/${POSTGRES_DB:-efficientai} REDIS_URL: redis://redis:6379/0 @@ -126,6 +130,8 @@ services: args: INSTALL_EXTRAS: "qualitative-voice" container_name: efficientai_worker + env_file: + - .env environment: # Worker uses Docker network, so it reaches DB/Redis via service names DATABASE_URL: postgresql://${POSTGRES_USER:-efficientai}:${POSTGRES_PASSWORD:-password}@db:5432/${POSTGRES_DB:-efficientai} @@ -162,6 +168,8 @@ services: args: INSTALL_EXTRAS: "" container_name: efficientai_worker_imports + env_file: + - .env environment: DATABASE_URL: postgresql://${POSTGRES_USER:-efficientai}:${POSTGRES_PASSWORD:-password}@db:5432/${POSTGRES_DB:-efficientai} REDIS_URL: redis://redis:6379/0 diff --git a/env.example b/env.example index c434c850..0de4f151 100644 --- a/env.example +++ b/env.example @@ -24,6 +24,10 @@ UPLOAD_DIR=/app/uploads MAX_FILE_SIZE_MB=500 ALLOWED_AUDIO_FORMATS=wav,mp3,flac,m4a +# S3 (optional — used when config.docker.yml references ${S3_ACCESS_KEY_ID}) +# S3_ACCESS_KEY_ID= +# S3_SECRET_ACCESS_KEY= + # Celery Configuration CELERY_BROKER_URL=redis://redis:6379/0 CELERY_RESULT_BACKEND=redis://redis:6379/0 From 8786a43520d9968fb7109fc45ab3bb8c03e35660 Mon Sep 17 00:00:00 2001 From: M Sami Date: Fri, 7 Aug 2026 20:17:54 +0530 Subject: [PATCH 05/32] chore: add config.docker.yml to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 106532ab..3fb88623 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ env/ ENV/ .venv config.yml +config.docker.yml # uv .python-version uv.lock From 55b3bfd05dcd23be019233ba48b950d6167b7775 Mon Sep 17 00:00:00 2001 From: M Sami Date: Fri, 7 Aug 2026 22:52:50 +0530 Subject: [PATCH 06/32] feat: Cache evaluation PDF reports by content fingerprint and reuse S3 artifacts on preview --- app/api/v1/routes/call_import_evaluations.py | 338 +++++++++++++++++- .../060_call_import_evaluation_pdf_reports.py | 76 ++++ ...call_import_eval_pdf_report_cache_index.py | 137 +++++++ app/models/database.py | 44 +++ .../call_import_pdf_report_storage.py | 207 +++++++++++ app/services/storage/azure_blob_service.py | 27 +- app/services/storage/blob_storage_service.py | 14 +- app/services/storage/gcs_service.py | 24 +- app/services/storage/s3_service.py | 13 +- env.example | 2 + frontend/src/lib/api.ts | 32 +- .../CallImportEvaluationDetail.tsx | 229 ++++++++++-- frontend/src/types/api.ts | 29 ++ .../test_call_import_evaluation_pdf_report.py | 131 +++++++ 14 files changed, 1233 insertions(+), 70 deletions(-) create mode 100644 app/migrations/060_call_import_evaluation_pdf_reports.py create mode 100644 app/migrations/061_call_import_eval_pdf_report_cache_index.py create mode 100644 app/services/reporting/call_import_pdf_report_storage.py diff --git a/app/api/v1/routes/call_import_evaluations.py b/app/api/v1/routes/call_import_evaluations.py index 12069420..94c371d0 100644 --- a/app/api/v1/routes/call_import_evaluations.py +++ b/app/api/v1/routes/call_import_evaluations.py @@ -11,7 +11,7 @@ import re import statistics from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple -from uuid import UUID +from uuid import UUID, uuid4 from datetime import date, datetime, timedelta, timezone @@ -20,6 +20,7 @@ from loguru import logger from pydantic import BaseModel, Field, field_validator from sqlalchemy import desc, func, or_, text +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified @@ -45,6 +46,7 @@ CallImport, CallImportEvaluation, CallImportEvaluationReportSnapshot, + CallImportEvaluationPdfReport, CallImportEvaluationRow, CallImportRow, Metric, @@ -99,6 +101,15 @@ from app.services.reporting.call_import_evaluation_pdf_report import ( call_import_evaluation_pdf_report_service, ) +from app.services.reporting.call_import_pdf_report_storage import ( + build_pdf_report_s3_key, + compute_pdf_report_cache_fingerprint, + compute_pdf_report_config_fingerprint, + compute_pdf_report_content_fingerprint, + config_summary_from_report_config, + find_cached_pdf_report, + presigned_urls_for_pdf_report, +) from app.services.call_import_metric_clusters import ( METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, estimate_metric_clusters_llm_calls, @@ -159,6 +170,35 @@ def _clean_vendor_name(cls, value: str) -> str: return cleaned +class CallImportEvaluationPdfReportResponse(BaseModel): + id: str + filename: str + preview_url: Optional[str] = None + download_url: Optional[str] = None + created_at: datetime + created_by: Optional[str] = None + report_type: str + vendor_name: str + config_summary: Optional[str] = None + storage_available: bool = True + cache_hit: bool = False + + +class CallImportEvaluationPdfReportListItem(BaseModel): + id: str + filename: Optional[str] = None + vendor_name: str + report_type: str + created_by: Optional[str] = None + created_at: datetime + config_summary: Optional[str] = None + cache_fingerprint: Optional[str] = None + + +class CallImportEvaluationPdfReportListResponse(BaseModel): + items: List[CallImportEvaluationPdfReportListItem] + + class CallImportEvaluationBaselineCandidate(BaseModel): evaluation_id: str name: str @@ -2112,6 +2152,57 @@ def _report_filename_slug(value: str) -> str: return slug or "client" +def _pdf_report_actor(principal: Principal) -> tuple[Optional[str], Optional[UUID]]: + created_by = principal.email + if not created_by and principal.user_id: + created_by = str(principal.user_id) + return created_by, principal.user_id + + +def _pdf_report_response_from_row( + row: CallImportEvaluationPdfReport, + *, + cache_hit: bool = False, +) -> CallImportEvaluationPdfReportResponse: + filename = row.filename or "report.pdf" + preview_url, download_url = presigned_urls_for_pdf_report( + row.s3_key or "", + filename, + ) + return CallImportEvaluationPdfReportResponse( + id=str(row.id), + filename=filename, + preview_url=preview_url, + download_url=download_url, + created_at=row.created_at or datetime.now(timezone.utc), + created_by=row.created_by, + report_type=row.report_type, + vendor_name=row.vendor_name, + config_summary=config_summary_from_report_config( + row.report_config if isinstance(row.report_config, dict) else {} + ), + storage_available=bool(row.s3_key), + cache_hit=cache_hit, + ) + + +def _pdf_report_list_item_from_row( + row: CallImportEvaluationPdfReport, +) -> CallImportEvaluationPdfReportListItem: + return CallImportEvaluationPdfReportListItem( + id=str(row.id), + filename=row.filename, + vendor_name=row.vendor_name, + report_type=row.report_type, + created_by=row.created_by, + created_at=row.created_at or datetime.now(timezone.utc), + config_summary=config_summary_from_report_config( + row.report_config if isinstance(row.report_config, dict) else {} + ), + cache_fingerprint=row.cache_fingerprint, + ) + + def _report_branding_for_import_workspace( db: Session, organization_id: UUID, @@ -3405,8 +3496,9 @@ async def generate_call_import_evaluation_pdf_report( payload: CallImportEvaluationPdfReportRequest, api_key: str = Depends(get_api_key), organization_id: UUID = Depends(get_organization_id), + principal: Principal = Depends(get_principal), db: Session = Depends(get_db), -) -> StreamingResponse: +): del api_key call_import = _require_import(db, call_import_id, organization_id) @@ -3527,17 +3619,6 @@ async def generate_call_import_evaluation_pdf_report( cached_prompt_improvements, report_config, ) - narrative = _generate_report_narrative( - db, - organization_id, - metric_aggregates=metric_aggregates, - insight_aggregates=insight_aggregates if is_internal else [], - period_delta_by_metric=period_delta_by_metric, - evidence_samples=evidence_samples if is_internal else {}, - report_config=report_config, - ) - - generated_at = datetime.now(timezone.utc) branding_images, custom_heading = _report_branding_for_import_workspace( db, organization_id, @@ -3562,6 +3643,76 @@ async def generate_call_import_evaluation_pdf_report( pdf_aggregates, child_names_by_parent=pdf_child_map, ) + + from app.services.storage.s3_service import s3_service + + config_fingerprint = compute_pdf_report_config_fingerprint( + report_type=payload.report_type, + include_period_delta=bool(payload.include_period_delta), + include_weekly_delta=bool(payload.include_weekly_delta), + baseline_evaluation_id=payload.baseline_evaluation_id, + internal_brand_image_id=payload.internal_brand_image_id, + external_brand_image_id=payload.external_brand_image_id, + use_case=payload.use_case, + report_config=report_config, + report_heading=custom_heading, + vendor_name=payload.vendor_name, + platform_base_url=payload.platform_base_url, + period_label=period_label, + ) + content_fingerprint = compute_pdf_report_content_fingerprint( + evaluation_status=evaluation.status, + completed_rows=int(evaluation.completed_rows or 0), + total_rows=int(evaluation.total_rows or 0), + failed_rows=int(evaluation.failed_rows or 0), + metric_aggregates=metric_aggregates, + insight_aggregates=insight_aggregates, + period_delta_by_metric=period_delta_by_metric, + benchmark_context=benchmark_context, + metric_metadata=[ + { + "id": str(metric.id), + "name": metric.name, + "description": metric.description, + } + for metric in metrics + ], + failure_policies=failure_policies_for_pdf, + tldr_summary=cached_tldr_summary, + user_insights_for_pdf=generated_insights_for_pdf, + metric_clusters_for_pdf=metric_clusters_for_pdf, + prompt_improvements_for_pdf=prompt_improvements_for_pdf, + ) + cache_fingerprint = compute_pdf_report_cache_fingerprint( + config_fingerprint=config_fingerprint, + content_fingerprint=content_fingerprint, + ) + if s3_service.is_enabled(): + cached_pdf_report = find_cached_pdf_report( + db, + evaluation_id=evaluation.id, + organization_id=organization_id, + cache_fingerprint=cache_fingerprint, + ) + if cached_pdf_report is not None: + logger.info( + "Reusing stored PDF report {} for evaluation {} (cache fingerprint match)", + cached_pdf_report.id, + eval_id, + ) + return _pdf_report_response_from_row(cached_pdf_report, cache_hit=True) + + narrative = _generate_report_narrative( + db, + organization_id, + metric_aggregates=metric_aggregates, + insight_aggregates=insight_aggregates if is_internal else [], + period_delta_by_metric=period_delta_by_metric, + evidence_samples=evidence_samples if is_internal else {}, + report_config=report_config, + ) + + generated_at = datetime.now(timezone.utc) try: pdf_started = datetime.now(timezone.utc) pdf_bytes = await asyncio.to_thread( @@ -3635,17 +3786,168 @@ async def generate_call_import_evaluation_pdf_report( .count(), ) db.add(snapshot) - db.commit() + db.flush() filename = ( f"{_report_filename_slug(payload.vendor_name)}-" f"{payload.report_type}-quality-metric-audit-{eval_id}.pdf" ) - return StreamingResponse( - iter([pdf_bytes]), - media_type="application/pdf", - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + + if not s3_service.is_enabled(): + db.commit() + return StreamingResponse( + iter([pdf_bytes]), + media_type="application/pdf", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + report_id = uuid4() + s3_key = build_pdf_report_s3_key( + organization_id=organization_id, + call_import_id=call_import.id, + evaluation_id=evaluation.id, + report_id=report_id, ) + try: + s3_service.upload_file_by_key( + file_content=pdf_bytes, + key=s3_key, + content_type="application/pdf", + ) + except Exception as exc: # noqa: BLE001 + logger.exception( + "Failed to upload PDF report for evaluation {} to object storage", + eval_id, + ) + db.rollback() + raise HTTPException( + status_code=500, + detail=f"Failed to store PDF report: {exc}", + ) from exc + + created_by, created_by_user_id = _pdf_report_actor(principal) + pdf_report = CallImportEvaluationPdfReport( + id=report_id, + evaluation_id=evaluation.id, + call_import_id=call_import.id, + organization_id=organization_id, + workspace_id=call_import.workspace_id, + snapshot_id=snapshot.id, + vendor_name=payload.vendor_name, + report_type=payload.report_type, + filename=filename, + s3_key=s3_key, + report_config=report_config, + cache_fingerprint=cache_fingerprint, + created_by=created_by, + created_by_user_id=created_by_user_id, + ) + db.add(pdf_report) + try: + db.commit() + except IntegrityError: + db.rollback() + try: + s3_service.delete_file_by_key(s3_key) + except Exception: # noqa: BLE001 + logger.warning( + "Failed to delete orphan PDF after cache race for evaluation {}", + eval_id, + ) + raced_winner = find_cached_pdf_report( + db, + evaluation_id=evaluation.id, + organization_id=organization_id, + cache_fingerprint=cache_fingerprint, + ) + if raced_winner is not None: + logger.info( + "PDF report cache race resolved for evaluation {} (winner {})", + eval_id, + raced_winner.id, + ) + return _pdf_report_response_from_row(raced_winner, cache_hit=True) + raise HTTPException( + status_code=500, + detail="Failed to store PDF report due to a concurrent duplicate request.", + ) from None + db.refresh(pdf_report) + return _pdf_report_response_from_row(pdf_report) + + +@router.get( + "/{eval_id}/pdf-reports", + response_model=CallImportEvaluationPdfReportListResponse, + operation_id="listCallImportEvaluationPdfReports", +) +async def list_call_import_evaluation_pdf_reports( + call_import_id: UUID, + eval_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationPdfReportListResponse: + del api_key + _require_import(db, call_import_id, organization_id) + evaluation = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.id == eval_id, + CallImportEvaluation.call_import_id == call_import_id, + CallImportEvaluation.organization_id == organization_id, + ) + .first() + ) + if not evaluation: + raise HTTPException(status_code=404, detail="Call import evaluation not found") + + rows = ( + db.query(CallImportEvaluationPdfReport) + .filter( + CallImportEvaluationPdfReport.evaluation_id == eval_id, + CallImportEvaluationPdfReport.organization_id == organization_id, + ) + .order_by(desc(CallImportEvaluationPdfReport.created_at)) + .all() + ) + return CallImportEvaluationPdfReportListResponse( + items=[_pdf_report_list_item_from_row(row) for row in rows], + ) + + +@router.get( + "/{eval_id}/pdf-reports/{report_id}", + response_model=CallImportEvaluationPdfReportResponse, + operation_id="getCallImportEvaluationPdfReport", +) +async def get_call_import_evaluation_pdf_report( + call_import_id: UUID, + eval_id: UUID, + report_id: UUID, + api_key: str = Depends(get_api_key), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +) -> CallImportEvaluationPdfReportResponse: + del api_key + _require_import(db, call_import_id, organization_id) + row = ( + db.query(CallImportEvaluationPdfReport) + .filter( + CallImportEvaluationPdfReport.id == report_id, + CallImportEvaluationPdfReport.evaluation_id == eval_id, + CallImportEvaluationPdfReport.call_import_id == call_import_id, + CallImportEvaluationPdfReport.organization_id == organization_id, + ) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="PDF report not found") + if not row.s3_key: + raise HTTPException( + status_code=404, + detail="PDF report file is not available in object storage", + ) + return _pdf_report_response_from_row(row) @router.patch( diff --git a/app/migrations/060_call_import_evaluation_pdf_reports.py b/app/migrations/060_call_import_evaluation_pdf_reports.py new file mode 100644 index 00000000..8f2c6012 --- /dev/null +++ b/app/migrations/060_call_import_evaluation_pdf_reports.py @@ -0,0 +1,76 @@ +"""Migration: stored PDF reports for call import evaluations.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add call_import_evaluation_pdf_reports for S3-stored evaluation PDFs" + + +def _table_exists(db: Session, table_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.tables + WHERE table_name = :table_name + """ + ), + {"table_name": table_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if _table_exists(db, "call_import_evaluation_pdf_reports"): + print("call_import_evaluation_pdf_reports already exists, skipping") + db.commit() + return + + db.execute( + text( + """ + CREATE TABLE call_import_evaluation_pdf_reports ( + id UUID PRIMARY KEY, + evaluation_id UUID NOT NULL REFERENCES call_import_evaluations(id) ON DELETE CASCADE, + call_import_id UUID NOT NULL REFERENCES call_imports(id) ON DELETE CASCADE, + organization_id UUID NOT NULL REFERENCES organizations(id), + workspace_id UUID NOT NULL REFERENCES workspaces(id), + snapshot_id UUID REFERENCES call_import_evaluation_report_snapshots(id) ON DELETE SET NULL, + vendor_name VARCHAR(120) NOT NULL, + report_type VARCHAR(20) NOT NULL DEFAULT 'external', + filename VARCHAR(255), + s3_key VARCHAR(512), + report_config JSONB NOT NULL DEFAULT '{}'::jsonb, + cache_fingerprint VARCHAR(64), + created_by TEXT, + created_by_user_id UUID, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_call_import_eval_pdf_reports_eval + ON call_import_evaluation_pdf_reports (evaluation_id, created_at DESC) + """ + ) + ) + db.execute( + text( + """ + CREATE UNIQUE INDEX IF NOT EXISTS + uq_call_import_eval_pdf_reports_eval_cache_fp + ON call_import_evaluation_pdf_reports (evaluation_id, cache_fingerprint) + WHERE cache_fingerprint IS NOT NULL + """ + ) + ) + print("Created call_import_evaluation_pdf_reports") + db.commit() + + +def downgrade(db: Session): + db.execute(text("DROP TABLE IF EXISTS call_import_evaluation_pdf_reports")) + db.commit() diff --git a/app/migrations/061_call_import_eval_pdf_report_cache_index.py b/app/migrations/061_call_import_eval_pdf_report_cache_index.py new file mode 100644 index 00000000..baa579b0 --- /dev/null +++ b/app/migrations/061_call_import_eval_pdf_report_cache_index.py @@ -0,0 +1,137 @@ +"""Migration: PDF report cache fingerprint column rename and lookup index.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Rename config_fingerprint to cache_fingerprint and add unique " + "(evaluation_id, cache_fingerprint) for scale-safe cache lookups" +) + + +def _table_exists(db: Session, table_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.tables + WHERE table_name = :table_name + """ + ), + {"table_name": table_name}, + ).first() + return row is not None + + +def _column_exists(db: Session, column_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'call_import_evaluation_pdf_reports' + AND column_name = :column_name + """ + ), + {"column_name": column_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _table_exists(db, "call_import_evaluation_pdf_reports"): + print("call_import_evaluation_pdf_reports missing, skipping 061") + db.commit() + return + + if _column_exists(db, "config_fingerprint") and not _column_exists( + db, "cache_fingerprint" + ): + db.execute( + text( + """ + ALTER TABLE call_import_evaluation_pdf_reports + RENAME COLUMN config_fingerprint TO cache_fingerprint + """ + ) + ) + print("Renamed config_fingerprint -> cache_fingerprint") + + if not _column_exists(db, "cache_fingerprint"): + print("cache_fingerprint column missing, skipping index work") + db.commit() + return + + db.execute( + text( + """ + DELETE FROM call_import_evaluation_pdf_reports stale + USING call_import_evaluation_pdf_reports keep + WHERE stale.evaluation_id = keep.evaluation_id + AND stale.cache_fingerprint = keep.cache_fingerprint + AND stale.cache_fingerprint IS NOT NULL + AND stale.id <> keep.id + AND ( + stale.created_at < keep.created_at + OR ( + stale.created_at = keep.created_at + AND stale.id::text < keep.id::text + ) + ) + """ + ) + ) + + db.execute( + text( + """ + DROP INDEX IF EXISTS ix_call_import_eval_pdf_reports_fingerprint + """ + ) + ) + db.execute( + text( + """ + CREATE UNIQUE INDEX IF NOT EXISTS + uq_call_import_eval_pdf_reports_eval_cache_fp + ON call_import_evaluation_pdf_reports (evaluation_id, cache_fingerprint) + WHERE cache_fingerprint IS NOT NULL + """ + ) + ) + print("Ensured unique (evaluation_id, cache_fingerprint) index") + db.commit() + + +def downgrade(db: Session): + if not _table_exists(db, "call_import_evaluation_pdf_reports"): + db.commit() + return + + db.execute( + text( + """ + DROP INDEX IF EXISTS uq_call_import_eval_pdf_reports_eval_cache_fp + """ + ) + ) + if _column_exists(db, "cache_fingerprint") and not _column_exists( + db, "config_fingerprint" + ): + db.execute( + text( + """ + ALTER TABLE call_import_evaluation_pdf_reports + RENAME COLUMN cache_fingerprint TO config_fingerprint + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_call_import_eval_pdf_reports_fingerprint + ON call_import_evaluation_pdf_reports (config_fingerprint) + """ + ) + ) + db.commit() diff --git a/app/models/database.py b/app/models/database.py index 3ce548de..68bcfebc 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -2552,6 +2552,50 @@ class CallImportEvaluationReportSnapshot(Base): ) +class CallImportEvaluationPdfReport(Base): + """Stored PDF artifact for a call import evaluation report generation.""" + + __tablename__ = "call_import_evaluation_pdf_reports" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + snapshot_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluation_report_snapshots.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + vendor_name = Column(String(120), nullable=False) + report_type = Column(String(20), nullable=False, default="external") + filename = Column(String(255), nullable=True) + s3_key = Column(String(512), nullable=True) + report_config = Column(JSON, nullable=False, default=dict, server_default="{}") + cache_fingerprint = Column(String(64), nullable=True) + created_by = Column(String, nullable=True) + created_by_user_id = Column(UUID(as_uuid=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + # --------------------------------------------------------------------------- # Judge Alignment (AlignEval-style hybrid integration) # diff --git a/app/services/reporting/call_import_pdf_report_storage.py b/app/services/reporting/call_import_pdf_report_storage.py new file mode 100644 index 00000000..17992d05 --- /dev/null +++ b/app/services/reporting/call_import_pdf_report_storage.py @@ -0,0 +1,207 @@ +"""Helpers for call import evaluation PDF report storage and config fingerprinting.""" + +from __future__ import annotations + +import hashlib +import json +from typing import TYPE_CHECKING, Any +from uuid import UUID + +from app.services.storage.s3_service import s3_service + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + + from app.models.database import CallImportEvaluationPdfReport + + +def build_pdf_report_s3_key( + *, + organization_id: UUID, + call_import_id: UUID, + evaluation_id: UUID, + report_id: UUID, +) -> str: + prefix = s3_service.prefix or "" + return ( + f"{prefix}organizations/{organization_id}/call_imports/{call_import_id}/" + f"evaluations/{evaluation_id}/reports/{report_id}.pdf" + ) + + +def _canonicalize_report_config(value: Any) -> Any: + if isinstance(value, dict): + return {str(k): _canonicalize_report_config(v) for k, v in sorted(value.items())} + if isinstance(value, list): + normalized = [_canonicalize_report_config(item) for item in value] + try: + return sorted( + normalized, + key=lambda item: json.dumps(item, sort_keys=True, default=str), + ) + except TypeError: + return normalized + return value + + +def _fingerprint_digest(payload: dict[str, Any]) -> str: + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _json_fingerprint_value(value: Any) -> Any: + if value is None: + return None + if hasattr(value, "model_dump"): + return value.model_dump(mode="json") + if isinstance(value, dict): + return {str(k): _json_fingerprint_value(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_json_fingerprint_value(item) for item in value] + return value + + +def compute_pdf_report_config_fingerprint( + *, + report_type: str, + include_period_delta: bool, + include_weekly_delta: bool, + baseline_evaluation_id: str | None, + internal_brand_image_id: str | None, + external_brand_image_id: str | None, + use_case: str | None, + report_config: dict[str, Any], + report_heading: str | None, + vendor_name: str | None = None, + platform_base_url: str | None = None, + period_label: str | None = None, +) -> str: + payload = { + "report_type": report_type, + "include_period_delta": include_period_delta, + "include_weekly_delta": include_weekly_delta, + "baseline_evaluation_id": baseline_evaluation_id, + "internal_brand_image_id": internal_brand_image_id, + "external_brand_image_id": external_brand_image_id, + "use_case": use_case, + "report_config": _canonicalize_report_config(report_config or {}), + "report_heading": (report_heading or "").strip(), + "vendor_name": (vendor_name or "").strip(), + "platform_base_url": (platform_base_url or "").strip(), + "period_label": (period_label or "").strip(), + } + return _fingerprint_digest(payload) + + +def compute_pdf_report_content_fingerprint( + *, + evaluation_status: str, + completed_rows: int, + total_rows: int, + failed_rows: int, + metric_aggregates: list[dict[str, Any]], + insight_aggregates: list[dict[str, Any]], + period_delta_by_metric: dict[str, Any], + benchmark_context: Any, + metric_metadata: list[dict[str, Any]], + failure_policies: dict[str, Any], + tldr_summary: Any = None, + user_insights_for_pdf: Any = None, + metric_clusters_for_pdf: Any = None, + prompt_improvements_for_pdf: Any = None, +) -> str: + payload = { + "evaluation_status": evaluation_status, + "completed_rows": completed_rows, + "total_rows": total_rows, + "failed_rows": failed_rows, + "metric_aggregates": _canonicalize_report_config(metric_aggregates or []), + "insight_aggregates": _canonicalize_report_config(insight_aggregates or []), + "period_delta_by_metric": _canonicalize_report_config(period_delta_by_metric or {}), + "benchmark_context": _json_fingerprint_value(benchmark_context), + "metric_metadata": _canonicalize_report_config(metric_metadata or []), + "failure_policies": _json_fingerprint_value(failure_policies or {}), + "tldr_summary": _json_fingerprint_value(tldr_summary), + "user_insights_for_pdf": _json_fingerprint_value(user_insights_for_pdf), + "metric_clusters_for_pdf": _json_fingerprint_value(metric_clusters_for_pdf), + "prompt_improvements_for_pdf": _json_fingerprint_value(prompt_improvements_for_pdf), + } + return _fingerprint_digest(payload) + + +def compute_pdf_report_cache_fingerprint( + *, + config_fingerprint: str, + content_fingerprint: str, +) -> str: + combined = f"{config_fingerprint}:{content_fingerprint}" + return hashlib.sha256(combined.encode("utf-8")).hexdigest() + + +def find_cached_pdf_report( + db: "Session", + *, + evaluation_id: UUID, + organization_id: UUID, + cache_fingerprint: str, +) -> "CallImportEvaluationPdfReport | None": + from sqlalchemy import desc + + from app.models.database import CallImportEvaluationPdfReport + + if not cache_fingerprint: + return None + return ( + db.query(CallImportEvaluationPdfReport) + .filter( + CallImportEvaluationPdfReport.evaluation_id == evaluation_id, + CallImportEvaluationPdfReport.organization_id == organization_id, + CallImportEvaluationPdfReport.cache_fingerprint == cache_fingerprint, + CallImportEvaluationPdfReport.s3_key.isnot(None), + ) + .order_by(desc(CallImportEvaluationPdfReport.created_at)) + .first() + ) + + +def config_summary_from_report_config(report_config: dict[str, Any] | None) -> str: + cfg = report_config if isinstance(report_config, dict) else {} + quality_ids = cfg.get("quality_metric_ids") or [] + insight_ids = cfg.get("insights") or [] + user_insight_ids = cfg.get("user_insight_ids") or [] + metric_count = len(quality_ids) if isinstance(quality_ids, list) else 0 + insight_count = len(insight_ids) if isinstance(insight_ids, list) else 0 + user_count = len(user_insight_ids) if isinstance(user_insight_ids, list) else 0 + parts: list[str] = [] + if metric_count: + parts.append(f"{metric_count} quality metric{'s' if metric_count != 1 else ''}") + if insight_count: + parts.append(f"{insight_count} insight{'s' if insight_count != 1 else ''}") + if user_count: + parts.append(f"{user_count} user insight{'s' if user_count != 1 else ''}") + return ", ".join(parts) if parts else "default sections" + + +def presigned_urls_for_pdf_report( + s3_key: str, + filename: str, + *, + expiration: int = 3600, +) -> tuple[str | None, str | None]: + if not s3_key or not s3_service.is_enabled(): + return None, None + safe_name = filename.replace('"', "'") + try: + preview_url = s3_service.generate_presigned_url_by_key( + s3_key, + expiration=expiration, + response_content_disposition="inline", + ) + download_url = s3_service.generate_presigned_url_by_key( + s3_key, + expiration=expiration, + response_content_disposition=f'attachment; filename="{safe_name}"', + ) + return preview_url, download_url + except Exception: + return None, None diff --git a/app/services/storage/azure_blob_service.py b/app/services/storage/azure_blob_service.py index 3b1cb19e..9e6e8425 100644 --- a/app/services/storage/azure_blob_service.py +++ b/app/services/storage/azure_blob_service.py @@ -553,7 +553,13 @@ def generate_presigned_url( key = self._get_key(file_id, file_format) return self.generate_presigned_url_by_key(key, expiration=expiration) - def generate_presigned_url_by_key(self, key: str, expiration: int = 3600) -> str: + def generate_presigned_url_by_key( + self, + key: str, + expiration: int = 3600, + *, + response_content_disposition: str | None = None, + ) -> str: """Generate a SAS URL for temporary file access by key.""" self._ensure_initialized() if not self.is_enabled(): @@ -574,14 +580,17 @@ def generate_presigned_url_by_key(self, key: str, expiration: int = 3600) -> str ) try: - sas_token = generate_blob_sas( - account_name=account_name, - container_name=self.bucket_name, - blob_name=key, - account_key=account_key, - permission=BlobSasPermissions(read=True), - expiry=datetime.now(UTC) + timedelta(seconds=expiration), - ) + sas_kwargs: dict = { + "account_name": account_name, + "container_name": self.bucket_name, + "blob_name": key, + "account_key": account_key, + "permission": BlobSasPermissions(read=True), + "expiry": datetime.now(UTC) + timedelta(seconds=expiration), + } + if response_content_disposition: + sas_kwargs["content_disposition"] = response_content_disposition + sas_token = generate_blob_sas(**sas_kwargs) blob_client = self.container_client.get_blob_client(key) return f"{blob_client.url}?{sas_token}" except Exception as e: diff --git a/app/services/storage/blob_storage_service.py b/app/services/storage/blob_storage_service.py index 5f4b3bc4..03374464 100644 --- a/app/services/storage/blob_storage_service.py +++ b/app/services/storage/blob_storage_service.py @@ -128,8 +128,18 @@ def generate_presigned_url( ) -> str: return self._backend().generate_presigned_url(file_id, file_format, expiration) - def generate_presigned_url_by_key(self, key: str, expiration: int = 3600) -> str: - return self._backend().generate_presigned_url_by_key(key, expiration) + def generate_presigned_url_by_key( + self, + key: str, + expiration: int = 3600, + *, + response_content_disposition: str | None = None, + ) -> str: + return self._backend().generate_presigned_url_by_key( + key, + expiration, + response_content_disposition=response_content_disposition, + ) blob_storage_service = BlobStorageService() diff --git a/app/services/storage/gcs_service.py b/app/services/storage/gcs_service.py index 06a53550..07213e62 100644 --- a/app/services/storage/gcs_service.py +++ b/app/services/storage/gcs_service.py @@ -553,7 +553,13 @@ def generate_presigned_url( key = self._get_key(file_id, file_format) return self.generate_presigned_url_by_key(key, expiration=expiration) - def generate_presigned_url_by_key(self, key: str, expiration: int = 3600) -> str: + def generate_presigned_url_by_key( + self, + key: str, + expiration: int = 3600, + *, + response_content_disposition: str | None = None, + ) -> str: """Generate a signed URL for temporary file access by key.""" self._ensure_initialized() if not self.is_enabled(): @@ -567,23 +573,27 @@ def generate_presigned_url_by_key(self, key: str, expiration: int = 3600) -> str if credentials is None and iam_params is None: raise StorageError(_GCS_SIGNING_UNAVAILABLE_MSG) + signed_url_kwargs: dict = { + "version": "v4", + "expiration": timedelta(seconds=expiration), + "method": "GET", + } + if response_content_disposition: + signed_url_kwargs["response_disposition"] = response_content_disposition + try: blob = self.bucket.blob(key) if credentials is not None: url = blob.generate_signed_url( - version="v4", - expiration=timedelta(seconds=expiration), - method="GET", credentials=credentials, + **signed_url_kwargs, ) else: sa_email, access_token = iam_params url = blob.generate_signed_url( - version="v4", - expiration=timedelta(seconds=expiration), - method="GET", service_account_email=sa_email, access_token=access_token, + **signed_url_kwargs, ) return url except GoogleCloudError as e: diff --git a/app/services/storage/s3_service.py b/app/services/storage/s3_service.py index 1c22d168..b8376a85 100644 --- a/app/services/storage/s3_service.py +++ b/app/services/storage/s3_service.py @@ -478,7 +478,13 @@ def generate_presigned_url(self, file_id: uuid.UUID, file_format: str, expiratio except Exception as e: raise StorageError(f"Unexpected error generating presigned URL: {str(e)}") - def generate_presigned_url_by_key(self, key: str, expiration: int = 3600) -> str: + def generate_presigned_url_by_key( + self, + key: str, + expiration: int = 3600, + *, + response_content_disposition: str | None = None, + ) -> str: """Generate a presigned URL for temporary file access by key.""" self._ensure_initialized() if not self.is_enabled(): @@ -486,9 +492,12 @@ def generate_presigned_url_by_key(self, key: str, expiration: int = 3600) -> str raise StorageError(error_msg) try: + params: dict[str, str] = {"Bucket": self.bucket_name, "Key": key} + if response_content_disposition: + params["ResponseContentDisposition"] = response_content_disposition url = self.s3_client.generate_presigned_url( "get_object", - Params={"Bucket": self.bucket_name, "Key": key}, + Params=params, ExpiresIn=expiration, ) return url diff --git a/env.example b/env.example index 0de4f151..cd34c25a 100644 --- a/env.example +++ b/env.example @@ -25,6 +25,8 @@ MAX_FILE_SIZE_MB=500 ALLOWED_AUDIO_FORMATS=wav,mp3,flac,m4a # S3 (optional — used when config.docker.yml references ${S3_ACCESS_KEY_ID}) +# Required for call-import PDF report history (stored artifacts); without blob storage, +# PDF generation still streams inline but is not versioned. # S3_ACCESS_KEY_ID= # S3_SECRET_ACCESS_KEY= diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 04e1b025..15724db3 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -43,6 +43,8 @@ import type { CallImportPreviewResponse, CallImportEvaluation, CallImportEvaluationBaselineCandidatesResponse, + CallImportEvaluationPdfReport, + CallImportEvaluationPdfReportListResponse, CallImportEvaluationLLMOverride, CallImportEvaluationListResponse, CallImportEvaluationRow, @@ -2882,7 +2884,7 @@ class ApiClient { reportConfig?: Record platformBaseUrl?: string | null }, - ): Promise { + ): Promise { const response = await this.client.post( `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/pdf-report`, { @@ -2897,7 +2899,33 @@ class ApiClient { report_config: options?.reportConfig || {}, platform_base_url: options?.platformBaseUrl || null, }, - { responseType: 'blob' }, + { responseType: 'arraybuffer' }, + ) + const contentType = String(response.headers['content-type'] || '') + if (contentType.includes('application/pdf')) { + return new Blob([response.data], { type: 'application/pdf' }) + } + const text = new TextDecoder().decode(response.data) + return JSON.parse(text) as CallImportEvaluationPdfReport + } + + async listCallImportEvaluationPdfReports( + callImportId: string, + evaluationId: string, + ): Promise { + const response = await this.client.get( + `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/pdf-reports`, + ) + return response.data + } + + async getCallImportEvaluationPdfReport( + callImportId: string, + evaluationId: string, + reportId: string, + ): Promise { + const response = await this.client.get( + `/api/v1/call-imports/${callImportId}/evaluations/${evaluationId}/pdf-reports/${reportId}`, ) return response.data } diff --git a/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx b/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx index 0c42f91a..7e68664c 100644 --- a/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx +++ b/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx @@ -435,6 +435,8 @@ export default function CallImportEvaluationDetail() { const [forceFailPendingOpen, setForceFailPendingOpen] = useState(false) const [downloadMenuOpen, setDownloadMenuOpen] = useState(false) const downloadMenuRef = useRef(null) + const [pdfReportMenuOpen, setPdfReportMenuOpen] = useState(false) + const pdfReportMenuRef = useRef(null) const [pdfReportOpen, setPdfReportOpen] = useState(false) const [pdfWizardStep, setPdfWizardStep] = useState(1) const [pdfUserInsightsTriggering, setPdfUserInsightsTriggering] = @@ -890,6 +892,12 @@ export default function CallImportEvaluationDetail() { }, }) + const pdfReportsQuery = useQuery({ + queryKey: ['call-import-evaluation-pdf-reports', activeWorkspaceId, id, evalId], + queryFn: () => apiClient.listCallImportEvaluationPdfReports(id!, evalId!), + enabled: pdfReportMenuOpen && !!id && !!evalId, + }) + useEffect(() => { if (!deepLinkConversationId && !deepLinkRowId) return if (deepLinkConversationId && searchQuery !== deepLinkConversationId) { @@ -1817,6 +1825,33 @@ export default function CallImportEvaluationDetail() { ) } + const openStoredPdfPreview = async (reportId: string) => { + if (!id || !evalId) return + try { + const report = await apiClient.getCallImportEvaluationPdfReport( + id, + evalId, + reportId, + ) + if (!report.preview_url) { + throw new Error('Preview URL is not available.') + } + if (pdfPreviewUrl?.startsWith('blob:')) { + window.URL.revokeObjectURL(pdfPreviewUrl) + } + setPdfPreviewUrl(report.preview_url) + setPdfPreviewFilename(report.filename || 'report.pdf') + setPdfPreviewOpen(true) + setPdfReportMenuOpen(false) + } catch (e: unknown) { + console.error('Failed to open stored PDF report', e) + showToast( + getApiErrorMessage(e, 'Failed to open PDF preview.'), + 'error', + ) + } + } + const handlePdfReportSubmit = async () => { if (!id || !evalId || pdfReportLoading) return const vendorName = pdfVendorName.trim() @@ -1827,20 +1862,45 @@ export default function CallImportEvaluationDetail() { setPdfReportLoadingAction('download') setPdfReportError(null) try { - const blob = await generatePdfReportBlob(vendorName) + const result = await generatePdfReportBlob(vendorName) const vendorSlug = vendorName .toLowerCase() .replace(/[^a-z0-9]+/g, '-') .replace(/^-+|-+$/g, '') || 'client' - const url = window.URL.createObjectURL(blob) - const link = document.createElement('a') - link.href = url - link.download = `${vendorSlug}-${pdfReportType}-quality-metric-audit-${evalId}.pdf` - document.body.appendChild(link) - link.click() - link.remove() - window.URL.revokeObjectURL(url) + const defaultFilename = `${vendorSlug}-${pdfReportType}-quality-metric-audit-${evalId}.pdf` + + if (result instanceof Blob) { + const url = window.URL.createObjectURL(result) + const link = document.createElement('a') + link.href = url + link.download = defaultFilename + document.body.appendChild(link) + link.click() + link.remove() + window.URL.revokeObjectURL(url) + } else { + const downloadUrl = result.download_url + if (!downloadUrl) { + throw new Error('Download URL is not available.') + } + const link = document.createElement('a') + link.href = downloadUrl + link.download = result.filename || defaultFilename + link.target = '_blank' + link.rel = 'noopener noreferrer' + document.body.appendChild(link) + link.click() + link.remove() + await queryClient.invalidateQueries({ + queryKey: [ + 'call-import-evaluation-pdf-reports', + activeWorkspaceId, + id, + evalId, + ], + }) + } setPdfReportOpen(false) setPdfWizardStep(1) setPdfVendorName('') @@ -1866,18 +1926,43 @@ export default function CallImportEvaluationDetail() { setPdfReportLoadingAction('preview') setPdfReportError(null) try { - const blob = await generatePdfReportBlob(vendorName) + const result = await generatePdfReportBlob(vendorName) const vendorSlug = vendorName .toLowerCase() .replace(/[^a-z0-9]+/g, '-') .replace(/^-+|-+$/g, '') || 'client' - if (pdfPreviewUrl) window.URL.revokeObjectURL(pdfPreviewUrl) - const url = window.URL.createObjectURL(blob) - setPdfPreviewUrl(url) - setPdfPreviewFilename( - `${vendorSlug}-${pdfReportType}-quality-metric-audit-${evalId}.pdf`, - ) + + if (result instanceof Blob) { + if (pdfPreviewUrl?.startsWith('blob:')) { + window.URL.revokeObjectURL(pdfPreviewUrl) + } + const url = window.URL.createObjectURL(result) + setPdfPreviewUrl(url) + setPdfPreviewFilename( + `${vendorSlug}-${pdfReportType}-quality-metric-audit-${evalId}.pdf`, + ) + } else { + if (!result.preview_url) { + throw new Error('Preview URL is not available.') + } + if (pdfPreviewUrl?.startsWith('blob:')) { + window.URL.revokeObjectURL(pdfPreviewUrl) + } + setPdfPreviewUrl(result.preview_url) + setPdfPreviewFilename( + result.filename || + `${vendorSlug}-${pdfReportType}-quality-metric-audit-${evalId}.pdf`, + ) + await queryClient.invalidateQueries({ + queryKey: [ + 'call-import-evaluation-pdf-reports', + activeWorkspaceId, + id, + evalId, + ], + }) + } setPdfPreviewOpen(true) } catch (e: unknown) { console.error('Failed to preview PDF report', e) @@ -2165,6 +2250,20 @@ export default function CallImportEvaluationDetail() { return () => document.removeEventListener('mousedown', handleClickOutside) }, [downloadMenuOpen]) + useEffect(() => { + if (!pdfReportMenuOpen) return + const handleClickOutside = (event: MouseEvent) => { + if ( + pdfReportMenuRef.current && + !pdfReportMenuRef.current.contains(event.target as Node) + ) { + setPdfReportMenuOpen(false) + } + } + document.addEventListener('mousedown', handleClickOutside) + return () => document.removeEventListener('mousedown', handleClickOutside) + }, [pdfReportMenuOpen]) + if (!id || !evalId) { return
Missing identifiers.
} @@ -2347,20 +2446,90 @@ export default function CallImportEvaluationDetail() { Re-run metrics )} - +
+ + {pdfReportMenuOpen && ( +
+
+ +
+
+ History +
+
+ {pdfReportsQuery.isLoading ? ( +

Loading…

+ ) : (pdfReportsQuery.data?.items?.length ?? 0) === 0 ? ( +

+ No stored reports yet. +

+ ) : ( +
    + {pdfReportsQuery.data!.items.map((item) => ( +
  • + +
    • + ))} +
+ )} +
+
+ )} +
+ + {!collapsed && ( +
+ {justSaved && ( +
+
+ +

+ Mapping saved. Continue to Run Evaluation below to + fetch recordings, diarize transcripts, and score each row. +

+
+
+ )}
@@ -300,9 +388,11 @@ export default function MappingPanel({ callImport }: MappingPanelProps) { isLoading={mappingMutation.isPending} disabled={!canSubmit} > - {isAlreadyMapped ? 'Save mapping' : 'Save and continue'} + {isMapped ? 'Save mapping' : 'Save and continue'}
+
+ )}
) } diff --git a/frontend/src/pages/callImports/components/RunEvaluationStep.tsx b/frontend/src/pages/callImports/components/RunEvaluationStep.tsx index 7fef9cf2..1a78da35 100644 --- a/frontend/src/pages/callImports/components/RunEvaluationStep.tsx +++ b/frontend/src/pages/callImports/components/RunEvaluationStep.tsx @@ -4,6 +4,7 @@ import Button from '../../../components/Button' interface RunEvaluationStepProps { onRunEvaluation: () => void disabled?: boolean + highlighted?: boolean } /** @@ -14,9 +15,16 @@ interface RunEvaluationStepProps { export default function RunEvaluationStep({ onRunEvaluation, disabled = false, + highlighted = false, }: RunEvaluationStepProps) { return ( -
+
From 7e09155b1cb4b37d57c9d69266b4f24eee7d8af8 Mon Sep 17 00:00:00 2001 From: M Sami Date: Wed, 12 Aug 2026 00:40:30 +0530 Subject: [PATCH 15/32] feat: integrate LLM usage tracking and reporting across various components --- .gitignore | 5 + app/api/v1/api.py | 2 + app/api/v1/routes/call_import_evaluations.py | 64 ++- app/api/v1/routes/chat.py | 38 +- app/api/v1/routes/org_usage.py | 426 +++++++++++++++ app/app_factory.py | 2 + app/cli.py | 16 +- app/core/usage_context_middleware.py | 43 ++ app/dependencies.py | 16 + app/migrations/062_llm_usage_daily.py | 92 ++++ app/models/database.py | 37 ++ app/services/ai/llm_gateway.py | 26 +- app/services/ai/llm_service.py | 24 + app/services/usage/__init__.py | 32 ++ app/services/usage/context.py | 177 +++++++ app/services/usage/llm_usage.py | 491 ++++++++++++++++++ app/services/usage/normalize.py | 99 ++++ app/services/usage/voice_usage_processor.py | 80 +++ app/services/voice_agent/bot_fast_api.py | 35 +- app/workers/config.py | 9 + app/workers/tasks/__init__.py | 2 + app/workers/tasks/evaluate_call_import_row.py | 21 + app/workers/tasks/flush_usage_counters.py | 18 + .../generate_evaluation_tldr_insights.py | 29 +- app/workers/tasks/process_evaluator_result.py | 22 + .../tasks/transcribe_call_import_row.py | 49 ++ frontend/src/App.tsx | 4 + frontend/src/components/Layout.tsx | 8 + frontend/src/lib/api.ts | 82 +++ frontend/src/pages/usage/Usage.tsx | 294 +++++++++++ tests/test_services/test_usage/__init__.py | 0 .../test_usage/test_llm_usage.py | 387 ++++++++++++++ 32 files changed, 2584 insertions(+), 46 deletions(-) create mode 100644 app/api/v1/routes/org_usage.py create mode 100644 app/core/usage_context_middleware.py create mode 100644 app/migrations/062_llm_usage_daily.py create mode 100644 app/services/usage/__init__.py create mode 100644 app/services/usage/context.py create mode 100644 app/services/usage/llm_usage.py create mode 100644 app/services/usage/normalize.py create mode 100644 app/services/usage/voice_usage_processor.py create mode 100644 app/workers/tasks/flush_usage_counters.py create mode 100644 frontend/src/pages/usage/Usage.tsx create mode 100644 tests/test_services/test_usage/__init__.py create mode 100644 tests/test_services/test_usage/test_llm_usage.py diff --git a/.gitignore b/.gitignore index df987a07..8a24e3a7 100644 --- a/.gitignore +++ b/.gitignore @@ -110,4 +110,9 @@ frontend/dist/ # Enterprise license private keys enterprise/keys/*.pem +# Celery Beat persistent schedule (SQLite + WAL sidecars) +celerybeat-schedule +celerybeat-schedule-* +celerybeat-schedule.* + # \ No newline at end of file diff --git a/app/api/v1/api.py b/app/api/v1/api.py index d6dc6d9d..51d81945 100644 --- a/app/api/v1/api.py +++ b/app/api/v1/api.py @@ -45,6 +45,7 @@ workspace_iam, dashboard, llm_gateway, + org_usage, ) api_router = APIRouter() @@ -93,3 +94,4 @@ api_router.include_router(workspace_iam.router) api_router.include_router(dashboard.router) api_router.include_router(llm_gateway.router) +api_router.include_router(org_usage.router) diff --git a/app/api/v1/routes/call_import_evaluations.py b/app/api/v1/routes/call_import_evaluations.py index 109004a3..030c98ef 100644 --- a/app/api/v1/routes/call_import_evaluations.py +++ b/app/api/v1/routes/call_import_evaluations.py @@ -3702,16 +3702,32 @@ async def generate_call_import_evaluation_pdf_report( ) return _pdf_report_response_from_row(cached_pdf_report, cache_hit=True) - narrative = _generate_report_narrative( - db, - organization_id, - metric_aggregates=metric_aggregates, - insight_aggregates=insight_aggregates if is_internal else [], - period_delta_by_metric=period_delta_by_metric, - evidence_samples=evidence_samples if is_internal else {}, - report_config=report_config, + from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + llm_usage_context, ) + with llm_usage_context( + LLMUsageContext( + organization_id=organization_id, + workspace_id=getattr(call_import, "workspace_id", None) + or evaluation.workspace_id, + product_section=LLMUsageProductSection.CALL_IMPORT_EVALUATIONS, + resource_id=evaluation.id, + resource_type="call_import_evaluation", + ) + ): + narrative = _generate_report_narrative( + db, + organization_id, + metric_aggregates=metric_aggregates, + insight_aggregates=insight_aggregates if is_internal else [], + period_delta_by_metric=period_delta_by_metric, + evidence_samples=evidence_samples if is_internal else {}, + report_config=report_config, + ) + generated_at = datetime.now(timezone.utc) try: pdf_started = datetime.now(timezone.utc) @@ -5337,21 +5353,35 @@ def _generate_and_persist_tldr_summary( from app.services.ai.llm_resolver import get_llm_provider_and_model from app.services.ai.llm_service import llm_service + from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + llm_usage_context, + ) provider_enum, model_str = get_llm_provider_and_model( organization_id, db, provider, model ) try: - llm_result = llm_service.generate_response( - messages=messages, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - temperature=0.4, - max_tokens=1400, - ) + with llm_usage_context( + LLMUsageContext( + organization_id=organization_id, + workspace_id=evaluation.workspace_id, + product_section=LLMUsageProductSection.CALL_IMPORT_EVALUATIONS, + resource_id=evaluation.id, + resource_type="call_import_evaluation", + ) + ): + llm_result = llm_service.generate_response( + messages=messages, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + temperature=0.4, + max_tokens=1400, + ) except Exception as e: logger.error(f"[CallImportInsights] LLM call failed: {e}") raise HTTPException( diff --git a/app/api/v1/routes/chat.py b/app/api/v1/routes/chat.py index a8317844..6fc026f8 100644 --- a/app/api/v1/routes/chat.py +++ b/app/api/v1/routes/chat.py @@ -47,20 +47,32 @@ async def chat_completion( ): """Generate a chat completion using the specified AI provider and model.""" try: - # Convert ChatMessage to dict format expected by LLM service - messages = [{"role": msg.role, "content": msg.content} for msg in request.messages] - - result = llm_service.generate_response( - messages=messages, - llm_provider=request.provider, - llm_model=request.model, - organization_id=organization_id, - db=db, - llm_config=request.llm_config, - temperature=request.temperature, - max_tokens=request.max_tokens, - task_defaults={"temperature": 0.7}, + from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + llm_usage_context, ) + + messages = [{"role": msg.role, "content": msg.content} for msg in request.messages] + + with llm_usage_context( + LLMUsageContext( + organization_id=organization_id, + workspace_id=workspace_id, + product_section=LLMUsageProductSection.CHAT, + ) + ): + result = llm_service.generate_response( + messages=messages, + llm_provider=request.provider, + llm_model=request.model, + organization_id=organization_id, + db=db, + llm_config=request.llm_config, + temperature=request.temperature, + max_tokens=request.max_tokens, + task_defaults={"temperature": 0.7}, + ) background_tasks.add_task( record_chat_completion, diff --git a/app/api/v1/routes/org_usage.py b/app/api/v1/routes/org_usage.py new file mode 100644 index 00000000..05f984a0 --- /dev/null +++ b/app/api/v1/routes/org_usage.py @@ -0,0 +1,426 @@ +"""Org-scoped LLM Usage API (tokens + call counts).""" + +from __future__ import annotations + +from datetime import date, datetime, timedelta, timezone +from typing import List, Literal, Optional +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel, Field +from sqlalchemy import func +from sqlalchemy.orm import Session + +from app.database import get_db +from app.dependencies import get_organization_id +from app.models.database import CallImportEvaluation, LLMUsageDaily, Workspace +from app.services.usage.llm_usage import flush_usage_to_catalog, merge_usage_totals + +router = APIRouter(prefix="/organizations/usage", tags=["Usage"]) + +GroupBy = Literal["workspace", "product_section", "model", "resource"] + +SECTION_LABELS = { + "call_import_evaluations": "Call Import Evaluations", + "call_imports": "Call Imports", + "playground": "Playground", + "voice_playground": "Voice Playground", + "evaluators": "Evaluators", + "metrics": "Metrics", + "chat": "Chat", + "judge_alignment": "Judge Alignment", + "prompt_optimization": "Prompt Optimization", + "personas": "Personas", + "agents": "Agents", + "prompt_partials": "Prompt Partials", + "conversation_evaluations": "Conversation Evaluations", + "telephony": "Telephony", + "test_agent": "Test Agent", + "other": "Other", +} + + +class UsageTotals(BaseModel): + prompt_tokens: int = 0 + completion_tokens: int = 0 + total_tokens: int = 0 + cache_read_tokens: int = 0 + cache_creation_tokens: int = 0 + reasoning_tokens: int = 0 + call_count: int = 0 + + +class UsageSummaryResponse(BaseModel): + start: date + end: date + totals: UsageTotals + last_updated_at: Optional[datetime] = None + + +class UsageBreakdownRow(BaseModel): + workspace_id: Optional[UUID] = None + workspace_name: Optional[str] = None + product_section: Optional[str] = None + product_section_label: Optional[str] = None + model: Optional[str] = None + resource_id: Optional[UUID] = None + resource_type: Optional[str] = None + resource_label: Optional[str] = None + prompt_tokens: int = 0 + completion_tokens: int = 0 + total_tokens: int = 0 + cache_read_tokens: int = 0 + cache_creation_tokens: int = 0 + reasoning_tokens: int = 0 + call_count: int = 0 + + +class UsageBreakdownResponse(BaseModel): + start: date + end: date + group_by: GroupBy + rows: List[UsageBreakdownRow] + total_count: int + last_updated_at: Optional[datetime] = None + + +class UsageFiltersResponse(BaseModel): + workspaces: List[dict] = Field(default_factory=list) + product_sections: List[dict] = Field(default_factory=list) + models: List[str] = Field(default_factory=list) + resources: List[dict] = Field(default_factory=list) + + +def _default_range() -> tuple[date, date]: + end = datetime.now(timezone.utc).date() + start = end - timedelta(days=29) + return start, end + + +def _apply_filters( + query, + *, + organization_id: UUID, + start: date, + end: date, + workspace_id: Optional[UUID], + product_section: Optional[str], + model: Optional[str], + resource_id: Optional[UUID], +): + query = query.filter( + LLMUsageDaily.organization_id == organization_id, + LLMUsageDaily.usage_date >= start, + LLMUsageDaily.usage_date <= end, + ) + if workspace_id is not None: + query = query.filter(LLMUsageDaily.workspace_id == workspace_id) + if product_section: + query = query.filter(LLMUsageDaily.product_section == product_section) + if model: + query = query.filter(LLMUsageDaily.model == model) + if resource_id is not None: + query = query.filter(LLMUsageDaily.resource_id == resource_id) + return query + + +def _last_updated(db: Session, organization_id: UUID) -> Optional[datetime]: + return ( + db.query(func.max(LLMUsageDaily.updated_at)) + .filter(LLMUsageDaily.organization_id == organization_id) + .scalar() + ) + + +@router.get("/summary", response_model=UsageSummaryResponse) +def get_usage_summary( + start: Optional[date] = Query(None), + end: Optional[date] = Query(None), + workspace_id: Optional[UUID] = Query(None), + product_section: Optional[str] = Query(None), + model: Optional[str] = Query(None), + resource_id: Optional[UUID] = Query(None), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +): + if start is None or end is None: + start, end = _default_range() + if end < start: + raise HTTPException(status_code=400, detail="end must be >= start") + + flush_usage_to_catalog(db, organization_id) + + query = _apply_filters( + db.query(LLMUsageDaily), + organization_id=organization_id, + start=start, + end=end, + workspace_id=workspace_id, + product_section=product_section, + model=model, + resource_id=resource_id, + ) + rows = query.all() + totals = merge_usage_totals(rows) + return UsageSummaryResponse( + start=start, + end=end, + totals=UsageTotals(**totals), + last_updated_at=_last_updated(db, organization_id), + ) + + +@router.get("/breakdown", response_model=UsageBreakdownResponse) +def get_usage_breakdown( + start: Optional[date] = Query(None), + end: Optional[date] = Query(None), + group_by: GroupBy = Query("workspace"), + workspace_id: Optional[UUID] = Query(None), + product_section: Optional[str] = Query(None), + model: Optional[str] = Query(None), + resource_id: Optional[UUID] = Query(None), + limit: int = Query(100, ge=1, le=500), + offset: int = Query(0, ge=0), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +): + if start is None or end is None: + start, end = _default_range() + if end < start: + raise HTTPException(status_code=400, detail="end must be >= start") + + flush_usage_to_catalog(db, organization_id) + + dim = { + "workspace": LLMUsageDaily.workspace_id, + "product_section": LLMUsageDaily.product_section, + "model": LLMUsageDaily.model, + "resource": LLMUsageDaily.resource_id, + }[group_by] + + aggregates = [ + func.coalesce(func.sum(LLMUsageDaily.prompt_tokens), 0).label("prompt_tokens"), + func.coalesce(func.sum(LLMUsageDaily.completion_tokens), 0).label( + "completion_tokens" + ), + func.coalesce(func.sum(LLMUsageDaily.cache_read_tokens), 0).label( + "cache_read_tokens" + ), + func.coalesce(func.sum(LLMUsageDaily.cache_creation_tokens), 0).label( + "cache_creation_tokens" + ), + func.coalesce(func.sum(LLMUsageDaily.reasoning_tokens), 0).label( + "reasoning_tokens" + ), + func.coalesce(func.sum(LLMUsageDaily.call_count), 0).label("call_count"), + ] + + select_cols = [dim] + group_cols = [dim] + if group_by == "resource": + select_cols.append(LLMUsageDaily.resource_type) + group_cols.append(LLMUsageDaily.resource_type) + + query = _apply_filters( + db.query(*select_cols, *aggregates), + organization_id=organization_id, + start=start, + end=end, + workspace_id=workspace_id, + product_section=product_section, + model=model, + resource_id=resource_id, + ).group_by(*group_cols) + + total_count = query.count() + results = ( + query.order_by(func.sum(LLMUsageDaily.call_count).desc()) + .offset(offset) + .limit(limit) + .all() + ) + + workspace_names = { + w.id: w.name + for w in db.query(Workspace) + .filter(Workspace.organization_id == organization_id) + .all() + } + resource_labels: dict = {} + if group_by == "resource": + resource_ids = [r[0] for r in results if r[0] is not None] + if resource_ids: + evals = ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.organization_id == organization_id, + CallImportEvaluation.id.in_(resource_ids), + ) + .all() + ) + for evaluation in evals: + label = (evaluation.name or "").strip() or str(evaluation.id)[:8] + resource_labels[evaluation.id] = label + + rows: List[UsageBreakdownRow] = [] + for result in results: + if group_by == "workspace": + ws_id = result[0] + metrics = result[1:] + rows.append( + UsageBreakdownRow( + workspace_id=ws_id, + workspace_name=workspace_names.get(ws_id) if ws_id else "Unknown", + prompt_tokens=int(metrics[0]), + completion_tokens=int(metrics[1]), + total_tokens=int(metrics[0]) + int(metrics[1]), + cache_read_tokens=int(metrics[2]), + cache_creation_tokens=int(metrics[3]), + reasoning_tokens=int(metrics[4]), + call_count=int(metrics[5]), + ) + ) + elif group_by == "product_section": + section = result[0] + metrics = result[1:] + rows.append( + UsageBreakdownRow( + product_section=section, + product_section_label=SECTION_LABELS.get(section or "", section), + prompt_tokens=int(metrics[0]), + completion_tokens=int(metrics[1]), + total_tokens=int(metrics[0]) + int(metrics[1]), + cache_read_tokens=int(metrics[2]), + cache_creation_tokens=int(metrics[3]), + reasoning_tokens=int(metrics[4]), + call_count=int(metrics[5]), + ) + ) + elif group_by == "model": + model_name = result[0] + metrics = result[1:] + rows.append( + UsageBreakdownRow( + model=model_name, + prompt_tokens=int(metrics[0]), + completion_tokens=int(metrics[1]), + total_tokens=int(metrics[0]) + int(metrics[1]), + cache_read_tokens=int(metrics[2]), + cache_creation_tokens=int(metrics[3]), + reasoning_tokens=int(metrics[4]), + call_count=int(metrics[5]), + ) + ) + else: + res_id, res_type = result[0], result[1] + metrics = result[2:] + rows.append( + UsageBreakdownRow( + resource_id=res_id, + resource_type=res_type, + resource_label=resource_labels.get(res_id) + if res_id + else "Unscoped", + prompt_tokens=int(metrics[0]), + completion_tokens=int(metrics[1]), + total_tokens=int(metrics[0]) + int(metrics[1]), + cache_read_tokens=int(metrics[2]), + cache_creation_tokens=int(metrics[3]), + reasoning_tokens=int(metrics[4]), + call_count=int(metrics[5]), + ) + ) + + return UsageBreakdownResponse( + start=start, + end=end, + group_by=group_by, + rows=rows, + total_count=total_count, + last_updated_at=_last_updated(db, organization_id), + ) + + +@router.get("/filters", response_model=UsageFiltersResponse) +def get_usage_filters( + start: Optional[date] = Query(None), + end: Optional[date] = Query(None), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +): + if start is None or end is None: + start, end = _default_range() + + flush_usage_to_catalog(db, organization_id) + + base = db.query(LLMUsageDaily).filter( + LLMUsageDaily.organization_id == organization_id, + LLMUsageDaily.usage_date >= start, + LLMUsageDaily.usage_date <= end, + ) + + workspace_ids = { + row[0] + for row in base.with_entities(LLMUsageDaily.workspace_id).distinct().all() + if row[0] is not None + } + workspaces = [ + {"id": str(w.id), "name": w.name} + for w in db.query(Workspace) + .filter(Workspace.id.in_(workspace_ids) if workspace_ids else False) + .order_by(Workspace.name) + .all() + ] + + sections = sorted( + { + row[0] + for row in base.with_entities(LLMUsageDaily.product_section).distinct().all() + if row[0] + } + ) + models = sorted( + { + row[0] + for row in base.with_entities(LLMUsageDaily.model).distinct().all() + if row[0] + } + ) + + resource_rows = ( + base.with_entities(LLMUsageDaily.resource_id, LLMUsageDaily.resource_type) + .filter(LLMUsageDaily.resource_id.isnot(None)) + .distinct() + .limit(100) + .all() + ) + resource_ids = [r[0] for r in resource_rows] + eval_names = {} + if resource_ids: + for evaluation in ( + db.query(CallImportEvaluation) + .filter(CallImportEvaluation.id.in_(resource_ids)) + .all() + ): + eval_names[evaluation.id] = (evaluation.name or "").strip() or str( + evaluation.id + )[:8] + + resources = [ + { + "id": str(rid), + "type": rtype, + "label": eval_names.get(rid, str(rid)[:8]), + } + for rid, rtype in resource_rows + if rid is not None + ] + + return UsageFiltersResponse( + workspaces=workspaces, + product_sections=[ + {"id": s, "label": SECTION_LABELS.get(s, s)} for s in sections + ], + models=models, + resources=resources, + ) diff --git a/app/app_factory.py b/app/app_factory.py index 2b6b8d61..c2af3e09 100644 --- a/app/app_factory.py +++ b/app/app_factory.py @@ -19,6 +19,7 @@ from app.core.operational_access_middleware import OperationalAccessMiddleware from app.core.rbac_middleware import ReaderReadOnlyMiddleware from app.core.security_headers_middleware import SecurityHeadersMiddleware +from app.core.usage_context_middleware import LLMUsageContextMiddleware from app.database import init_db logger = logging.getLogger(__name__) @@ -98,6 +99,7 @@ def _add_common_middleware(app: FastAPI) -> None: if _includes_http_routes(): app.add_middleware(MigrationCheckMiddleware) app.add_middleware(ReaderReadOnlyMiddleware) + app.add_middleware(LLMUsageContextMiddleware) if settings.OBSERVABILITY_ENABLED and settings.LOKI_ENABLED and settings.LOKI_MULTI_TENANT: from app.core.observability_middleware import OrgLoggingMiddleware diff --git a/app/cli.py b/app/cli.py index 3078344f..453b2a2d 100644 --- a/app/cli.py +++ b/app/cli.py @@ -692,6 +692,7 @@ def start_all( # Store worker processes for cleanup. worker_process = None worker_imports_process = None + beat_process = None telephony_process = None def _terminate(proc, label: str): @@ -711,8 +712,9 @@ def _terminate(proc, label: str): def cleanup_processes(): """Clean up spawned processes.""" - nonlocal worker_process, worker_imports_process, telephony_process + nonlocal worker_process, worker_imports_process, beat_process, telephony_process _terminate(telephony_process, "Telephony media server") + _terminate(beat_process, "Celery beat") _terminate(worker_process, "Celery worker (default)") _terminate(worker_imports_process, "Celery worker (imports)") @@ -867,6 +869,18 @@ def _stream_telephony(): ), prefix="[WORKER-IMPORTS]", ) + + beat_process = _spawn_worker( + [ + "celery", + "-A", + "app.workers.celery_app", + "beat", + f"--loglevel={worker_loglevel}", + ], + label="Celery beat (periodic flush_usage_counters)", + prefix="[BEAT]", + ) except FileNotFoundError: click.echo("❌ Celery not found. Please install it: pip install celery", err=True) sys.exit(1) diff --git a/app/core/usage_context_middleware.py b/app/core/usage_context_middleware.py new file mode 100644 index 00000000..c9531c98 --- /dev/null +++ b/app/core/usage_context_middleware.py @@ -0,0 +1,43 @@ +"""Clear LLM usage ContextVar at request boundaries to avoid thread reuse leaks.""" + +from __future__ import annotations + +from uuid import UUID + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request + +from app.services.usage.context import ( + infer_product_section_from_path, + reset_usage_context, + reset_usage_hints, + set_usage_context, + set_usage_hints, +) + + +class LLMUsageContextMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + section = infer_product_section_from_path(request.url.path) + request.state.usage_product_section = section + + workspace_hint = None + raw_ws = request.headers.get("x-workspace-id") or request.query_params.get( + "workspace_id" + ) + if raw_ws: + try: + workspace_hint = UUID(raw_ws) + except (TypeError, ValueError): + workspace_hint = None + + ctx_token = set_usage_context(None) + hint_tokens = set_usage_hints( + workspace_id=workspace_hint, + product_section=section, + ) + try: + return await call_next(request) + finally: + reset_usage_hints(hint_tokens) + reset_usage_context(ctx_token) diff --git a/app/dependencies.py b/app/dependencies.py index 6277777a..c3597d75 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -193,6 +193,22 @@ def get_workspace_context( is_org_admin=org_role == RoleEnum.ADMIN, ) request.state.workspace_context = ctx + + from app.services.usage.context import ( + LLMUsageProductSection, + ensure_usage_context, + ) + + section = getattr( + request.state, "usage_product_section", LLMUsageProductSection.OTHER + ) + ensure_usage_context( + organization_id, + workspace_id=workspace.id, + product_section=section + if isinstance(section, LLMUsageProductSection) + else LLMUsageProductSection.OTHER, + ) return ctx diff --git a/app/migrations/062_llm_usage_daily.py b/app/migrations/062_llm_usage_daily.py new file mode 100644 index 00000000..e160c955 --- /dev/null +++ b/app/migrations/062_llm_usage_daily.py @@ -0,0 +1,92 @@ +"""Migration: LLM usage daily rollups for org-scoped Usage reporting.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add llm_usage_daily table for LLM token/call rollups" + + +def upgrade(db: Session): + exists = db.execute( + text( + """ + SELECT 1 FROM information_schema.tables + WHERE table_name = 'llm_usage_daily' + """ + ) + ).first() + if exists: + print("llm_usage_daily already exists, skipping 062") + db.commit() + return + + db.execute( + text( + """ + CREATE TABLE llm_usage_daily ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id UUID NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + workspace_id UUID REFERENCES workspaces(id) ON DELETE SET NULL, + product_section VARCHAR(64) NOT NULL, + model VARCHAR(255) NOT NULL, + resource_id UUID, + resource_type VARCHAR(64), + usage_date DATE NOT NULL, + prompt_tokens BIGINT NOT NULL DEFAULT 0, + completion_tokens BIGINT NOT NULL DEFAULT 0, + cache_read_tokens BIGINT NOT NULL DEFAULT 0, + cache_creation_tokens BIGINT NOT NULL DEFAULT 0, + reasoning_tokens BIGINT NOT NULL DEFAULT 0, + call_count BIGINT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX ix_llm_usage_daily_org_date + ON llm_usage_daily (organization_id, usage_date) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX ix_llm_usage_daily_org_workspace_date + ON llm_usage_daily (organization_id, workspace_id, usage_date) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX ix_llm_usage_daily_org_resource_date + ON llm_usage_daily (organization_id, resource_id, usage_date) + """ + ) + ) + db.execute( + text( + """ + CREATE UNIQUE INDEX uq_llm_usage_daily_bucket + ON llm_usage_daily ( + organization_id, + COALESCE(workspace_id, '00000000-0000-0000-0000-000000000000'::uuid), + product_section, + model, + COALESCE(resource_id, '00000000-0000-0000-0000-000000000000'::uuid), + usage_date + ) + """ + ) + ) + db.commit() + print("Created llm_usage_daily table") + + +def downgrade(db: Session): + db.execute(text("DROP TABLE IF EXISTS llm_usage_daily")) + db.commit() diff --git a/app/models/database.py b/app/models/database.py index 68bcfebc..9f32f212 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -2777,3 +2777,40 @@ class JudgeRun(Base): created_by = Column(String, nullable=True) dataset = relationship("JudgeDataset", back_populates="runs") + + +class LLMUsageDaily(Base): + """Daily LLM usage rollups for org-scoped Usage reporting.""" + + __tablename__ = "llm_usage_daily" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + product_section = Column(String(64), nullable=False, index=True) + model = Column(String(255), nullable=False, index=True) + resource_id = Column(UUID(as_uuid=True), nullable=True, index=True) + resource_type = Column(String(64), nullable=True) + usage_date = Column(Date, nullable=False, index=True) + prompt_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") + completion_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") + cache_read_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") + cache_creation_tokens = Column( + BigInteger, nullable=False, default=0, server_default="0" + ) + reasoning_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") + call_count = Column(BigInteger, nullable=False, default=0, server_default="0") + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) diff --git a/app/services/ai/llm_gateway.py b/app/services/ai/llm_gateway.py index b0731840..750b1641 100644 --- a/app/services/ai/llm_gateway.py +++ b/app/services/ai/llm_gateway.py @@ -705,4 +705,28 @@ def litellm_completion( db=db, credential=credential, ) - return litellm.completion(**kwargs) + response = litellm.completion(**kwargs) + try: + from app.services.usage.context import ( + LLMUsageProductSection, + ensure_usage_context, + reset_usage_context, + ) + from app.services.usage.llm_usage import record_llm_usage + from app.services.usage.normalize import normalize_llm_usage + + usage_token = ensure_usage_context( + organization_id, + product_section=LLMUsageProductSection.OTHER, + ) + try: + model_name = str(kwargs.get("model") or "unknown") + if "/" in model_name: + model_name = model_name.rsplit("/", 1)[-1] + record_llm_usage(model_name, normalize_llm_usage(raw_response=response)) + finally: + if usage_token is not None: + reset_usage_context(usage_token) + except Exception as exc: + logger.debug("litellm_completion usage record skipped: {}", exc) + return response diff --git a/app/services/ai/llm_service.py b/app/services/ai/llm_service.py index fb22285d..a0b2090a 100644 --- a/app/services/ai/llm_service.py +++ b/app/services/ai/llm_service.py @@ -549,6 +549,30 @@ def generate_response( "raw_response": response, "processing_time": time.time() - start_time, } + try: + from app.services.usage.context import ( + LLMUsageProductSection, + ensure_usage_context, + reset_usage_context, + ) + from app.services.usage.normalize import normalize_llm_usage + from app.services.usage.llm_usage import record_llm_usage + + usage_token = ensure_usage_context( + organization_id, + product_section=LLMUsageProductSection.OTHER, + ) + try: + snapshot = normalize_llm_usage(raw_response=response) + result["usage"]["cache_read_tokens"] = snapshot.cache_read_tokens + result["usage"]["cache_creation_tokens"] = snapshot.cache_creation_tokens + result["usage"]["reasoning_tokens"] = snapshot.reasoning_tokens + record_llm_usage(llm_model, snapshot) + finally: + if usage_token is not None: + reset_usage_context(usage_token) + except Exception as exc: + logger.debug("llm usage record skipped: {}", exc) return result diff --git a/app/services/usage/__init__.py b/app/services/usage/__init__.py new file mode 100644 index 00000000..646e9adb --- /dev/null +++ b/app/services/usage/__init__.py @@ -0,0 +1,32 @@ +"""LLM usage tracking (tokens, calls) with Redis buffer and catalog rollups.""" + +from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + ensure_usage_context, + infer_product_section_from_path, + llm_usage_context, + reset_usage_context, + reset_usage_hints, + set_usage_context, + set_usage_hints, +) +from app.services.usage.llm_usage import flush_all_usage_to_catalog, flush_usage_to_catalog, record_llm_usage +from app.services.usage.normalize import UsageSnapshot, normalize_llm_usage + +__all__ = [ + "LLMUsageContext", + "LLMUsageProductSection", + "UsageSnapshot", + "ensure_usage_context", + "flush_all_usage_to_catalog", + "flush_usage_to_catalog", + "infer_product_section_from_path", + "llm_usage_context", + "normalize_llm_usage", + "record_llm_usage", + "reset_usage_context", + "reset_usage_hints", + "set_usage_context", + "set_usage_hints", +] diff --git a/app/services/usage/context.py b/app/services/usage/context.py new file mode 100644 index 00000000..80e84fac --- /dev/null +++ b/app/services/usage/context.py @@ -0,0 +1,177 @@ +"""Usage attribution context (org, workspace, product section, resource).""" + +from __future__ import annotations + +from contextlib import contextmanager +from contextvars import ContextVar, Token +from dataclasses import dataclass +from enum import Enum +from typing import Iterator, Optional +from uuid import UUID + + +class LLMUsageProductSection(str, Enum): + CALL_IMPORT_EVALUATIONS = "call_import_evaluations" + CALL_IMPORTS = "call_imports" + PLAYGROUND = "playground" + VOICE_PLAYGROUND = "voice_playground" + EVALUATORS = "evaluators" + METRICS = "metrics" + CHAT = "chat" + JUDGE_ALIGNMENT = "judge_alignment" + PROMPT_OPTIMIZATION = "prompt_optimization" + PERSONAS = "personas" + AGENTS = "agents" + PROMPT_PARTIALS = "prompt_partials" + CONVERSATION_EVALUATIONS = "conversation_evaluations" + TELEPHONY = "telephony" + TEST_AGENT = "test_agent" + OTHER = "other" + + +@dataclass(frozen=True) +class LLMUsageContext: + organization_id: UUID + workspace_id: Optional[UUID] = None + product_section: LLMUsageProductSection = LLMUsageProductSection.OTHER + resource_id: Optional[UUID] = None + resource_type: Optional[str] = None + + +_usage_context_var: ContextVar[Optional[LLMUsageContext]] = ContextVar( + "llm_usage_context", default=None +) +_usage_workspace_hint_var: ContextVar[Optional[UUID]] = ContextVar( + "llm_usage_workspace_hint", default=None +) +_usage_section_hint_var: ContextVar[LLMUsageProductSection] = ContextVar( + "llm_usage_section_hint", default=LLMUsageProductSection.OTHER +) + +# Path fragments under /api/v1 → product section (longest match wins). +_PATH_SECTION_RULES: tuple[tuple[str, LLMUsageProductSection], ...] = ( + ("call-import-evaluations", LLMUsageProductSection.CALL_IMPORT_EVALUATIONS), + ("call-imports", LLMUsageProductSection.CALL_IMPORTS), + ("voice-playground", LLMUsageProductSection.VOICE_PLAYGROUND), + ("playground", LLMUsageProductSection.PLAYGROUND), + ("evaluators", LLMUsageProductSection.EVALUATORS), + ("metrics", LLMUsageProductSection.METRICS), + ("chat", LLMUsageProductSection.CHAT), + ("judge-alignment", LLMUsageProductSection.JUDGE_ALIGNMENT), + ("prompt-optimization", LLMUsageProductSection.PROMPT_OPTIMIZATION), + ("personas", LLMUsageProductSection.PERSONAS), + ("agents", LLMUsageProductSection.AGENTS), + ("prompt-partials", LLMUsageProductSection.PROMPT_PARTIALS), + ("conversation-evaluations", LLMUsageProductSection.CONVERSATION_EVALUATIONS), + ("telephony", LLMUsageProductSection.TELEPHONY), + ("test-agent", LLMUsageProductSection.TEST_AGENT), +) + + +def get_usage_context() -> Optional[LLMUsageContext]: + return _usage_context_var.get() + + +def set_usage_context(ctx: Optional[LLMUsageContext]) -> Token: + return _usage_context_var.set(ctx) + + +def reset_usage_context(token: Token) -> None: + _usage_context_var.reset(token) + + +def set_usage_hints( + *, + workspace_id: Optional[UUID] = None, + product_section: Optional[LLMUsageProductSection] = None, +) -> tuple[Token, Token]: + ws_token = _usage_workspace_hint_var.set(workspace_id) + section = product_section or LLMUsageProductSection.OTHER + section_token = _usage_section_hint_var.set(section) + return ws_token, section_token + + +def reset_usage_hints(tokens: tuple[Token, Token]) -> None: + _usage_workspace_hint_var.reset(tokens[0]) + _usage_section_hint_var.reset(tokens[1]) + + +def get_usage_workspace_hint() -> Optional[UUID]: + return _usage_workspace_hint_var.get() + + +def get_usage_section_hint() -> LLMUsageProductSection: + return _usage_section_hint_var.get() + + +@contextmanager +def llm_usage_context(ctx: LLMUsageContext) -> Iterator[None]: + token = set_usage_context(ctx) + try: + yield + finally: + reset_usage_context(token) + + +def infer_product_section_from_path(path: str) -> LLMUsageProductSection: + normalized = (path or "").lower() + for fragment, section in _PATH_SECTION_RULES: + if f"/{fragment}" in normalized or normalized.endswith(fragment): + return section + return LLMUsageProductSection.OTHER + + +def ensure_usage_context( + organization_id: UUID, + *, + workspace_id: Optional[UUID] = None, + product_section: LLMUsageProductSection = LLMUsageProductSection.OTHER, + resource_id: Optional[UUID] = None, + resource_type: Optional[str] = None, +) -> Token | None: + """Set or enrich usage context. Returns token to reset, or None if unchanged.""" + resolved_workspace = workspace_id or get_usage_workspace_hint() + resolved_section = product_section + if resolved_section == LLMUsageProductSection.OTHER: + hint = get_usage_section_hint() + if hint != LLMUsageProductSection.OTHER: + resolved_section = hint + + current = get_usage_context() + if current is None: + return set_usage_context( + LLMUsageContext( + organization_id=organization_id, + workspace_id=resolved_workspace, + product_section=resolved_section, + resource_id=resource_id, + resource_type=resource_type, + ) + ) + + upgraded_workspace = current.workspace_id or resolved_workspace + upgraded_section = ( + resolved_section + if current.product_section == LLMUsageProductSection.OTHER + and resolved_section != LLMUsageProductSection.OTHER + else current.product_section + ) + upgraded_resource_id = current.resource_id or resource_id + upgraded_resource_type = current.resource_type or resource_type + if ( + upgraded_workspace == current.workspace_id + and upgraded_section == current.product_section + and upgraded_resource_id == current.resource_id + and upgraded_resource_type == current.resource_type + ): + return None + + return set_usage_context( + LLMUsageContext( + organization_id=current.organization_id, + workspace_id=upgraded_workspace, + product_section=upgraded_section, + resource_id=upgraded_resource_id, + resource_type=upgraded_resource_type, + ) + ) diff --git a/app/services/usage/llm_usage.py b/app/services/usage/llm_usage.py new file mode 100644 index 00000000..4c1fcecb --- /dev/null +++ b/app/services/usage/llm_usage.py @@ -0,0 +1,491 @@ +"""Redis-buffered LLM usage counters with catalog rollup flush.""" + +from __future__ import annotations + +import time +import uuid +from datetime import date, datetime, timezone +from typing import Any, Dict, Iterable, List, Optional, Tuple +from uuid import UUID + +import redis +from loguru import logger +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.config import settings +from app.services.usage.context import ( + LLMUsageContext, + get_usage_context, +) +from app.services.usage.normalize import UsageSnapshot + +_redis: redis.Redis | None = None + +_NONE = "__none__" +_PENDING_TTL_SECONDS = 14 * 24 * 60 * 60 +_FLUSH_LOCK_TTL_SECONDS = 45 +_FLUSH_LOCK_WAIT_SECONDS = 3.0 +_METRIC_FIELDS = ( + "prompt_tokens", + "completion_tokens", + "cache_read_tokens", + "cache_creation_tokens", + "reasoning_tokens", + "call_count", +) + +# Atomically move pending hash → claim key so only one flusher owns the deltas. +_CLAIM_LUA = """ +if redis.call('EXISTS', KEYS[1]) == 0 then + return 0 +end +redis.call('RENAME', KEYS[1], KEYS[2]) +return 1 +""" + + +def _client() -> redis.Redis: + global _redis + if _redis is None: + _redis = redis.from_url(settings.REDIS_URL, decode_responses=True) + return _redis + + +def _token(value: Optional[UUID]) -> str: + return str(value) if value else _NONE + + +def _pending_hash_key(organization_id: UUID) -> str: + return f"usage:pending:{organization_id}" + + +def _flush_lock_key(organization_id: UUID) -> str: + return f"usage:flush:lock:{organization_id}" + + +def _claim_hash_key(organization_id: UUID, claim_id: str) -> str: + return f"usage:flushing:{organization_id}:{claim_id}" + + +def _bucket_prefix( + *, + workspace_id: Optional[UUID], + product_section: str, + model: str, + resource_id: Optional[UUID], + resource_type: Optional[str], + usage_date: date, +) -> str: + return "|".join( + [ + _token(workspace_id), + product_section, + model, + _token(resource_id), + resource_type or _NONE, + usage_date.isoformat(), + ] + ) + + +def _parse_bucket_prefix(prefix: str) -> Optional[Dict[str, Any]]: + parts = prefix.split("|") + if len(parts) != 6: + return None + ws_token, section, model, resource_token, resource_type, day_str = parts + try: + usage_date = date.fromisoformat(day_str) + except ValueError: + return None + workspace_id = None if ws_token == _NONE else UUID(ws_token) + resource_id = None if resource_token == _NONE else UUID(resource_token) + resolved_resource_type = None if resource_type == _NONE else resource_type + return { + "workspace_id": workspace_id, + "product_section": section, + "model": model, + "resource_id": resource_id, + "resource_type": resolved_resource_type, + "usage_date": usage_date, + } + + +def _resolve_context(ctx: Optional[LLMUsageContext]) -> LLMUsageContext: + if ctx is not None: + return ctx + current = get_usage_context() + if current is not None: + return current + raise ValueError("LLM usage context is not set") + + +def _deltas_from_usage(usage: UsageSnapshot) -> Dict[str, int]: + return { + "prompt_tokens": usage.prompt_tokens, + "completion_tokens": usage.completion_tokens, + "cache_read_tokens": usage.cache_read_tokens, + "cache_creation_tokens": usage.cache_creation_tokens, + "reasoning_tokens": usage.reasoning_tokens, + "call_count": 1, + } + + +def record_llm_usage( + model: str, + usage: UsageSnapshot, + *, + ctx: Optional[LLMUsageContext] = None, + usage_date: Optional[date] = None, +) -> None: + """Increment Redis counters for one LLM call (best-effort, never raises).""" + if not model: + model = "unknown" + try: + context = _resolve_context(ctx) + except ValueError: + logger.debug("llm usage record skipped: missing context") + return + + deltas = _deltas_from_usage(usage) + if not any(deltas.values()): + return + + day = usage_date or datetime.now(timezone.utc).date() + prefix = _bucket_prefix( + workspace_id=context.workspace_id, + product_section=context.product_section.value, + model=model, + resource_id=context.resource_id, + resource_type=context.resource_type, + usage_date=day, + ) + hash_key = _pending_hash_key(context.organization_id) + + try: + client = _client() + pipe = client.pipeline() + for metric, delta in deltas.items(): + if delta: + pipe.hincrby(hash_key, f"{prefix}|{metric}", int(delta)) + pipe.sadd("usage:pending:orgs", str(context.organization_id)) + pipe.expire(hash_key, _PENDING_TTL_SECONDS) + pipe.execute() + except redis.RedisError as exc: + logger.warning("llm usage counter skipped: {}", exc) + + +def _parse_hash_to_buckets(raw: Dict[str, str]) -> Dict[str, Dict[str, int]]: + buckets: Dict[str, Dict[str, int]] = {} + for field, value in raw.items(): + if "|" not in field: + continue + prefix, metric = field.rsplit("|", 1) + if metric not in _METRIC_FIELDS: + continue + amount = int(value or 0) + if not amount: + continue + bucket = buckets.setdefault(prefix, {}) + bucket[metric] = bucket.get(metric, 0) + amount + return buckets + + +def _read_hash_buckets(hash_key: str) -> Dict[str, Dict[str, int]]: + try: + client = _client() + raw = client.hgetall(hash_key) + except redis.RedisError as exc: + logger.warning("llm usage read hash failed: {}", exc) + return {} + return _parse_hash_to_buckets(raw) + + +def _restore_buckets_to_pending( + organization_id: UUID, buckets: Dict[str, Dict[str, int]] +) -> None: + try: + client = _client() + pipe = client.pipeline() + hash_key = _pending_hash_key(organization_id) + for prefix, metrics in buckets.items(): + for metric, amount in metrics.items(): + if amount: + pipe.hincrby(hash_key, f"{prefix}|{metric}", amount) + pipe.sadd("usage:pending:orgs", str(organization_id)) + pipe.expire(hash_key, _PENDING_TTL_SECONDS) + pipe.execute() + except redis.RedisError as exc: + logger.warning("llm usage redis restore failed: {}", exc) + + +def _acquire_flush_lock(organization_id: UUID) -> bool: + try: + client = _client() + lock_key = _flush_lock_key(organization_id) + if client.set(lock_key, "1", nx=True, ex=_FLUSH_LOCK_TTL_SECONDS): + return True + deadline = time.monotonic() + _FLUSH_LOCK_WAIT_SECONDS + while time.monotonic() < deadline: + time.sleep(0.05) + if client.set(lock_key, "1", nx=True, ex=_FLUSH_LOCK_TTL_SECONDS): + return True + if client.get(lock_key) is None: + continue + return False + except redis.RedisError as exc: + logger.warning("llm usage flush lock failed: {}", exc) + return False + + +def _release_flush_lock(organization_id: UUID) -> None: + try: + _client().delete(_flush_lock_key(organization_id)) + except redis.RedisError: + pass + + +def _claim_pending( + organization_id: UUID, +) -> Tuple[Optional[str], Dict[str, Dict[str, int]]]: + """Rename pending → claim key. Returns (claim_key, buckets) or (None, {}).""" + claim_id = str(uuid.uuid4()) + pending_key = _pending_hash_key(organization_id) + claim_key = _claim_hash_key(organization_id, claim_id) + try: + client = _client() + claimed = int(client.eval(_CLAIM_LUA, 2, pending_key, claim_key) or 0) + if not claimed: + return None, {} + buckets = _read_hash_buckets(claim_key) + if not buckets: + client.delete(claim_key) + return None, {} + return claim_key, buckets + except redis.RedisError as exc: + logger.warning("llm usage claim failed: {}", exc) + return None, {} + + +def _ack_claim(claim_key: str, organization_id: UUID) -> None: + try: + client = _client() + client.delete(claim_key) + pending_key = _pending_hash_key(organization_id) + if not client.exists(pending_key): + client.srem("usage:pending:orgs", str(organization_id)) + except redis.RedisError: + pass + + +def _upsert_bucket( + db: Session, + organization_id: UUID, + bucket: Dict[str, Any], + deltas: Dict[str, int], + *, + resource_type: Optional[str], +) -> None: + params = { + "organization_id": str(organization_id), + "workspace_id": str(bucket["workspace_id"]) if bucket["workspace_id"] else None, + "product_section": bucket["product_section"], + "model": bucket["model"], + "resource_id": str(bucket["resource_id"]) if bucket["resource_id"] else None, + "resource_type": resource_type, + "usage_date": bucket["usage_date"].isoformat(), + "prompt_tokens": int(deltas.get("prompt_tokens", 0)), + "completion_tokens": int(deltas.get("completion_tokens", 0)), + "cache_read_tokens": int(deltas.get("cache_read_tokens", 0)), + "cache_creation_tokens": int(deltas.get("cache_creation_tokens", 0)), + "reasoning_tokens": int(deltas.get("reasoning_tokens", 0)), + "call_count": int(deltas.get("call_count", 0)), + } + result = db.execute( + text( + """ + UPDATE llm_usage_daily SET + prompt_tokens = prompt_tokens + :prompt_tokens, + completion_tokens = completion_tokens + :completion_tokens, + cache_read_tokens = cache_read_tokens + :cache_read_tokens, + cache_creation_tokens = cache_creation_tokens + :cache_creation_tokens, + reasoning_tokens = reasoning_tokens + :reasoning_tokens, + call_count = call_count + :call_count, + resource_type = COALESCE(resource_type, :resource_type), + updated_at = now() + WHERE organization_id = CAST(:organization_id AS uuid) + AND product_section = :product_section + AND model = :model + AND usage_date = CAST(:usage_date AS date) + AND workspace_id IS NOT DISTINCT FROM CAST(:workspace_id AS uuid) + AND resource_id IS NOT DISTINCT FROM CAST(:resource_id AS uuid) + """ + ), + params, + ) + if result.rowcount: + return + + db.execute( + text( + """ + INSERT INTO llm_usage_daily ( + id, organization_id, workspace_id, product_section, model, + resource_id, resource_type, usage_date, + prompt_tokens, completion_tokens, cache_read_tokens, + cache_creation_tokens, reasoning_tokens, call_count, + created_at, updated_at + ) VALUES ( + gen_random_uuid(), CAST(:organization_id AS uuid), + CAST(:workspace_id AS uuid), :product_section, :model, + CAST(:resource_id AS uuid), :resource_type, CAST(:usage_date AS date), + :prompt_tokens, :completion_tokens, :cache_read_tokens, + :cache_creation_tokens, :reasoning_tokens, :call_count, + now(), now() + ) + """ + ), + params, + ) + + +def _is_missing_organization_fk(exc: BaseException) -> bool: + """True when insert failed because organization_id is not in organizations.""" + text_blob = " ".join( + str(part) + for part in (exc, getattr(exc, "orig", None), getattr(exc, "args", None)) + if part is not None + ).lower() + return "llm_usage_daily_organization_id_fkey" in text_blob + + +def flush_usage_to_catalog(db: Session, organization_id: UUID) -> int: + """Claim Redis deltas, commit to llm_usage_daily, then ack the claim.""" + if not _acquire_flush_lock(organization_id): + return 0 + + claim_key = None + buckets: Dict[str, Dict[str, int]] = {} + try: + claim_key, buckets = _claim_pending(organization_id) + if not claim_key or not buckets: + return 0 + + flushed = 0 + try: + for prefix, deltas in buckets.items(): + parsed = _parse_bucket_prefix(prefix) + if not parsed: + continue + _upsert_bucket( + db, + organization_id, + parsed, + deltas, + resource_type=parsed.get("resource_type"), + ) + flushed += 1 + db.commit() + except Exception as exc: + db.rollback() + if _is_missing_organization_fk(exc): + # Stale/test org ids must not be restored — that loops forever on beat. + logger.warning( + "llm usage flush dropped for unknown organization {}: {}", + organization_id, + exc, + ) + if claim_key: + _ack_claim(claim_key, organization_id) + return 0 + logger.warning("llm usage catalog flush failed, restoring redis: {}", exc) + _restore_buckets_to_pending(organization_id, buckets) + if claim_key: + try: + _client().delete(claim_key) + except redis.RedisError: + pass + return 0 + + _ack_claim(claim_key, organization_id) + return flushed + finally: + _release_flush_lock(organization_id) + + +def _recover_orphaned_claims() -> None: + """Re-queue claim hashes left behind by a crashed flusher.""" + try: + client = _client() + for claim_key in client.scan_iter(match="usage:flushing:*", count=100): + parts = claim_key.split(":") + if len(parts) < 4: + continue + try: + org_id = UUID(parts[2]) + except ValueError: + continue + if client.exists(_flush_lock_key(org_id)): + continue + buckets = _read_hash_buckets(claim_key) + if buckets: + _restore_buckets_to_pending(org_id, buckets) + client.delete(claim_key) + except redis.RedisError as exc: + logger.warning("llm usage orphan claim recovery failed: {}", exc) + + +def list_pending_organization_ids() -> List[UUID]: + try: + client = _client() + raw_ids = client.smembers("usage:pending:orgs") + result: List[UUID] = [] + stale: List[str] = [] + for value in raw_ids: + try: + org_id = UUID(value) + except ValueError: + stale.append(value) + continue + if client.exists(_pending_hash_key(org_id)): + result.append(org_id) + else: + stale.append(value) + if stale: + client.srem("usage:pending:orgs", *stale) + return result + except (redis.RedisError, ValueError): + return [] + + +def flush_all_usage_to_catalog(db_factory) -> int: + """Flush all orgs with pending usage (Celery beat).""" + _recover_orphaned_claims() + total = 0 + for org_id in list_pending_organization_ids(): + db = db_factory() + try: + total += flush_usage_to_catalog(db, org_id) + except Exception as exc: + db.rollback() + logger.warning("flush_all usage failed for {}: {}", org_id, exc) + finally: + db.close() + return total + + +def merge_usage_totals( + rows: Iterable[Any], +) -> Dict[str, int]: + totals = {field: 0 for field in _METRIC_FIELDS} + for row in rows: + totals["prompt_tokens"] += int(getattr(row, "prompt_tokens", 0) or 0) + totals["completion_tokens"] += int(getattr(row, "completion_tokens", 0) or 0) + totals["cache_read_tokens"] += int(getattr(row, "cache_read_tokens", 0) or 0) + totals["cache_creation_tokens"] += int( + getattr(row, "cache_creation_tokens", 0) or 0 + ) + totals["reasoning_tokens"] += int(getattr(row, "reasoning_tokens", 0) or 0) + totals["call_count"] += int(getattr(row, "call_count", 0) or 0) + totals["total_tokens"] = totals["prompt_tokens"] + totals["completion_tokens"] + return totals diff --git a/app/services/usage/normalize.py b/app/services/usage/normalize.py new file mode 100644 index 00000000..bfdcf5f6 --- /dev/null +++ b/app/services/usage/normalize.py @@ -0,0 +1,99 @@ +"""Normalize LiteLLM / provider usage objects into UsageSnapshot.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Mapping, Optional + + +@dataclass(frozen=True) +class UsageSnapshot: + prompt_tokens: int + completion_tokens: int + cache_read_tokens: int = 0 + cache_creation_tokens: int = 0 + reasoning_tokens: int = 0 + + @property + def total_tokens(self) -> int: + return self.prompt_tokens + self.completion_tokens + + +def _as_int(value: Any) -> int: + try: + return int(value or 0) + except (TypeError, ValueError): + return 0 + + +def _usage_mapping(raw: Any) -> Mapping[str, Any]: + if raw is None: + return {} + if isinstance(raw, Mapping): + return raw + if hasattr(raw, "model_dump"): + try: + return raw.model_dump() + except Exception: + pass + if hasattr(raw, "__dict__"): + return { + key: value + for key, value in vars(raw).items() + if not key.startswith("_") + } + return {} + + +def _details_dict(usage: Mapping[str, Any], key: str) -> Mapping[str, Any]: + details = usage.get(key) + if details is None: + return {} + if isinstance(details, Mapping): + return details + if hasattr(details, "model_dump"): + try: + return details.model_dump() + except Exception: + pass + if hasattr(details, "__dict__"): + return { + k: v for k, v in vars(details).items() if not k.startswith("_") + } + return {} + + +def normalize_llm_usage(raw_response: Any = None, *, usage: Any = None) -> UsageSnapshot: + """Extract token buckets from a LiteLLM response or raw usage object.""" + usage_obj = usage + if usage_obj is None and raw_response is not None: + usage_obj = getattr(raw_response, "usage", None) + + data = _usage_mapping(usage_obj) + prompt_tokens = _as_int(data.get("prompt_tokens") or data.get("input_tokens")) + completion_tokens = _as_int( + data.get("completion_tokens") or data.get("output_tokens") + ) + + cache_read = _as_int(data.get("cache_read_input_tokens")) + cache_creation = _as_int(data.get("cache_creation_input_tokens")) + + prompt_details = _details_dict(data, "prompt_tokens_details") + if not cache_read: + cache_read = _as_int(prompt_details.get("cached_tokens")) + if not cache_creation: + cache_creation = _as_int( + prompt_details.get("cache_write_tokens") + or prompt_details.get("cache_creation_tokens") + ) + + completion_details = _details_dict(data, "completion_tokens_details") + reasoning_tokens = _as_int(completion_details.get("reasoning_tokens")) + + return UsageSnapshot( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cache_read_tokens=cache_read, + cache_creation_tokens=cache_creation, + reasoning_tokens=reasoning_tokens, + ) diff --git a/app/services/usage/voice_usage_processor.py b/app/services/usage/voice_usage_processor.py new file mode 100644 index 00000000..56d65f57 --- /dev/null +++ b/app/services/usage/voice_usage_processor.py @@ -0,0 +1,80 @@ +"""Voice pipeline processor that records LLM token usage from MetricsFrames.""" + +from __future__ import annotations + +from typing import Optional +from uuid import UUID + +from loguru import logger + + +def create_llm_usage_recorder( + *, + organization_id: UUID | str | None, + workspace_id: UUID | str | None = None, + product_section: str = "playground", + resource_id: UUID | str | None = None, + resource_type: Optional[str] = None, +): + """Build a FrameProcessor that records LLM usage from MetricsFrames. + + Returns None when organization_id is missing or efficientai is unavailable. + """ + if not organization_id: + return None + + try: + from efficientai.frames.frames import Frame, MetricsFrame + from efficientai.metrics.metrics import LLMUsageMetricsData + from efficientai.processors.frame_processor import FrameDirection, FrameProcessor + + from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + set_usage_context, + ) + from app.services.usage.llm_usage import record_llm_usage + from app.services.usage.normalize import UsageSnapshot + except Exception as exc: + logger.debug("voice usage recorder unavailable: {}", exc) + return None + + try: + section = LLMUsageProductSection(product_section) + except ValueError: + section = LLMUsageProductSection.OTHER + + org_uuid = UUID(str(organization_id)) + ws_uuid = UUID(str(workspace_id)) if workspace_id else None + res_uuid = UUID(str(resource_id)) if resource_id else None + + set_usage_context( + LLMUsageContext( + organization_id=org_uuid, + workspace_id=ws_uuid, + product_section=section, + resource_id=res_uuid, + resource_type=resource_type, + ) + ) + + class LLMUsageRecorderProcessor(FrameProcessor): + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, MetricsFrame): + for item in frame.data or []: + if isinstance(item, LLMUsageMetricsData) and item.value is not None: + tokens = item.value + snapshot = UsageSnapshot( + prompt_tokens=int(tokens.prompt_tokens or 0), + completion_tokens=int(tokens.completion_tokens or 0), + cache_read_tokens=int(tokens.cache_read_input_tokens or 0), + cache_creation_tokens=int( + tokens.cache_creation_input_tokens or 0 + ), + reasoning_tokens=int(tokens.reasoning_tokens or 0), + ) + record_llm_usage(item.model or "unknown", snapshot) + await self.push_frame(frame, direction) + + return LLMUsageRecorderProcessor() diff --git a/app/services/voice_agent/bot_fast_api.py b/app/services/voice_agent/bot_fast_api.py index 38fa2733..3e2d25e4 100644 --- a/app/services/voice_agent/bot_fast_api.py +++ b/app/services/voice_agent/bot_fast_api.py @@ -98,7 +98,7 @@ def _get_imports(): """ -async def run_bot(websocket_client, google_api_key: str, system_instruction: str = None, organization_id: str = None, agent_id: str = None, persona_id: str = None, scenario_id: str = None, evaluator_id: str = None, result_id: str = None, model_name: str = None, serializer=None, telephony_mode: bool = False, call_short_id: str = None, silence_hangup_secs: float | None = None): +async def run_bot(websocket_client, google_api_key: str, system_instruction: str = None, organization_id: str = None, agent_id: str = None, persona_id: str = None, scenario_id: str = None, evaluator_id: str = None, result_id: str = None, model_name: str = None, serializer=None, telephony_mode: bool = False, call_short_id: str = None, silence_hangup_secs: float | None = None, workspace_id: str = None): """ Run the voice agent bot with the provided Google API key. @@ -246,6 +246,15 @@ async def on_silence_hangup(): if user_transcript_processor: pipeline_processors.append(user_transcript_processor) pipeline_processors.append(llm) + from app.services.usage.voice_usage_processor import create_llm_usage_recorder + + usage_recorder = create_llm_usage_recorder( + organization_id=organization_id, + workspace_id=workspace_id, + product_section="telephony" if telephony_mode else "playground", + ) + if usage_recorder: + pipeline_processors.append(usage_recorder) if agent_transcript_processor: pipeline_processors.append(agent_transcript_processor) pipeline_processors.extend([ @@ -278,18 +287,30 @@ async def on_client_disconnected(transport, client): # RTVI events for efficientai client UI rtvi = imports["RTVIProcessor"](config=imports["RTVIConfig"](config=[])) - pipeline = imports["Pipeline"]( + from app.services.usage.voice_usage_processor import create_llm_usage_recorder + + usage_recorder = create_llm_usage_recorder( + organization_id=organization_id, + workspace_id=workspace_id, + product_section="playground", + ) + pipeline_steps = [ + ws_transport.input(), + user_recorder, + context_aggregator.user(), + rtvi, + llm, + ] + if usage_recorder: + pipeline_steps.append(usage_recorder) + pipeline_steps.extend( [ - ws_transport.input(), - user_recorder, - context_aggregator.user(), - rtvi, - llm, bot_recorder, ws_transport.output(), context_aggregator.assistant(), ] ) + pipeline = imports["Pipeline"](pipeline_steps) task = imports["PipelineTask"]( pipeline, diff --git a/app/workers/config.py b/app/workers/config.py index dc1cb711..3448429d 100644 --- a/app/workers/config.py +++ b/app/workers/config.py @@ -137,4 +137,13 @@ "generate_evaluation_prompt_improvements": {"queue": "evaluations"}, "generate_agent_flowchart": {"queue": "celery"}, "map_agent_flowchart_prompt_sections": {"queue": "celery"}, + "flush_usage_counters": {"queue": "celery"}, +} + +# Periodic flush of Redis LLM usage counters into catalog rollups. +celery_app.conf.beat_schedule = { + "flush-llm-usage-counters": { + "task": "flush_usage_counters", + "schedule": 120.0, + }, } diff --git a/app/workers/tasks/__init__.py b/app/workers/tasks/__init__.py index e4862d8e..04eb0380 100644 --- a/app/workers/tasks/__init__.py +++ b/app/workers/tasks/__init__.py @@ -22,6 +22,7 @@ from . import initiate_vobiz_outbound from . import finalize_telephony_recording from . import call_import_bulk_ops +from . import flush_usage_counters from app.workers.concurrency import eval_dispatch from app.workers.concurrency import fair_dispatch from app.workers.concurrency import fair_diarization_dispatch @@ -115,3 +116,4 @@ materialize_call_import_evaluation_task = ( call_import_bulk_ops.materialize_call_import_evaluation_task ) +flush_usage_counters_task = flush_usage_counters.flush_usage_counters_task diff --git a/app/workers/tasks/evaluate_call_import_row.py b/app/workers/tasks/evaluate_call_import_row.py index 29247d24..e87b3285 100644 --- a/app/workers/tasks/evaluate_call_import_row.py +++ b/app/workers/tasks/evaluate_call_import_row.py @@ -332,6 +332,7 @@ def evaluate_call_import_row_task( slot_task_id = _eval_slot_task_id or self.request.id scoring_inputs: dict[str, Any] | None = None restricted_metric_uuids: list[UUID] | None = None + usage_ctx_token = None try: from app.db_sharding.row_ops import ( close_row_sessions, @@ -363,6 +364,22 @@ def evaluate_call_import_row_task( row_db.commit() return {"status": "failed", "reason": "evaluation_missing"} + from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + set_usage_context, + ) + + usage_ctx_token = set_usage_context( + LLMUsageContext( + organization_id=evaluation.organization_id, + workspace_id=evaluation.workspace_id, + product_section=LLMUsageProductSection.CALL_IMPORT_EVALUATIONS, + resource_id=evaluation.id, + resource_type="call_import_evaluation", + ) + ) + previous_row_status = eval_row.status eval_row.status = "running" @@ -761,6 +778,10 @@ def evaluate_call_import_row_task( if row_db is not None: close_row_sessions(row_db, catalog_db) finally: + if usage_ctx_token is not None: + from app.services.usage.context import reset_usage_context + + reset_usage_context(usage_ctx_token) from app.workers.concurrency.fair_dispatch import ( finish_eval_work_and_redispatch, ) diff --git a/app/workers/tasks/flush_usage_counters.py b/app/workers/tasks/flush_usage_counters.py new file mode 100644 index 00000000..3dc158d0 --- /dev/null +++ b/app/workers/tasks/flush_usage_counters.py @@ -0,0 +1,18 @@ +"""Celery task: flush Redis LLM usage counters into catalog rollups.""" + +from __future__ import annotations + +from loguru import logger + +from app.database import SessionLocal +from app.workers.config import celery_app + + +@celery_app.task(name="flush_usage_counters") +def flush_usage_counters_task() -> dict: + from app.services.usage.llm_usage import flush_all_usage_to_catalog + + flushed = flush_all_usage_to_catalog(SessionLocal) + if flushed: + logger.info("Flushed {} LLM usage rollup buckets", flushed) + return {"flushed_buckets": flushed} diff --git a/app/workers/tasks/generate_evaluation_tldr_insights.py b/app/workers/tasks/generate_evaluation_tldr_insights.py index c1e5de73..b236d0f7 100644 --- a/app/workers/tasks/generate_evaluation_tldr_insights.py +++ b/app/workers/tasks/generate_evaluation_tldr_insights.py @@ -40,14 +40,29 @@ def generate_evaluation_tldr_insights_task( if evaluation is None: return {"error": "evaluation_not_found", "status_code": 404} + from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + llm_usage_context, + ) + try: - summary = _generate_and_persist_tldr_summary( - db, - evaluation, - organization_id=UUID(organization_id), - provider=provider, - model=model, - ) + with llm_usage_context( + LLMUsageContext( + organization_id=evaluation.organization_id, + workspace_id=evaluation.workspace_id, + product_section=LLMUsageProductSection.CALL_IMPORT_EVALUATIONS, + resource_id=evaluation.id, + resource_type="call_import_evaluation", + ) + ): + summary = _generate_and_persist_tldr_summary( + db, + evaluation, + organization_id=UUID(organization_id), + provider=provider, + model=model, + ) except HTTPException as exc: return {"error": exc.detail, "status_code": exc.status_code} diff --git a/app/workers/tasks/process_evaluator_result.py b/app/workers/tasks/process_evaluator_result.py index d338f294..dc0e1b64 100644 --- a/app/workers/tasks/process_evaluator_result.py +++ b/app/workers/tasks/process_evaluator_result.py @@ -488,6 +488,23 @@ def process_evaluator_result_task(self, result_id: str): logger.error(f"[EvaluatorResult {result_id}] Job not found in database") return {"error": "Evaluator result not found"} + from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + set_usage_context, + reset_usage_context, + ) + + usage_token = set_usage_context( + LLMUsageContext( + organization_id=result.organization_id, + workspace_id=result.workspace_id, + product_section=LLMUsageProductSection.EVALUATORS, + resource_id=result.id, + resource_type="evaluator_result", + ) + ) + logger.info(f"[EvaluatorResult {result.result_id}] Starting processing task") result.celery_task_id = self.request.id @@ -746,4 +763,9 @@ def process_evaluator_result_task(self, result_id: str): except Exception as exc: raise self.retry(exc=exc, countdown=60) finally: + try: + if "usage_token" in locals() and usage_token is not None: + reset_usage_context(usage_token) + except Exception: + pass db.close() diff --git a/app/workers/tasks/transcribe_call_import_row.py b/app/workers/tasks/transcribe_call_import_row.py index c6e89198..0f342ab3 100644 --- a/app/workers/tasks/transcribe_call_import_row.py +++ b/app/workers/tasks/transcribe_call_import_row.py @@ -391,6 +391,11 @@ def _persist_diarization_failure( def _run_diarization_pipeline(ctx: dict[str, Any]) -> dict[str, Any]: """STT / S3 / LLM diarisation without a long-lived DB session.""" from app.models.enums import ModelProvider + from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + llm_usage_context, + ) from app.workers.tasks.helpers.llm_diarisation import ( LLMDiarisationError, diarize_audio_with_llm, @@ -406,6 +411,48 @@ def _run_diarization_pipeline(ctx: dict[str, Any]) -> dict[str, Any]: llm_credential_uuid = ctx["llm_credential_uuid"] effective_prompt = ctx["effective_prompt"] + with llm_usage_context( + LLMUsageContext( + organization_id=organization_id, + workspace_id=ctx.get("workspace_id"), + product_section=LLMUsageProductSection.CALL_IMPORTS, + resource_id=ctx.get("call_import_id"), + resource_type="call_import", + ) + ): + return _run_diarization_pipeline_inner( + ctx, + row_id=row_id, + normalised_mode=normalised_mode, + recording_key=recording_key, + organization_id=organization_id, + llm_provider_value=llm_provider_value, + llm_model_value=llm_model_value, + llm_credential_uuid=llm_credential_uuid, + effective_prompt=effective_prompt, + ModelProvider=ModelProvider, + LLMDiarisationError=LLMDiarisationError, + diarize_audio_with_llm=diarize_audio_with_llm, + diarize_transcript_with_llm=diarize_transcript_with_llm, + ) + + +def _run_diarization_pipeline_inner( + ctx: dict[str, Any], + *, + row_id, + normalised_mode, + recording_key, + organization_id, + llm_provider_value, + llm_model_value, + llm_credential_uuid, + effective_prompt, + ModelProvider, + LLMDiarisationError, + diarize_audio_with_llm, + diarize_transcript_with_llm, +) -> dict[str, Any]: plain_text: Optional[str] = None raw_turns: Optional[List[Dict[str, Any]]] = None @@ -911,6 +958,8 @@ def transcribe_call_import_row_task( "normalised_mode": normalised_mode, "recording_key": recording_key, "organization_id": row.organization_id, + "workspace_id": getattr(row, "workspace_id", None), + "call_import_id": row.call_import_id, "stt_provider": provider_enum.value if provider_enum else None, "stt_model": stt_model, "credential_uuid": credential_uuid, diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 2d745a02..d18118b0 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -11,6 +11,9 @@ import SelectOrganization from './pages/auth/SelectOrganization' // Dashboard import Dashboard from './pages/dashboard/Dashboard' +// Usage +import Usage from './pages/usage/Usage' + // Prompt Partials import PromptPartials from './pages/promptPartials/PromptPartials' @@ -182,6 +185,7 @@ function App() { } /> } /> } /> + } /> } diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index c621a485..bfe7c0ad 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -25,6 +25,7 @@ import { Mic, Bot, Activity, + PieChart, Bell, History, Key, @@ -99,6 +100,13 @@ const navigationSections: NavSection[] = [ { name: 'Calls', href: '/observability/calls', icon: Phone }, ], }, + { + title: 'Usage', + icon: PieChart, + items: [ + { name: 'Overview', href: '/usage', icon: PieChart }, + ], + }, { title: 'Alerting', icon: Bell, diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 1ac0029d..f8701723 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -2991,6 +2991,88 @@ class ApiClient { return response.data } + async getOrgUsageSummary(params: { + start?: string + end?: string + workspace_id?: string + product_section?: string + model?: string + resource_id?: string + }): Promise<{ + start: string + end: string + totals: { + prompt_tokens: number + completion_tokens: number + total_tokens: number + cache_read_tokens: number + cache_creation_tokens: number + reasoning_tokens: number + call_count: number + } + last_updated_at?: string | null + }> { + const response = await this.client.get('/api/v1/organizations/usage/summary', { + params, + }) + return response.data + } + + async getOrgUsageBreakdown(params: { + start?: string + end?: string + group_by?: 'workspace' | 'product_section' | 'model' | 'resource' + workspace_id?: string + product_section?: string + model?: string + resource_id?: string + limit?: number + offset?: number + }): Promise<{ + start: string + end: string + group_by: string + rows: Array<{ + workspace_id?: string | null + workspace_name?: string | null + product_section?: string | null + product_section_label?: string | null + model?: string | null + resource_id?: string | null + resource_type?: string | null + resource_label?: string | null + prompt_tokens: number + completion_tokens: number + total_tokens: number + cache_read_tokens: number + cache_creation_tokens: number + reasoning_tokens: number + call_count: number + }> + total_count: number + last_updated_at?: string | null + }> { + const response = await this.client.get('/api/v1/organizations/usage/breakdown', { + params, + }) + return response.data + } + + async getOrgUsageFilters(params?: { + start?: string + end?: string + }): Promise<{ + workspaces: Array<{ id: string; name: string }> + product_sections: Array<{ id: string; label: string }> + models: string[] + resources: Array<{ id: string; type?: string; label: string }> + }> { + const response = await this.client.get('/api/v1/organizations/usage/filters', { + params, + }) + return response.data + } + async getAllModels(): Promise> { const response = await this.client.get('/api/v1/model-config/models') return response.data diff --git a/frontend/src/pages/usage/Usage.tsx b/frontend/src/pages/usage/Usage.tsx new file mode 100644 index 00000000..0f7cdcb9 --- /dev/null +++ b/frontend/src/pages/usage/Usage.tsx @@ -0,0 +1,294 @@ +import { useMemo } from 'react' +import { useQuery } from '@tanstack/react-query' +import { useSearchParams } from 'react-router-dom' +import { Card, CardBody, Spinner } from '@heroui/react' +import { Activity } from 'lucide-react' +import { apiClient } from '../../lib/api' + +type GroupBy = 'workspace' | 'product_section' | 'model' | 'resource' + +function formatNumber(value: number): string { + return new Intl.NumberFormat().format(value || 0) +} + +function toDateInput(d: Date): string { + return d.toISOString().slice(0, 10) +} + +function FilterField({ + label, + children, +}: { + label: string + children: React.ReactNode +}) { + return ( + + ) +} + +const fieldClassName = + 'h-9 rounded-lg border border-gray-200 bg-white px-3 text-sm text-gray-900 outline-none focus:border-indigo-400 focus:ring-1 focus:ring-indigo-200' + +export default function Usage() { + const [searchParams, setSearchParams] = useSearchParams() + const today = useMemo(() => new Date(), []) + const defaultStart = useMemo(() => { + const d = new Date() + d.setDate(d.getDate() - 29) + return toDateInput(d) + }, []) + + const start = searchParams.get('start') || defaultStart + const end = searchParams.get('end') || toDateInput(today) + const groupBy = (searchParams.get('group_by') as GroupBy) || 'workspace' + const workspaceId = searchParams.get('workspace_id') || '' + const productSection = searchParams.get('product_section') || '' + const model = searchParams.get('model') || '' + const resourceId = searchParams.get('resource_id') || '' + + const setParam = (key: string, value: string) => { + const next = new URLSearchParams(searchParams) + if (!value) next.delete(key) + else next.set(key, value) + setSearchParams(next) + } + + const filterParams = { + start, + end, + workspace_id: workspaceId || undefined, + product_section: productSection || undefined, + model: model || undefined, + resource_id: resourceId || undefined, + } + + const { data: summary, isLoading: summaryLoading } = useQuery({ + queryKey: ['org-usage', 'summary', filterParams], + queryFn: () => apiClient.getOrgUsageSummary(filterParams), + }) + + const { data: breakdown, isLoading: breakdownLoading } = useQuery({ + queryKey: ['org-usage', 'breakdown', groupBy, filterParams], + queryFn: () => + apiClient.getOrgUsageBreakdown({ + ...filterParams, + group_by: groupBy, + limit: 100, + }), + }) + + const { data: filters } = useQuery({ + queryKey: ['org-usage', 'filters', start, end], + queryFn: () => apiClient.getOrgUsageFilters({ start, end }), + }) + + const totals = summary?.totals + const rows = breakdown?.rows || [] + + const dimensionLabel = (row: (typeof rows)[number]): string => { + if (groupBy === 'workspace') return row.workspace_name || 'Unknown' + if (groupBy === 'product_section') + return row.product_section_label || row.product_section || '—' + if (groupBy === 'model') return row.model || '—' + return row.resource_label || row.resource_id || 'Unscoped' + } + + return ( +
+
+
+

+ + Usage +

+

+ Organization-wide LLM tokens and calls. Filter by workspace, product + section, model, or evaluation. +

+
+ {summary?.last_updated_at && ( +

+ Updated {new Date(summary.last_updated_at).toLocaleString()} +

+ )} +
+ +
+ + + + +
+ + {(totals?.cache_read_tokens || totals?.cache_creation_tokens || totals?.reasoning_tokens) ? ( +
+ + + +
+ ) : null} + + + +
+ + setParam('start', e.target.value)} + /> + + + setParam('end', e.target.value)} + /> + + + + + + + + + + + + + +
+ + + + + + {breakdownLoading ? ( +
+ +
+ ) : rows.length === 0 ? ( +
+ No usage in this period. Run evaluations or playground calls, then + refresh in a minute. +
+ ) : ( + + + + + + + + + + + + + {rows.map((row, idx) => ( + + + + + + + + + ))} + +
+ {groupBy === 'workspace' + ? 'Workspace' + : groupBy === 'product_section' + ? 'Section' + : groupBy === 'model' + ? 'Model' + : 'Resource'} + CallsInputOutputTotalCache read
{dimensionLabel(row)} + {formatNumber(row.call_count)} + + {formatNumber(row.prompt_tokens)} + + {formatNumber(row.completion_tokens)} + + {formatNumber(row.total_tokens)} + + {formatNumber(row.cache_read_tokens)} +
+ )} + + +
+ ) +} + +function StatCard({ + label, + value, + loading, +}: { + label: string + value?: number + loading?: boolean +}) { + return ( + + +

+ {label} +

+

+ {loading ? '—' : formatNumber(value || 0)} +

+ + + ) +} diff --git a/tests/test_services/test_usage/__init__.py b/tests/test_services/test_usage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_services/test_usage/test_llm_usage.py b/tests/test_services/test_usage/test_llm_usage.py new file mode 100644 index 00000000..0fdd77bf --- /dev/null +++ b/tests/test_services/test_usage/test_llm_usage.py @@ -0,0 +1,387 @@ +"""Tests for Redis-buffered LLM usage counters and flush durability.""" + +from __future__ import annotations + +from datetime import date +from typing import Any, Dict, List, Optional +from unittest.mock import MagicMock +from uuid import uuid4 + +import pytest + +from app.services.usage import llm_usage as usage_mod +from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + ensure_usage_context, + get_usage_context, + infer_product_section_from_path, + llm_usage_context, + reset_usage_context, + set_usage_context, +) +from app.services.usage.normalize import UsageSnapshot + + +class _FakePipeline: + def __init__(self, client: "_FakeRedis"): + self._client = client + self._ops: List[tuple] = [] + + def hincrby(self, key: str, field: str, amount: int): + self._ops.append(("hincrby", key, field, amount)) + return self + + def sadd(self, key: str, *members: str): + self._ops.append(("sadd", key, members)) + return self + + def expire(self, key: str, ttl: int): + self._ops.append(("expire", key, ttl)) + return self + + def execute(self): + results = [] + for op in self._ops: + kind = op[0] + if kind == "hincrby": + results.append(self._client.hincrby(op[1], op[2], op[3])) + elif kind == "sadd": + results.append(self._client.sadd(op[1], *op[2])) + elif kind == "expire": + results.append(self._client.expire(op[1], op[2])) + self._ops.clear() + return results + + +class _FakeRedis: + """Minimal Redis stand-in for usage counter tests.""" + + def __init__(self): + self.hashes: Dict[str, Dict[str, int]] = {} + self.sets: Dict[str, set] = {} + self.kv: Dict[str, str] = {} + + def pipeline(self): + return _FakePipeline(self) + + def hincrby(self, key: str, field: str, amount: int) -> int: + bucket = self.hashes.setdefault(key, {}) + bucket[field] = int(bucket.get(field, 0)) + int(amount) + return bucket[field] + + def hgetall(self, key: str) -> Dict[str, str]: + return {k: str(v) for k, v in self.hashes.get(key, {}).items()} + + def hdel(self, key: str, *fields: str) -> int: + bucket = self.hashes.get(key, {}) + deleted = 0 + for field in fields: + if field in bucket: + del bucket[field] + deleted += 1 + if key in self.hashes and not self.hashes[key]: + del self.hashes[key] + return deleted + + def hlen(self, key: str) -> int: + return len(self.hashes.get(key, {})) + + def sadd(self, key: str, *members: str) -> int: + s = self.sets.setdefault(key, set()) + before = len(s) + s.update(members) + return len(s) - before + + def srem(self, key: str, *members: str) -> int: + s = self.sets.get(key, set()) + removed = 0 + for member in members: + if member in s: + s.remove(member) + removed += 1 + return removed + + def smembers(self, key: str) -> set: + return set(self.sets.get(key, set())) + + def exists(self, key: str) -> int: + return int(key in self.hashes or key in self.sets or key in self.kv) + + def delete(self, *keys: str) -> int: + deleted = 0 + for key in keys: + if key in self.hashes: + del self.hashes[key] + deleted += 1 + if key in self.sets: + del self.sets[key] + deleted += 1 + if key in self.kv: + del self.kv[key] + deleted += 1 + return deleted + + def expire(self, key: str, ttl: int) -> bool: + return self.exists(key) == 1 + + def set(self, key: str, value: str, nx: bool = False, ex: Optional[int] = None) -> Optional[bool]: + if nx and key in self.kv: + return None + self.kv[key] = value + return True + + def get(self, key: str) -> Optional[str]: + return self.kv.get(key) + + def rename(self, src: str, dst: str) -> bool: + if src not in self.hashes: + raise KeyError(src) + self.hashes[dst] = self.hashes.pop(src) + return True + + def eval(self, script: str, numkeys: int, *keys_and_args: Any) -> int: + pending, claim = keys_and_args[0], keys_and_args[1] + if pending not in self.hashes: + return 0 + self.rename(pending, claim) + return 1 + + def scan_iter(self, match: str = "*", count: int = 100): + prefix = match.rstrip("*") + for key in list(self.hashes.keys()): + if key.startswith(prefix): + yield key + + +@pytest.fixture(autouse=True) +def fake_redis(monkeypatch): + """Always use in-memory Redis for this module — never touch real REDIS_URL.""" + client = _FakeRedis() + usage_mod._redis = client + + def _forbid_real_redis(*_args, **_kwargs): + raise AssertionError( + "usage tests must not open real Redis; fake_redis fixture failed to isolate" + ) + + monkeypatch.setattr(usage_mod.redis, "from_url", _forbid_real_redis) + yield client + usage_mod._redis = None + + +@pytest.fixture +def org_ctx(): + org_id = uuid4() + workspace_id = uuid4() + ctx = LLMUsageContext( + organization_id=org_id, + workspace_id=workspace_id, + product_section=LLMUsageProductSection.CALL_IMPORT_EVALUATIONS, + resource_id=uuid4(), + resource_type="call_import_evaluation", + ) + return org_id, workspace_id, ctx + + +def test_record_increments_pending_and_counts_zero_token_calls(fake_redis, org_ctx): + org_id, _workspace_id, ctx = org_ctx + with llm_usage_context(ctx): + usage_mod.record_llm_usage( + "gpt-test", + UsageSnapshot(prompt_tokens=10, completion_tokens=5), + usage_date=date(2026, 8, 11), + ) + usage_mod.record_llm_usage( + "gpt-test", + UsageSnapshot(prompt_tokens=0, completion_tokens=0), + usage_date=date(2026, 8, 11), + ) + + pending_key = usage_mod._pending_hash_key(org_id) + fields = fake_redis.hgetall(pending_key) + assert str(org_id) in fake_redis.smembers("usage:pending:orgs") + + prompt = sum(int(v) for k, v in fields.items() if k.endswith("|prompt_tokens")) + completion = sum( + int(v) for k, v in fields.items() if k.endswith("|completion_tokens") + ) + calls = sum(int(v) for k, v in fields.items() if k.endswith("|call_count")) + assert prompt == 10 + assert completion == 5 + assert calls == 2 + + +def test_record_skipped_without_context(fake_redis): + usage_mod.record_llm_usage( + "gpt-test", + UsageSnapshot(prompt_tokens=10, completion_tokens=5), + ) + assert fake_redis.hashes == {} + assert fake_redis.sets == {} + + +def test_flush_commits_and_acks_claim(fake_redis, org_ctx): + org_id, _workspace_id, ctx = org_ctx + with llm_usage_context(ctx): + usage_mod.record_llm_usage( + "gpt-test", + UsageSnapshot(prompt_tokens=4, completion_tokens=2), + usage_date=date(2026, 8, 11), + ) + + db = MagicMock() + db.execute.return_value = MagicMock(rowcount=1) + + flushed = usage_mod.flush_usage_to_catalog(db, org_id) + assert flushed == 1 + db.commit.assert_called_once() + assert usage_mod._pending_hash_key(org_id) not in fake_redis.hashes + assert not any(k.startswith("usage:flushing:") for k in fake_redis.hashes) + assert str(org_id) not in fake_redis.smembers("usage:pending:orgs") + assert fake_redis.get(usage_mod._flush_lock_key(org_id)) is None + + +def test_flush_restores_redis_when_db_fails(fake_redis, org_ctx): + org_id, _workspace_id, ctx = org_ctx + with llm_usage_context(ctx): + usage_mod.record_llm_usage( + "gpt-test", + UsageSnapshot(prompt_tokens=7, completion_tokens=3), + usage_date=date(2026, 8, 11), + ) + + db = MagicMock() + db.execute.side_effect = RuntimeError("db down") + + flushed = usage_mod.flush_usage_to_catalog(db, org_id) + assert flushed == 0 + db.rollback.assert_called() + db.commit.assert_not_called() + + pending = fake_redis.hgetall(usage_mod._pending_hash_key(org_id)) + prompt = sum(int(v) for k, v in pending.items() if k.endswith("|prompt_tokens")) + completion = sum( + int(v) for k, v in pending.items() if k.endswith("|completion_tokens") + ) + assert prompt == 7 + assert completion == 3 + assert str(org_id) in fake_redis.smembers("usage:pending:orgs") + assert not any(k.startswith("usage:flushing:") for k in fake_redis.hashes) + + +def test_flush_drops_pending_when_organization_missing(fake_redis, org_ctx): + """Unknown org FK must not restore Redis (avoids infinite beat retries).""" + from sqlalchemy.exc import IntegrityError + + org_id, _workspace_id, ctx = org_ctx + with llm_usage_context(ctx): + usage_mod.record_llm_usage( + "gpt-test", + UsageSnapshot(prompt_tokens=3, completion_tokens=1), + usage_date=date(2026, 8, 11), + ) + + db = MagicMock() + db.execute.side_effect = IntegrityError( + "INSERT", + {}, + Exception( + 'insert or update on table "llm_usage_daily" violates foreign key ' + 'constraint "llm_usage_daily_organization_id_fkey"' + ), + ) + + flushed = usage_mod.flush_usage_to_catalog(db, org_id) + assert flushed == 0 + db.rollback.assert_called() + assert usage_mod._pending_hash_key(org_id) not in fake_redis.hashes + assert str(org_id) not in fake_redis.smembers("usage:pending:orgs") + assert not any(k.startswith("usage:flushing:") for k in fake_redis.hashes) + + +def test_concurrent_flush_does_not_double_count(fake_redis, org_ctx): + org_id, _workspace_id, ctx = org_ctx + with llm_usage_context(ctx): + usage_mod.record_llm_usage( + "gpt-test", + UsageSnapshot(prompt_tokens=9, completion_tokens=1), + usage_date=date(2026, 8, 11), + ) + + db_a = MagicMock() + db_a.execute.return_value = MagicMock(rowcount=0) # force INSERT path + db_b = MagicMock() + db_b.execute.return_value = MagicMock(rowcount=1) + + first = usage_mod.flush_usage_to_catalog(db_a, org_id) + second = usage_mod.flush_usage_to_catalog(db_b, org_id) + + assert first == 1 + assert second == 0 + assert db_a.commit.call_count == 1 + assert db_b.commit.call_count == 0 + # INSERT once for the claimed bucket; second flusher never upserts. + assert db_a.execute.call_count == 2 # UPDATE miss + INSERT + assert db_b.execute.call_count == 0 + + +def test_ensure_usage_context_and_path_inference(): + assert ( + infer_product_section_from_path("/api/v1/call-imports/abc") + == LLMUsageProductSection.CALL_IMPORTS + ) + assert ( + infer_product_section_from_path("/api/v1/chat/completion") + == LLMUsageProductSection.CHAT + ) + assert ( + infer_product_section_from_path("/api/v1/unknown") + == LLMUsageProductSection.OTHER + ) + + org_id = uuid4() + workspace_id = uuid4() + token = set_usage_context(None) + try: + assert get_usage_context() is None + created = ensure_usage_context( + org_id, + product_section=LLMUsageProductSection.PLAYGROUND, + ) + assert created is not None + assert get_usage_context().organization_id == org_id + assert get_usage_context().product_section == LLMUsageProductSection.PLAYGROUND + + # Enrich missing workspace / upgrade OTHER section. + upgraded = ensure_usage_context( + org_id, + workspace_id=workspace_id, + product_section=LLMUsageProductSection.CHAT, + ) + assert upgraded is not None + assert get_usage_context().workspace_id == workspace_id + # Existing non-OTHER section is preserved. + assert get_usage_context().product_section == LLMUsageProductSection.PLAYGROUND + finally: + reset_usage_context(token) + + +def test_ensure_uses_workspace_and_section_hints(): + from app.services.usage.context import reset_usage_hints, set_usage_hints + + org_id = uuid4() + workspace_id = uuid4() + ctx_token = set_usage_context(None) + hint_tokens = set_usage_hints( + workspace_id=workspace_id, + product_section=LLMUsageProductSection.METRICS, + ) + try: + created = ensure_usage_context(org_id) + assert created is not None + assert get_usage_context().workspace_id == workspace_id + assert get_usage_context().product_section == LLMUsageProductSection.METRICS + finally: + reset_usage_hints(hint_tokens) + reset_usage_context(ctx_token) From f0ebac9382e17d504a46b6f277d526676f391cc4 Mon Sep 17 00:00:00 2001 From: M Sami Date: Thu, 13 Aug 2026 03:17:12 +0530 Subject: [PATCH 16/32] feat: enhance LLM/STT usage tracking with additional context and metrics across services --- app/api/v1/routes/agents.py | 34 +- app/api/v1/routes/call_import_evaluations.py | 30 +- app/api/v1/routes/org_usage.py | 1242 ++++++++++++++-- app/api/v1/routes/playground.py | 291 +--- app/api/v1/routes/vobiz_telephony.py | 1 + app/api/v1/routes/voice_agent.py | 1 + .../063_usage_kind_stt_and_buffer.py | 233 +++ .../064_usage_kind_tts_characters.py | 74 + app/migrations/065_usage_context_jsonb.py | 216 +++ ...ll_llm_usage_workspace_from_call_import.py | 279 ++++ .../067_fix_llm_usage_bucket_unique_index.py | 264 ++++ app/models/database.py | 10 +- app/services/ai/llm_gateway.py | 56 +- app/services/ai/llm_service.py | 4 +- app/services/ai/stt_clients/google.py | 24 + app/services/ai/transcription_service.py | 45 + app/services/ai/tts_service.py | 37 + .../evaluators/call_data_transcript.py | 195 +++ app/services/judge_alignment/gepa_bridge.py | 51 +- app/services/optimization/gepa_service.py | 26 +- .../testing/test_agent_bridge_service.py | 2 + app/services/usage/__init__.py | 22 +- app/services/usage/bucket_context.py | 114 ++ app/services/usage/call_import_context.py | 200 +++ app/services/usage/context.py | 111 +- app/services/usage/dates.py | 38 + app/services/usage/llm_usage.py | 779 ++++++++-- app/services/usage/usage_labels.py | 424 ++++++ app/services/usage/voice_usage_processor.py | 56 +- app/services/voice_agent/bot_fast_api.py | 8 +- app/services/voice_agent/voice_bundle.py | 32 + .../webrtc_bridge/test_agent_processor.py | 88 +- app/workers/tasks/agent_flowchart_jobs.py | 40 +- app/workers/tasks/evaluate_call_import_row.py | 16 +- .../generate_evaluation_metric_clusters.py | 30 +- ...generate_evaluation_prompt_improvements.py | 42 +- .../generate_evaluation_tldr_insights.py | 14 +- .../generate_evaluation_user_insights.py | 34 +- app/workers/tasks/process_evaluator_result.py | 671 +++++---- app/workers/tasks/run_judge_alignment.py | 8 +- app/workers/tasks/run_prompt_optimization.py | 27 +- .../tasks/transcribe_call_import_row.py | 32 +- app/workers/tasks/tts_comparison.py | 200 +-- frontend/src/lib/api.ts | 42 +- .../agents/components/AgentTalkSidebar.tsx | 29 +- frontend/src/pages/usage/SearchableSelect.tsx | 125 ++ frontend/src/pages/usage/Usage.tsx | 1305 ++++++++++++++--- .../src/pages/usage/UsageDateRangePicker.tsx | 252 ++++ frontend/src/pages/usage/UsageDrillPath.tsx | 40 + frontend/src/pages/usage/UsageFiltersBar.tsx | 353 +++++ frontend/src/pages/usage/usageProductHints.ts | 47 + frontend/src/pages/usage/usageTheme.ts | 18 + frontend/src/pages/usage/usageTimezone.ts | 11 + .../test_call_data_transcript.py | 34 + .../test_usage/test_bucket_context.py | 40 + .../test_usage/test_call_import_context.py | 120 ++ .../test_usage/test_llm_usage.py | 210 ++- .../test_usage/test_usage_dates.py | 32 + .../test_usage/test_usage_labels.py | 287 ++++ .../test_process_evaluator_result_helpers.py | 41 +- 60 files changed, 7935 insertions(+), 1152 deletions(-) create mode 100644 app/migrations/063_usage_kind_stt_and_buffer.py create mode 100644 app/migrations/064_usage_kind_tts_characters.py create mode 100644 app/migrations/065_usage_context_jsonb.py create mode 100644 app/migrations/066_backfill_llm_usage_workspace_from_call_import.py create mode 100644 app/migrations/067_fix_llm_usage_bucket_unique_index.py create mode 100644 app/services/evaluators/call_data_transcript.py create mode 100644 app/services/usage/bucket_context.py create mode 100644 app/services/usage/call_import_context.py create mode 100644 app/services/usage/dates.py create mode 100644 app/services/usage/usage_labels.py create mode 100644 frontend/src/pages/usage/SearchableSelect.tsx create mode 100644 frontend/src/pages/usage/UsageDateRangePicker.tsx create mode 100644 frontend/src/pages/usage/UsageDrillPath.tsx create mode 100644 frontend/src/pages/usage/UsageFiltersBar.tsx create mode 100644 frontend/src/pages/usage/usageProductHints.ts create mode 100644 frontend/src/pages/usage/usageTheme.ts create mode 100644 frontend/src/pages/usage/usageTimezone.ts create mode 100644 tests/test_services/test_evaluators/test_call_data_transcript.py create mode 100644 tests/test_services/test_usage/test_bucket_context.py create mode 100644 tests/test_services/test_usage/test_call_import_context.py create mode 100644 tests/test_services/test_usage/test_usage_dates.py create mode 100644 tests/test_services/test_usage/test_usage_labels.py diff --git a/app/api/v1/routes/agents.py b/app/api/v1/routes/agents.py index 9d1d77b0..4c4b295e 100644 --- a/app/api/v1/routes/agents.py +++ b/app/api/v1/routes/agents.py @@ -149,7 +149,10 @@ async def generate_agent_description( db: Session = Depends(get_db), ): """Generate an agent description using AI from a brief description.""" + from contextlib import nullcontext + from app.services.ai.llm_service import llm_service + from app.services.usage.context import llm_usage_context, usage_context_for_agent from app.services.testing.test_agent_simulation_prompt import ( format_scenarios_for_generation_context, format_scenarios_reference_appendix, @@ -202,15 +205,28 @@ async def generate_agent_description( ] try: - result = llm_service.generate_response( - messages=messages, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - temperature=0.7, - max_tokens=4000, - ) + usage_ctx = nullcontext() + if data.agent_id: + agent_for_usage = db.query(Agent).filter( + Agent.id == data.agent_id, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ).first() + if agent_for_usage: + usage_ctx = llm_usage_context( + usage_context_for_agent(agent_for_usage, workspace_id=workspace_id) + ) + + with usage_ctx: + result = llm_service.generate_response( + messages=messages, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + temperature=0.7, + max_tokens=4000, + ) content = result["text"] if data.append_scenarios_to_output and linked_scenarios: appendix = format_scenarios_reference_appendix(linked_scenarios) diff --git a/app/api/v1/routes/call_import_evaluations.py b/app/api/v1/routes/call_import_evaluations.py index 030c98ef..880393c7 100644 --- a/app/api/v1/routes/call_import_evaluations.py +++ b/app/api/v1/routes/call_import_evaluations.py @@ -3702,20 +3702,18 @@ async def generate_call_import_evaluation_pdf_report( ) return _pdf_report_response_from_row(cached_pdf_report, cache_hit=True) - from app.services.usage.context import ( - LLMUsageContext, - LLMUsageProductSection, - llm_usage_context, + from app.services.usage.call_import_context import ( + call_import_evaluation_usage_context, ) + from app.services.usage.context import llm_usage_context with llm_usage_context( - LLMUsageContext( + call_import_evaluation_usage_context( organization_id=organization_id, workspace_id=getattr(call_import, "workspace_id", None) or evaluation.workspace_id, - product_section=LLMUsageProductSection.CALL_IMPORT_EVALUATIONS, - resource_id=evaluation.id, - resource_type="call_import_evaluation", + evaluation_id=evaluation.id, + call_import_id=evaluation.call_import_id, ) ): narrative = _generate_report_narrative( @@ -5353,11 +5351,10 @@ def _generate_and_persist_tldr_summary( from app.services.ai.llm_resolver import get_llm_provider_and_model from app.services.ai.llm_service import llm_service - from app.services.usage.context import ( - LLMUsageContext, - LLMUsageProductSection, - llm_usage_context, + from app.services.usage.call_import_context import ( + call_import_evaluation_usage_context, ) + from app.services.usage.context import llm_usage_context provider_enum, model_str = get_llm_provider_and_model( organization_id, db, provider, model @@ -5365,12 +5362,11 @@ def _generate_and_persist_tldr_summary( try: with llm_usage_context( - LLMUsageContext( + call_import_evaluation_usage_context( organization_id=organization_id, workspace_id=evaluation.workspace_id, - product_section=LLMUsageProductSection.CALL_IMPORT_EVALUATIONS, - resource_id=evaluation.id, - resource_type="call_import_evaluation", + evaluation_id=evaluation.id, + call_import_id=evaluation.call_import_id, ) ): llm_result = llm_service.generate_response( @@ -5388,8 +5384,8 @@ def _generate_and_persist_tldr_summary( status_code=502, detail=f"LLM call failed: {e}" ) from e - summary = _parse_insights_response(llm_result.get("text", "")) total = int(evaluation.total_rows or 0) + summary = _parse_insights_response(llm_result.get("text", "")) ui_completed = min(int(evaluation.completed_rows or 0), total) if total else int( evaluation.completed_rows or 0 ) diff --git a/app/api/v1/routes/org_usage.py b/app/api/v1/routes/org_usage.py index 05f984a0..4c702138 100644 --- a/app/api/v1/routes/org_usage.py +++ b/app/api/v1/routes/org_usage.py @@ -2,23 +2,49 @@ from __future__ import annotations -from datetime import date, datetime, timedelta, timezone +from datetime import date, datetime, timezone +from app.services.usage.dates import usage_date_filter_bounds, usage_local_today from typing import List, Literal, Optional from uuid import UUID from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel, Field -from sqlalchemy import func +from sqlalchemy import and_, case, cast, func, or_, select, String +from sqlalchemy.dialects.postgresql import UUID as PG_UUID from sqlalchemy.orm import Session from app.database import get_db from app.dependencies import get_organization_id -from app.models.database import CallImportEvaluation, LLMUsageDaily, Workspace -from app.services.usage.llm_usage import flush_usage_to_catalog, merge_usage_totals +from app.models.database import ( + CallImport, + CallImportEvaluation, + CallImportEvaluationRow, + CallImportRow, + CallImportTag, + CallImportTagAssignment, + Agent, + LLMUsageDaily, + Workspace, +) +from app.services.usage.llm_usage import flush_usage_to_catalog +from app.services.usage.usage_labels import ( + labels_for_call_import_ids, + labels_for_resource_buckets, + usage_kind_label, + UsageNameResolver, + parse_uuid, +) router = APIRouter(prefix="/organizations/usage", tags=["Usage"]) -GroupBy = Literal["workspace", "product_section", "model", "resource"] +GroupBy = Literal[ + "workspace", + "product_section", + "model", + "resource", + "usage_kind", + "call_import", +] SECTION_LABELS = { "call_import_evaluations": "Call Import Evaluations", @@ -39,6 +65,27 @@ "other": "Other", } +_LABEL_ROW_LIMIT = 5000 + + +def _usage_row_weight(): + return ( + LLMUsageDaily.prompt_tokens + + LLMUsageDaily.completion_tokens + + LLMUsageDaily.cache_read_tokens + + LLMUsageDaily.cache_creation_tokens + + LLMUsageDaily.reasoning_tokens + + LLMUsageDaily.audio_seconds + + LLMUsageDaily.tts_characters + ) + + +def _label_row_order(): + return ( + _usage_row_weight().desc(), + LLMUsageDaily.call_count.desc(), + ) + class UsageTotals(BaseModel): prompt_tokens: int = 0 @@ -47,6 +94,8 @@ class UsageTotals(BaseModel): cache_read_tokens: int = 0 cache_creation_tokens: int = 0 reasoning_tokens: int = 0 + audio_seconds: int = 0 + tts_characters: int = 0 call_count: int = 0 @@ -66,12 +115,17 @@ class UsageBreakdownRow(BaseModel): resource_id: Optional[UUID] = None resource_type: Optional[str] = None resource_label: Optional[str] = None + call_import_id: Optional[UUID] = None + call_import_label: Optional[str] = None + usage_kind: Optional[str] = None prompt_tokens: int = 0 completion_tokens: int = 0 total_tokens: int = 0 cache_read_tokens: int = 0 cache_creation_tokens: int = 0 reasoning_tokens: int = 0 + audio_seconds: int = 0 + tts_characters: int = 0 call_count: int = 0 @@ -81,20 +135,249 @@ class UsageBreakdownResponse(BaseModel): group_by: GroupBy rows: List[UsageBreakdownRow] total_count: int + truncated_at_limit: bool = False last_updated_at: Optional[datetime] = None class UsageFiltersResponse(BaseModel): workspaces: List[dict] = Field(default_factory=list) product_sections: List[dict] = Field(default_factory=list) + call_imports: List[dict] = Field(default_factory=list) + evaluations: List[dict] = Field(default_factory=list) models: List[str] = Field(default_factory=list) resources: List[dict] = Field(default_factory=list) + usage_kinds: List[dict] = Field(default_factory=list) + datasets: List[str] = Field(default_factory=list) + tags: List[dict] = Field(default_factory=list) -def _default_range() -> tuple[date, date]: - end = datetime.now(timezone.utc).date() - start = end - timedelta(days=29) - return start, end +def _parse_usage_range( + start: Optional[date], + end: Optional[date], + tz: Optional[str], +) -> tuple[date, date, date, date]: + """Return display_start, display_end, filter_start, filter_end.""" + today = usage_local_today(tz) + display_start = start or today + display_end = end or today + filter_start, filter_end = usage_date_filter_bounds( + display_start, display_end, tz + ) + return display_start, display_end, filter_start, filter_end + + +def _evaluation_id_expr(): + return func.coalesce( + LLMUsageDaily.context["evaluation_id"].astext, + case( + ( + LLMUsageDaily.context["resource_type"].astext + == "call_import_evaluation", + LLMUsageDaily.context["resource_id"].astext, + ), + else_=None, + ), + ) + + +def _resource_id_expr(): + """Resource rollup key: explicit resource_id or agent_id fallback.""" + return func.coalesce( + LLMUsageDaily.context["resource_id"].astext, + LLMUsageDaily.context["agent_id"].astext, + ) + + +def _resource_type_expr(): + rid_expr = _resource_id_expr() + return func.coalesce( + LLMUsageDaily.context["resource_type"].astext, + case( + (LLMUsageDaily.context["agent_id"].astext.isnot(None), "agent"), + ( + and_( + LLMUsageDaily.product_section == "agents", + rid_expr.isnot(None), + ), + "agent", + ), + else_=None, + ), + ) + + +def _call_import_group_expr(): + """Resolve call import id from context, resource row, evaluation, or import row.""" + eval_call_import = ( + select(CallImportEvaluation.call_import_id) + .where( + CallImportEvaluation.id == cast(_evaluation_id_expr(), PG_UUID) + ) + .correlate(LLMUsageDaily) + .scalar_subquery() + ) + row_call_import = ( + select(CallImportRow.call_import_id) + .where( + CallImportRow.id + == cast(LLMUsageDaily.context["call_import_row_id"].astext, PG_UUID) + ) + .correlate(LLMUsageDaily) + .scalar_subquery() + ) + eval_row_call_import = ( + select(CallImportEvaluation.call_import_id) + .select_from(CallImportEvaluationRow) + .join( + CallImportEvaluation, + CallImportEvaluation.id == CallImportEvaluationRow.evaluation_id, + ) + .where( + CallImportEvaluationRow.id + == cast(LLMUsageDaily.context["evaluation_row_id"].astext, PG_UUID) + ) + .correlate(LLMUsageDaily) + .scalar_subquery() + ) + return cast( + func.coalesce( + LLMUsageDaily.context["call_import_id"].astext, + case( + ( + LLMUsageDaily.context["resource_type"].astext == "call_import", + LLMUsageDaily.context["resource_id"].astext, + ), + else_=None, + ), + cast(eval_call_import, String), + cast(row_call_import, String), + cast(eval_row_call_import, String), + ), + String, + ) + + +def _resource_scope_filter(resource_id: UUID): + """Match usage attributed to a product resource (agent, simulation, etc.).""" + rid = str(resource_id) + return or_( + LLMUsageDaily.context["resource_id"].astext == rid, + LLMUsageDaily.context["agent_id"].astext == rid, + ) + + +def _evaluation_scope_filter( + evaluation_id: UUID, + organization_id: UUID, + db: Session, +): + eid = str(evaluation_id) + row_ids = [ + str(row[0]) + for row in db.query(CallImportEvaluationRow.id) + .filter( + CallImportEvaluationRow.evaluation_id == evaluation_id, + ) + .all() + ] + clauses = [ + LLMUsageDaily.context["evaluation_id"].astext == eid, + and_( + LLMUsageDaily.context["resource_id"].astext == eid, + LLMUsageDaily.context["resource_type"].astext == "call_import_evaluation", + ), + ] + if row_ids: + clauses.append(LLMUsageDaily.context["evaluation_row_id"].astext.in_(row_ids)) + return or_(*clauses) + + +def _call_import_scope_filter( + call_import_id: UUID, + organization_id: UUID, + db: Session, +): + """Match usage tied to a call import (direct context, resource row, or eval runs).""" + cid = str(call_import_id) + eval_ids = [ + str(row[0]) + for row in db.query(CallImportEvaluation.id) + .filter( + CallImportEvaluation.organization_id == organization_id, + CallImportEvaluation.call_import_id == call_import_id, + ) + .all() + ] + clauses = [ + LLMUsageDaily.context["call_import_id"].astext == cid, + and_( + LLMUsageDaily.context["resource_id"].astext == cid, + LLMUsageDaily.context["resource_type"].astext == "call_import", + ), + ] + if eval_ids: + clauses.append(LLMUsageDaily.context["evaluation_id"].astext.in_(eval_ids)) + clauses.append( + and_( + LLMUsageDaily.context["resource_id"].astext.in_(eval_ids), + LLMUsageDaily.context["resource_type"].astext == "call_import_evaluation", + ) + ) + return or_(*clauses) + + +def _call_import_ids_for_filters( + db: Session, + *, + organization_id: UUID, + workspace_id: Optional[UUID] = None, + dataset: Optional[str] = None, + tag_id: Optional[UUID] = None, +) -> List[UUID]: + query = db.query(CallImport.id).filter( + CallImport.organization_id == organization_id, + ) + if workspace_id is not None: + query = query.filter(CallImport.workspace_id == workspace_id) + if dataset: + query = query.filter(CallImport.dataset == dataset) + if tag_id is not None: + query = query.filter( + CallImport.id.in_( + db.query(CallImportTagAssignment.call_import_id).filter( + CallImportTagAssignment.tag_id == tag_id, + ) + ) + ) + return [row[0] for row in query.all()] + + +def _call_import_ids_scope_filter( + allowed_import_ids: List[UUID], +): + if not allowed_import_ids: + return LLMUsageDaily.id.is_(None) + allowed = [str(uid) for uid in allowed_import_ids] + return cast(_call_import_group_expr(), String).in_(allowed) + + +def _workspace_scope_filter( + workspace_id: UUID, + organization_id: UUID, +): + """Match workspace-scoped rows plus legacy call-import usage with null workspace_id.""" + ws_call_import_ids = ( + select(cast(CallImport.id, String)) + .where( + CallImport.organization_id == organization_id, + CallImport.workspace_id == workspace_id, + ) + ) + legacy_call_import = and_( + LLMUsageDaily.workspace_id.is_(None), + _call_import_group_expr().in_(ws_call_import_ids), + ) + return or_(LLMUsageDaily.workspace_id == workspace_id, legacy_call_import) def _apply_filters( @@ -107,6 +390,13 @@ def _apply_filters( product_section: Optional[str], model: Optional[str], resource_id: Optional[UUID], + usage_kind: Optional[str] = None, + call_import_id: Optional[UUID] = None, + evaluation_id: Optional[UUID] = None, + evaluation_row_id: Optional[UUID] = None, + dataset: Optional[str] = None, + tag_id: Optional[UUID] = None, + db: Optional[Session] = None, ): query = query.filter( LLMUsageDaily.organization_id == organization_id, @@ -114,16 +404,449 @@ def _apply_filters( LLMUsageDaily.usage_date <= end, ) if workspace_id is not None: - query = query.filter(LLMUsageDaily.workspace_id == workspace_id) + query = query.filter( + _workspace_scope_filter(workspace_id, organization_id) + ) if product_section: query = query.filter(LLMUsageDaily.product_section == product_section) if model: query = query.filter(LLMUsageDaily.model == model) if resource_id is not None: - query = query.filter(LLMUsageDaily.resource_id == resource_id) + rid_filter = _resource_scope_filter(resource_id) + if db is not None: + query = query.filter( + or_( + _evaluation_scope_filter(resource_id, organization_id, db), + rid_filter, + ) + ) + else: + query = query.filter( + or_( + LLMUsageDaily.context["resource_id"].astext == str(resource_id), + LLMUsageDaily.context["agent_id"].astext == str(resource_id), + LLMUsageDaily.context["evaluation_id"].astext == str(resource_id), + LLMUsageDaily.context["evaluation_row_id"].astext == str(resource_id), + ) + ) + if call_import_id is not None: + if db is not None: + query = query.filter( + _call_import_scope_filter(call_import_id, organization_id, db) + ) + else: + query = query.filter( + LLMUsageDaily.context["call_import_id"].astext == str(call_import_id) + ) + if evaluation_id is not None: + if db is not None: + query = query.filter( + _evaluation_scope_filter(evaluation_id, organization_id, db) + ) + else: + query = query.filter( + LLMUsageDaily.context["evaluation_id"].astext == str(evaluation_id) + ) + if evaluation_row_id is not None: + query = query.filter( + LLMUsageDaily.context["evaluation_row_id"].astext + == str(evaluation_row_id) + ) + if usage_kind: + query = query.filter(LLMUsageDaily.usage_kind == usage_kind) + if dataset or tag_id is not None: + if db is None: + raise ValueError("db required for dataset/tag filters") + allowed = _call_import_ids_for_filters( + db, + organization_id=organization_id, + workspace_id=workspace_id, + dataset=dataset, + tag_id=tag_id, + ) + query = query.filter(_call_import_ids_scope_filter(allowed)) return query +def _filtered_query( + db: Session, + *, + organization_id: UUID, + start: date, + end: date, + workspace_id: Optional[UUID] = None, + product_section: Optional[str] = None, + model: Optional[str] = None, + resource_id: Optional[UUID] = None, + usage_kind: Optional[str] = None, + call_import_id: Optional[UUID] = None, + evaluation_id: Optional[UUID] = None, + evaluation_row_id: Optional[UUID] = None, + dataset: Optional[str] = None, + tag_id: Optional[UUID] = None, +): + return _apply_filters( + db.query(LLMUsageDaily), + organization_id=organization_id, + start=start, + end=end, + workspace_id=workspace_id, + product_section=product_section, + model=model, + resource_id=resource_id, + usage_kind=usage_kind, + call_import_id=call_import_id, + evaluation_id=evaluation_id, + evaluation_row_id=evaluation_row_id, + dataset=dataset, + tag_id=tag_id, + db=db, + ) + + +def _infer_agent_types_for_label_buckets( + db: Session, + organization_id: UUID, + grouped: dict[str, tuple[Optional[str], list]], +) -> None: + """When context JSON omits resource_type, infer agent rows from Agent.id.""" + candidate_ids: list[UUID] = [] + for rid, (rtype, _) in grouped.items(): + if rtype: + continue + uid = parse_uuid(rid) + if uid: + candidate_ids.append(uid) + if not candidate_ids: + return + known_agent_ids = { + row.id + for row in db.query(Agent.id).filter( + Agent.organization_id == organization_id, + Agent.id.in_(candidate_ids), + ).all() + } + for rid in grouped: + uid = parse_uuid(rid) + if uid and uid in known_agent_ids: + rtype, contexts = grouped[rid] + if not rtype: + grouped[rid] = ("agent", contexts) + + +def _resource_label_map( + db: Session, + organization_id: UUID, + query, +) -> dict[str, str]: + """Build resource_id -> hierarchical label from usage rows in query.""" + rows = ( + query.with_entities( + _resource_id_expr(), + _resource_type_expr(), + LLMUsageDaily.context, + ) + .filter( + or_( + LLMUsageDaily.context["resource_id"].astext.isnot(None), + LLMUsageDaily.context["agent_id"].astext.isnot(None), + ) + ) + .order_by(*_label_row_order()) + .limit(_LABEL_ROW_LIMIT) + .all() + ) + grouped: dict[str, tuple[Optional[str], list]] = {} + for raw_id, rtype, ctx in rows: + if not raw_id: + continue + key = str(raw_id) + if key not in grouped: + grouped[key] = (rtype, []) + grouped[key][1].append(ctx) + + _infer_agent_types_for_label_buckets(db, organization_id, grouped) + + buckets = [(rid, rtype, contexts) for rid, (rtype, contexts) in grouped.items()] + resolver = UsageNameResolver(db, organization_id) + contexts_for_preload = [] + for rid, rtype, contexts in buckets: + for ctx in contexts: + merged = dict(ctx or {}) + if rid: + merged.setdefault("resource_id", rid) + if rtype: + merged.setdefault("resource_type", rtype) + contexts_for_preload.append(merged) + resolver.preload(contexts_for_preload) + return labels_for_resource_buckets(buckets, resolver) + + +def _resource_filter_meta_map( + db: Session, + organization_id: UUID, + query, +) -> dict[str, dict[str, Optional[str]]]: + """Map resource id -> {type, product_section} for filter dropdowns.""" + rows = ( + query.with_entities( + _resource_id_expr(), + _resource_type_expr(), + LLMUsageDaily.product_section, + ) + .filter( + or_( + LLMUsageDaily.context["resource_id"].astext.isnot(None), + LLMUsageDaily.context["agent_id"].astext.isnot(None), + ) + ) + .distinct() + .limit(_LABEL_ROW_LIMIT) + .all() + ) + meta: dict[str, dict[str, Optional[str]]] = {} + for raw_id, rtype, section in rows: + if not raw_id: + continue + key = str(raw_id) + entry = meta.setdefault(key, {"type": None, "product_section": None}) + if rtype: + entry["type"] = str(rtype) + if section: + entry["product_section"] = str(section) + if not entry["type"] and section == "agents": + entry["type"] = "agent" + return meta + + +def _breakdown_resource_label( + raw_res_id: Optional[str], + res_type: Optional[str], + section: Optional[str], + resource_labels: dict[str, str], + db: Session, + organization_id: UUID, +) -> str: + if not raw_res_id: + return "Unscoped" + key = str(raw_res_id) + label = resource_labels.get(key) + if label and label != "Unscoped": + return label + effective_type = res_type or ("agent" if section == "agents" else None) + if effective_type == "agent": + resolver = UsageNameResolver(db, organization_id) + resolver.preload([{"resource_id": key, "resource_type": "agent"}]) + return resolver.agent_name(key) + return label or "Unscoped" + + +def _collect_call_import_ids_from_usage_rows( + db: Session, + organization_id: UUID, + rows: list, +) -> set[UUID]: + """Extract call import ids referenced in usage row context tuples.""" + import_ids: set[UUID] = set() + eval_ids: set[UUID] = set() + row_ids: set[UUID] = set() + eval_row_ids: set[UUID] = set() + + for row in rows: + cid_raw = row[0] if len(row) > 0 else None + res_raw = row[1] if len(row) > 1 else None + rtype = row[2] if len(row) > 2 else None + eval_raw = row[3] if len(row) > 3 else None + row_id_raw = row[4] if len(row) > 4 else None + eval_row_raw = row[5] if len(row) > 5 else None + + if cid_raw: + uid = parse_uuid(cid_raw) + if uid: + import_ids.add(uid) + if res_raw and rtype == "call_import": + uid = parse_uuid(res_raw) + if uid: + import_ids.add(uid) + if eval_raw: + uid = parse_uuid(eval_raw) + if uid: + eval_ids.add(uid) + if res_raw and rtype == "call_import_evaluation": + uid = parse_uuid(res_raw) + if uid: + eval_ids.add(uid) + if row_id_raw: + uid = parse_uuid(row_id_raw) + if uid: + row_ids.add(uid) + if eval_row_raw: + uid = parse_uuid(eval_row_raw) + if uid: + eval_row_ids.add(uid) + + if eval_ids: + for ev in ( + db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.organization_id == organization_id, + CallImportEvaluation.id.in_(eval_ids), + ) + .all() + ): + import_ids.add(ev.call_import_id) + + if row_ids: + for cid in ( + db.query(CallImportRow.call_import_id) + .filter(CallImportRow.id.in_(row_ids)) + .distinct() + .all() + ): + import_ids.add(cid[0]) + + if eval_row_ids: + for cid in ( + db.query(CallImportEvaluation.call_import_id) + .join( + CallImportEvaluationRow, + CallImportEvaluationRow.evaluation_id == CallImportEvaluation.id, + ) + .filter(CallImportEvaluationRow.id.in_(eval_row_ids)) + .distinct() + .all() + ): + import_ids.add(cid[0]) + + return import_ids + + +def _call_import_label_map( + db: Session, + organization_id: UUID, + query, +) -> dict[str, str]: + rows = ( + query.with_entities( + LLMUsageDaily.context["call_import_id"].astext, + LLMUsageDaily.context["resource_id"].astext, + LLMUsageDaily.context["resource_type"].astext, + LLMUsageDaily.context["evaluation_id"].astext, + LLMUsageDaily.context["call_import_row_id"].astext, + LLMUsageDaily.context["evaluation_row_id"].astext, + ) + .order_by(*_label_row_order()) + .limit(_LABEL_ROW_LIMIT) + .all() + ) + import_ids = _collect_call_import_ids_from_usage_rows( + db, organization_id, rows + ) + + if not import_ids: + return {} + resolver = UsageNameResolver(db, organization_id) + resolver.preload( + [{"call_import_id": str(uid)} for uid in import_ids] + ) + return labels_for_call_import_ids(list(import_ids), resolver) + + +def _call_import_filter_labels( + db: Session, + organization_id: UUID, + query, + workspace_id: Optional[UUID] = None, + dataset: Optional[str] = None, + tag_id: Optional[UUID] = None, +) -> dict[str, str]: + """Call imports for filters: usage in range plus workspace imports when scoped.""" + import_ids = _collect_call_import_ids_from_usage_rows( + db, + organization_id, + query.with_entities( + LLMUsageDaily.context["call_import_id"].astext, + LLMUsageDaily.context["resource_id"].astext, + LLMUsageDaily.context["resource_type"].astext, + LLMUsageDaily.context["evaluation_id"].astext, + LLMUsageDaily.context["call_import_row_id"].astext, + LLMUsageDaily.context["evaluation_row_id"].astext, + ).order_by(*_label_row_order()).limit(_LABEL_ROW_LIMIT).all(), + ) + + if workspace_id is not None: + scoped_ids = _call_import_ids_for_filters( + db, + organization_id=organization_id, + workspace_id=workspace_id, + dataset=dataset, + tag_id=tag_id, + ) + import_ids.update(scoped_ids) + + if not import_ids: + return {} + resolver = UsageNameResolver(db, organization_id) + resolver.preload([{"call_import_id": str(uid)} for uid in import_ids]) + return labels_for_call_import_ids(list(import_ids), resolver) + + +def _evaluation_label_map( + db: Session, + organization_id: UUID, + query, +) -> dict[str, str]: + """Evaluation id -> short label (name + id suffix) for filter dropdowns.""" + rows = ( + query.with_entities( + LLMUsageDaily.context["evaluation_id"].astext, + LLMUsageDaily.context["resource_id"].astext, + LLMUsageDaily.context["resource_type"].astext, + LLMUsageDaily.context, + ) + .filter( + or_( + LLMUsageDaily.context["evaluation_id"].astext.isnot(None), + LLMUsageDaily.context["resource_type"].astext == "call_import_evaluation", + ) + ) + .order_by(*_label_row_order()) + .limit(_LABEL_ROW_LIMIT) + .all() + ) + grouped: dict[str, tuple[Optional[str], list]] = {} + for eval_raw, res_raw, rtype, ctx in rows: + key_raw = eval_raw + if not key_raw and rtype == "call_import_evaluation" and res_raw: + key_raw = res_raw + if not key_raw: + continue + key = str(key_raw) + if key not in grouped: + grouped[key] = (rtype, []) + grouped[key][1].append(ctx) + + buckets = [(rid, rtype, contexts) for rid, (rtype, contexts) in grouped.items()] + resolver = UsageNameResolver(db, organization_id) + contexts_for_preload = [] + for _, rtype, contexts in buckets: + for ctx in contexts: + merged = dict(ctx or {}) + if rtype and "resource_type" not in merged: + merged["resource_type"] = rtype + contexts_for_preload.append(merged) + resolver.preload(contexts_for_preload) + + labels: dict[str, str] = {} + for raw_id, rtype, contexts in buckets: + ctx = max([dict(c or {}) for c in contexts], key=len, default={}) + if rtype and "resource_type" not in ctx: + ctx["resource_type"] = rtype + eval_key = ctx.get("evaluation_id") or raw_id + labels[str(raw_id)] = resolver.evaluation_name(str(eval_key)) + return labels + + def _last_updated(db: Session, organization_id: UUID) -> Optional[datetime]: return ( db.query(func.max(LLMUsageDaily.updated_at)) @@ -132,39 +855,120 @@ def _last_updated(db: Session, organization_id: UUID) -> Optional[datetime]: ) +def _summary_aggregate_query( + db: Session, + *, + organization_id: UUID, + start: date, + end: date, + workspace_id: Optional[UUID], + product_section: Optional[str], + model: Optional[str], + resource_id: Optional[UUID], + usage_kind: Optional[str], + call_import_id: Optional[UUID], + evaluation_id: Optional[UUID], + evaluation_row_id: Optional[UUID], + dataset: Optional[str] = None, + tag_id: Optional[UUID] = None, +): + return _apply_filters( + db.query( + func.coalesce(func.sum(LLMUsageDaily.prompt_tokens), 0).label("prompt_tokens"), + func.coalesce(func.sum(LLMUsageDaily.completion_tokens), 0).label( + "completion_tokens" + ), + func.coalesce(func.sum(LLMUsageDaily.cache_read_tokens), 0).label( + "cache_read_tokens" + ), + func.coalesce(func.sum(LLMUsageDaily.cache_creation_tokens), 0).label( + "cache_creation_tokens" + ), + func.coalesce(func.sum(LLMUsageDaily.reasoning_tokens), 0).label( + "reasoning_tokens" + ), + func.coalesce(func.sum(LLMUsageDaily.audio_seconds), 0).label("audio_seconds"), + func.coalesce(func.sum(LLMUsageDaily.tts_characters), 0).label("tts_characters"), + func.coalesce(func.sum(LLMUsageDaily.call_count), 0).label("call_count"), + ), + organization_id=organization_id, + start=start, + end=end, + workspace_id=workspace_id, + product_section=product_section, + model=model, + resource_id=resource_id, + usage_kind=usage_kind, + call_import_id=call_import_id, + evaluation_id=evaluation_id, + evaluation_row_id=evaluation_row_id, + dataset=dataset, + tag_id=tag_id, + db=db, + ) + + @router.get("/summary", response_model=UsageSummaryResponse) def get_usage_summary( start: Optional[date] = Query(None), end: Optional[date] = Query(None), + tz: Optional[str] = Query( + None, + description="IANA timezone for interpreting start/end calendar dates", + ), workspace_id: Optional[UUID] = Query(None), product_section: Optional[str] = Query(None), model: Optional[str] = Query(None), resource_id: Optional[UUID] = Query(None), + usage_kind: Optional[str] = Query(None), + call_import_id: Optional[UUID] = Query(None), + evaluation_id: Optional[UUID] = Query(None), + evaluation_row_id: Optional[UUID] = Query(None), + dataset: Optional[str] = Query(None), + tag_id: Optional[UUID] = Query(None), organization_id: UUID = Depends(get_organization_id), db: Session = Depends(get_db), ): - if start is None or end is None: - start, end = _default_range() - if end < start: + display_start, display_end, filter_start, filter_end = _parse_usage_range( + start, end, tz + ) + if display_end < display_start: raise HTTPException(status_code=400, detail="end must be >= start") flush_usage_to_catalog(db, organization_id) - query = _apply_filters( - db.query(LLMUsageDaily), + row = _summary_aggregate_query( + db, organization_id=organization_id, - start=start, - end=end, + start=filter_start, + end=filter_end, workspace_id=workspace_id, product_section=product_section, model=model, resource_id=resource_id, - ) - rows = query.all() - totals = merge_usage_totals(rows) + usage_kind=usage_kind, + call_import_id=call_import_id, + evaluation_id=evaluation_id, + evaluation_row_id=evaluation_row_id, + dataset=dataset, + tag_id=tag_id, + ).one() + prompt = int(row.prompt_tokens) + completion = int(row.completion_tokens) + totals = { + "prompt_tokens": prompt, + "completion_tokens": completion, + "total_tokens": prompt + completion, + "cache_read_tokens": int(row.cache_read_tokens), + "cache_creation_tokens": int(row.cache_creation_tokens), + "reasoning_tokens": int(row.reasoning_tokens), + "audio_seconds": int(row.audio_seconds), + "tts_characters": int(row.tts_characters), + "call_count": int(row.call_count), + } return UsageSummaryResponse( - start=start, - end=end, + start=display_start, + end=display_end, totals=UsageTotals(**totals), last_updated_at=_last_updated(db, organization_id), ) @@ -174,19 +978,30 @@ def get_usage_summary( def get_usage_breakdown( start: Optional[date] = Query(None), end: Optional[date] = Query(None), + tz: Optional[str] = Query( + None, + description="IANA timezone for interpreting start/end calendar dates", + ), group_by: GroupBy = Query("workspace"), workspace_id: Optional[UUID] = Query(None), product_section: Optional[str] = Query(None), model: Optional[str] = Query(None), resource_id: Optional[UUID] = Query(None), + usage_kind: Optional[str] = Query(None), + call_import_id: Optional[UUID] = Query(None), + evaluation_id: Optional[UUID] = Query(None), + evaluation_row_id: Optional[UUID] = Query(None), + dataset: Optional[str] = Query(None), + tag_id: Optional[UUID] = Query(None), limit: int = Query(100, ge=1, le=500), offset: int = Query(0, ge=0), organization_id: UUID = Depends(get_organization_id), db: Session = Depends(get_db), ): - if start is None or end is None: - start, end = _default_range() - if end < start: + display_start, display_end, filter_start, filter_end = _parse_usage_range( + start, end, tz + ) + if display_end < display_start: raise HTTPException(status_code=400, detail="end must be >= start") flush_usage_to_catalog(db, organization_id) @@ -195,7 +1010,9 @@ def get_usage_breakdown( "workspace": LLMUsageDaily.workspace_id, "product_section": LLMUsageDaily.product_section, "model": LLMUsageDaily.model, - "resource": LLMUsageDaily.resource_id, + "resource": cast(_resource_id_expr(), String), + "usage_kind": LLMUsageDaily.usage_kind, + "call_import": _call_import_group_expr(), }[group_by] aggregates = [ @@ -212,27 +1029,37 @@ def get_usage_breakdown( func.coalesce(func.sum(LLMUsageDaily.reasoning_tokens), 0).label( "reasoning_tokens" ), + func.coalesce(func.sum(LLMUsageDaily.audio_seconds), 0).label("audio_seconds"), + func.coalesce(func.sum(LLMUsageDaily.tts_characters), 0).label("tts_characters"), func.coalesce(func.sum(LLMUsageDaily.call_count), 0).label("call_count"), ] select_cols = [dim] group_cols = [dim] if group_by == "resource": - select_cols.append(LLMUsageDaily.resource_type) - group_cols.append(LLMUsageDaily.resource_type) + select_cols.append(cast(_resource_type_expr(), String)) + group_cols.append(cast(_resource_type_expr(), String)) + select_cols.append(LLMUsageDaily.product_section) + group_cols.append(LLMUsageDaily.product_section) query = _apply_filters( db.query(*select_cols, *aggregates), organization_id=organization_id, - start=start, - end=end, + start=filter_start, + end=filter_end, workspace_id=workspace_id, product_section=product_section, model=model, resource_id=resource_id, + usage_kind=usage_kind, + call_import_id=call_import_id, + evaluation_id=evaluation_id, + evaluation_row_id=evaluation_row_id, + dataset=dataset, + tag_id=tag_id, + db=db, ).group_by(*group_cols) - total_count = query.count() results = ( query.order_by(func.sum(LLMUsageDaily.call_count).desc()) .offset(offset) @@ -240,27 +1067,53 @@ def get_usage_breakdown( .all() ) - workspace_names = { - w.id: w.name - for w in db.query(Workspace) - .filter(Workspace.organization_id == organization_id) - .all() - } - resource_labels: dict = {} - if group_by == "resource": - resource_ids = [r[0] for r in results if r[0] is not None] - if resource_ids: - evals = ( - db.query(CallImportEvaluation) + workspace_names: dict = {} + if group_by == "workspace": + ws_ids = {r[0] for r in results if r[0] is not None} + if ws_ids: + workspace_names = { + w.id: w.name + for w in db.query(Workspace) .filter( - CallImportEvaluation.organization_id == organization_id, - CallImportEvaluation.id.in_(resource_ids), + Workspace.organization_id == organization_id, + Workspace.id.in_(ws_ids), ) .all() - ) - for evaluation in evals: - label = (evaluation.name or "").strip() or str(evaluation.id)[:8] - resource_labels[evaluation.id] = label + } + resource_labels: dict[str, str] = {} + call_import_labels: dict[str, str] = {} + if group_by == "resource": + label_query = _filtered_query( + db, + organization_id=organization_id, + start=filter_start, + end=filter_end, + workspace_id=workspace_id, + product_section=product_section, + model=model, + resource_id=resource_id, + usage_kind=usage_kind, + call_import_id=call_import_id, + evaluation_id=evaluation_id, + evaluation_row_id=evaluation_row_id, + ) + resource_labels = _resource_label_map(db, organization_id, label_query) + elif group_by == "call_import": + label_query = _filtered_query( + db, + organization_id=organization_id, + start=filter_start, + end=filter_end, + workspace_id=workspace_id, + product_section=product_section, + model=model, + resource_id=resource_id, + usage_kind=usage_kind, + call_import_id=call_import_id, + evaluation_id=evaluation_id, + evaluation_row_id=evaluation_row_id, + ) + call_import_labels = _call_import_label_map(db, organization_id, label_query) rows: List[UsageBreakdownRow] = [] for result in results: @@ -277,7 +1130,9 @@ def get_usage_breakdown( cache_read_tokens=int(metrics[2]), cache_creation_tokens=int(metrics[3]), reasoning_tokens=int(metrics[4]), - call_count=int(metrics[5]), + audio_seconds=int(metrics[5]), + tts_characters=int(metrics[6]), + call_count=int(metrics[7]), ) ) elif group_by == "product_section": @@ -293,7 +1148,9 @@ def get_usage_breakdown( cache_read_tokens=int(metrics[2]), cache_creation_tokens=int(metrics[3]), reasoning_tokens=int(metrics[4]), - call_count=int(metrics[5]), + audio_seconds=int(metrics[5]), + tts_characters=int(metrics[6]), + call_count=int(metrics[7]), ) ) elif group_by == "model": @@ -308,35 +1165,101 @@ def get_usage_breakdown( cache_read_tokens=int(metrics[2]), cache_creation_tokens=int(metrics[3]), reasoning_tokens=int(metrics[4]), - call_count=int(metrics[5]), + audio_seconds=int(metrics[5]), + tts_characters=int(metrics[6]), + call_count=int(metrics[7]), + ) + ) + elif group_by == "usage_kind": + kind = result[0] + metrics = result[1:] + rows.append( + UsageBreakdownRow( + usage_kind=kind, + prompt_tokens=int(metrics[0]), + completion_tokens=int(metrics[1]), + total_tokens=int(metrics[0]) + int(metrics[1]), + cache_read_tokens=int(metrics[2]), + cache_creation_tokens=int(metrics[3]), + reasoning_tokens=int(metrics[4]), + audio_seconds=int(metrics[5]), + tts_characters=int(metrics[6]), + call_count=int(metrics[7]), + ) + ) + elif group_by == "call_import": + raw_cid = result[0] + metrics = result[1:] + cid = None + if raw_cid: + try: + cid = UUID(str(raw_cid)) + except (ValueError, TypeError): + cid = None + label = ( + call_import_labels.get(str(raw_cid), "Unscoped") + if raw_cid + else "Unscoped" + ) + rows.append( + UsageBreakdownRow( + call_import_id=cid, + call_import_label=label, + prompt_tokens=int(metrics[0]), + completion_tokens=int(metrics[1]), + total_tokens=int(metrics[0]) + int(metrics[1]), + cache_read_tokens=int(metrics[2]), + cache_creation_tokens=int(metrics[3]), + reasoning_tokens=int(metrics[4]), + audio_seconds=int(metrics[5]), + tts_characters=int(metrics[6]), + call_count=int(metrics[7]), ) ) else: - res_id, res_type = result[0], result[1] - metrics = result[2:] + raw_res_id, res_type, section = result[0], result[1], result[2] + metrics = result[3:] + res_id = None + if raw_res_id: + try: + res_id = UUID(str(raw_res_id)) + except (ValueError, TypeError): + res_id = None rows.append( UsageBreakdownRow( resource_id=res_id, - resource_type=res_type, - resource_label=resource_labels.get(res_id) - if res_id - else "Unscoped", + resource_type=res_type or ( + "agent" if section == "agents" and raw_res_id else None + ), + resource_label=_breakdown_resource_label( + raw_res_id, + res_type, + section, + resource_labels, + db, + organization_id, + ), + product_section=section, + product_section_label=SECTION_LABELS.get(section or "", section), prompt_tokens=int(metrics[0]), completion_tokens=int(metrics[1]), total_tokens=int(metrics[0]) + int(metrics[1]), cache_read_tokens=int(metrics[2]), cache_creation_tokens=int(metrics[3]), reasoning_tokens=int(metrics[4]), - call_count=int(metrics[5]), + audio_seconds=int(metrics[5]), + tts_characters=int(metrics[6]), + call_count=int(metrics[7]), ) ) return UsageBreakdownResponse( - start=start, - end=end, + start=display_start, + end=display_end, group_by=group_by, rows=rows, - total_count=total_count, + total_count=len(rows), + truncated_at_limit=len(rows) >= limit, last_updated_at=_last_updated(db, organization_id), ) @@ -345,23 +1268,93 @@ def get_usage_breakdown( def get_usage_filters( start: Optional[date] = Query(None), end: Optional[date] = Query(None), + tz: Optional[str] = Query( + None, + description="IANA timezone for interpreting start/end calendar dates", + ), + workspace_id: Optional[UUID] = Query(None), + product_section: Optional[str] = Query(None), + model: Optional[str] = Query(None), + resource_id: Optional[UUID] = Query(None), + usage_kind: Optional[str] = Query(None), + call_import_id: Optional[UUID] = Query(None), + dataset: Optional[str] = Query(None), + tag_id: Optional[UUID] = Query(None), + q: Optional[str] = Query(None, description="Optional resource label search"), organization_id: UUID = Depends(get_organization_id), db: Session = Depends(get_db), ): - if start is None or end is None: - start, end = _default_range() + _, _, filter_start, filter_end = _parse_usage_range(start, end, tz) flush_usage_to_catalog(db, organization_id) - base = db.query(LLMUsageDaily).filter( - LLMUsageDaily.organization_id == organization_id, - LLMUsageDaily.usage_date >= start, - LLMUsageDaily.usage_date <= end, + workspace_base = _filtered_query( + db, + organization_id=organization_id, + start=filter_start, + end=filter_end, + dataset=dataset, + tag_id=tag_id, + ) + section_base = _filtered_query( + db, + organization_id=organization_id, + start=filter_start, + end=filter_end, + workspace_id=workspace_id, + dataset=dataset, + tag_id=tag_id, + ) + kind_base = _filtered_query( + db, + organization_id=organization_id, + start=filter_start, + end=filter_end, + workspace_id=workspace_id, + product_section=product_section, + call_import_id=call_import_id, + dataset=dataset, + tag_id=tag_id, + ) + model_base = _filtered_query( + db, + organization_id=organization_id, + start=filter_start, + end=filter_end, + workspace_id=workspace_id, + product_section=product_section, + usage_kind=usage_kind, + call_import_id=call_import_id, + dataset=dataset, + tag_id=tag_id, + ) + call_import_base = _filtered_query( + db, + organization_id=organization_id, + start=filter_start, + end=filter_end, + workspace_id=workspace_id, + product_section=product_section, + dataset=dataset, + tag_id=tag_id, + ) + resource_base = _filtered_query( + db, + organization_id=organization_id, + start=filter_start, + end=filter_end, + workspace_id=workspace_id, + product_section=product_section, + model=model, + usage_kind=usage_kind, + call_import_id=call_import_id, + dataset=dataset, + tag_id=tag_id, ) workspace_ids = { row[0] - for row in base.with_entities(LLMUsageDaily.workspace_id).distinct().all() + for row in workspace_base.with_entities(LLMUsageDaily.workspace_id).distinct().all() if row[0] is not None } workspaces = [ @@ -375,45 +1368,94 @@ def get_usage_filters( sections = sorted( { row[0] - for row in base.with_entities(LLMUsageDaily.product_section).distinct().all() + for row in section_base.with_entities(LLMUsageDaily.product_section).distinct().all() if row[0] } ) models = sorted( { row[0] - for row in base.with_entities(LLMUsageDaily.model).distinct().all() + for row in model_base.with_entities(LLMUsageDaily.model).distinct().all() + if row[0] + } + ) + kinds = sorted( + { + row[0] + for row in kind_base.with_entities(LLMUsageDaily.usage_kind).distinct().all() if row[0] } ) - resource_rows = ( - base.with_entities(LLMUsageDaily.resource_id, LLMUsageDaily.resource_type) - .filter(LLMUsageDaily.resource_id.isnot(None)) - .distinct() - .limit(100) - .all() + call_import_labels = _call_import_filter_labels( + db, + organization_id, + call_import_base, + workspace_id=workspace_id, + dataset=dataset, + tag_id=tag_id, ) - resource_ids = [r[0] for r in resource_rows] - eval_names = {} - if resource_ids: - for evaluation in ( - db.query(CallImportEvaluation) - .filter(CallImportEvaluation.id.in_(resource_ids)) - .all() - ): - eval_names[evaluation.id] = (evaluation.name or "").strip() or str( - evaluation.id - )[:8] + call_imports = [ + {"id": cid, "label": label} + for cid, label in sorted(call_import_labels.items(), key=lambda x: x[1].lower()) + ] - resources = [ - { - "id": str(rid), - "type": rtype, - "label": eval_names.get(rid, str(rid)[:8]), - } - for rid, rtype in resource_rows - if rid is not None + evaluation_labels = _evaluation_label_map(db, organization_id, resource_base) + needle = (q or "").strip().lower() + evaluations = [] + for rid, label in sorted(evaluation_labels.items(), key=lambda x: x[1].lower()): + if needle and needle not in label.lower(): + continue + evaluations.append({"id": rid, "label": label}) + + resource_labels = _resource_label_map(db, organization_id, resource_base) + resource_meta = _resource_filter_meta_map(db, organization_id, resource_base) + resources = [] + for rid, label in sorted(resource_labels.items(), key=lambda x: x[1].lower()): + if needle and needle not in label.lower(): + continue + info = resource_meta.get(rid, {}) + rtype = info.get("type") + if rtype == "call_import_evaluation": + continue + resources.append( + { + "id": rid, + "label": label, + "type": rtype, + "product_section": info.get("product_section"), + } + ) + + dataset_query = db.query(CallImport.dataset).filter( + CallImport.organization_id == organization_id, + CallImport.dataset.isnot(None), + CallImport.dataset != "", + ) + if workspace_id is not None: + dataset_query = dataset_query.filter(CallImport.workspace_id == workspace_id) + if dataset or tag_id is not None: + scoped_import_ids = _call_import_ids_for_filters( + db, + organization_id=organization_id, + workspace_id=workspace_id, + dataset=dataset, + tag_id=tag_id, + ) + if scoped_import_ids: + dataset_query = dataset_query.filter(CallImport.id.in_(scoped_import_ids)) + else: + dataset_query = dataset_query.filter(CallImport.id.is_(None)) + datasets = sorted({row[0] for row in dataset_query.distinct().all() if row[0]}) + + tags = [ + {"id": str(tag.id), "label": tag.name} + for tag in ( + db.query(CallImportTag) + .filter(CallImportTag.organization_id == organization_id) + .order_by(CallImportTag.name) + .all() + ) ] return UsageFiltersResponse( @@ -421,6 +1463,12 @@ def get_usage_filters( product_sections=[ {"id": s, "label": SECTION_LABELS.get(s, s)} for s in sections ], + call_imports=call_imports, + evaluations=evaluations, models=models, resources=resources, + usage_kinds=[{"id": k, "label": usage_kind_label(k)} for k in kinds], + datasets=datasets, + tags=tags, ) + diff --git a/app/api/v1/routes/playground.py b/app/api/v1/routes/playground.py index 19e97a65..4c4d93dd 100644 --- a/app/api/v1/routes/playground.py +++ b/app/api/v1/routes/playground.py @@ -38,215 +38,9 @@ from app.services.evaluators.evaluator_result_call_data import slim_call_data_for_evaluator_result from app.utils.call_recordings import generate_unique_call_short_id -router = APIRouter(prefix="/playground", tags=["playground"]) - - -def extract_transcript_from_call_data(call_data: Dict[str, Any], provider_platform: str) -> tuple: - """ - Extract transcript and speaker segments from provider call_data. - - Args: - call_data: Full call data from voice provider - provider_platform: The provider platform ("vapi", "retell", "elevenlabs", "smallest") - - Returns: - Tuple of (transcript_text, speaker_segments) - - transcript_text: Plain text transcript - - speaker_segments: List of segments with speaker labels - """ - transcript_text = "" - speaker_segments = [] - - if not call_data: - return transcript_text, speaker_segments - - provider_platform_lower = provider_platform.lower() if provider_platform else "" - - if provider_platform_lower == "vapi": - # Vapi: keep provider payload raw and derive transcript from transcript/messages. - transcript_text = call_data.get("transcript", "") - - # Get structured messages for speaker segments - transcript_object = call_data.get("transcript_object", []) - if not transcript_object: - # Try messages array - artifact = call_data.get("artifact", {}) if isinstance(call_data, dict) else {} - messages = call_data.get("messages", []) or artifact.get("messages", []) - for msg in messages: - role = msg.get("role", "unknown") - content = msg.get("message", "") or msg.get("content", "") - - if not content or role == "system": - continue - - # Map roles - if role in ["bot", "assistant"]: - normalized_role = "agent" - elif role == "user": - normalized_role = "user" - else: - continue - - speaker_segments.append({ - "speaker": "Agent" if normalized_role == "agent" else "User", - "text": content, - "start": msg.get("secondsFromStart", 0), - "end": msg.get("secondsFromStart", 0) + (msg.get("duration", 0) / 1000), - }) - else: - for entry in transcript_object: - role = entry.get("role", "unknown") - content = entry.get("content", "") - - if not content: - continue - - speaker_segments.append({ - "speaker": "Agent" if role == "agent" else "User", - "text": content, - "start": entry.get("seconds_from_start", 0), - "end": entry.get("seconds_from_start", 0) + (entry.get("duration_ms", 0) / 1000), - }) - - # Build transcript text from segments if not available - if not transcript_text and speaker_segments: - transcript_text = "\n".join([ - f"{seg['speaker']}: {seg['text']}" for seg in speaker_segments - ]) - - elif provider_platform_lower == "elevenlabs": - raw_transcript = call_data.get("transcript") - transcript_obj = call_data.get("transcript_object", []) - - # retrieve_call_metrics already processes the transcript into a - # formatted string + speaker_segments list, so handle both the - # pre-processed shape and the raw ElevenLabs API shape. - if isinstance(raw_transcript, str) and raw_transcript: - transcript_text = raw_transcript - if isinstance(transcript_obj, list): - for seg in transcript_obj: - speaker_segments.append({ - "speaker": seg.get("speaker", "Unknown"), - "text": seg.get("text", ""), - "start": seg.get("start", 0), - "end": seg.get("end", 0), - }) - elif isinstance(raw_transcript, list): - for entry in raw_transcript: - role = entry.get("role", "unknown") - content = entry.get("message", "") or entry.get("text", "") - if not content: - continue - speaker = "Agent" if role in ("agent", "assistant", "ai") else "User" - speaker_segments.append({ - "speaker": speaker, - "text": content, - "start": entry.get("time_in_call_secs", 0) or entry.get("start", 0), - "end": entry.get("time_in_call_secs", 0) or entry.get("end", 0), - }) - transcript_text = "\n".join( - f"{seg['speaker']}: {seg['text']}" for seg in speaker_segments - ) - - elif provider_platform_lower == "smallest": - transcript_raw = call_data.get("transcript") - transcript_object = call_data.get("transcript_object", []) - if isinstance(transcript_object, list) and transcript_object: - for entry in transcript_object: - if not isinstance(entry, dict): - continue - text = entry.get("text", "") - if not text: - continue - speaker = entry.get("speaker", "Unknown") - speaker_segments.append( - { - "speaker": speaker, - "text": text, - "start": entry.get("start", 0), - "end": entry.get("end", entry.get("start", 0)), - } - ) - if not transcript_text: - transcript_text = "\n".join( - f"{seg['speaker']}: {seg['text']}" for seg in speaker_segments - ) - elif isinstance(transcript_raw, list): - for entry in transcript_raw: - if not isinstance(entry, dict): - continue - role = str(entry.get("speaker") or entry.get("role") or "").lower() - speaker = "Agent" if role in ("agent", "assistant", "ai", "bot") else "User" - text = entry.get("text", "") or entry.get("message", "") or entry.get("content", "") - if not text: - continue - ts = entry.get("timeInCallSecs", 0) or entry.get("start", 0) or entry.get("timestamp", 0) - speaker_segments.append( - { - "speaker": speaker, - "text": text, - "start": ts, - "end": entry.get("end", ts), - } - ) - transcript_text = "\n".join( - f"{seg['speaker']}: {seg['text']}" for seg in speaker_segments - ) - elif isinstance(transcript_raw, str): - transcript_text = transcript_raw - - elif provider_platform_lower == "retell": - # Retell: transcript can be a string or list of objects - transcript_raw = call_data.get("transcript", "") - - if isinstance(transcript_raw, str): - transcript_text = transcript_raw - # Parse transcript text into speaker segments if it has pattern like "Agent: text\nUser: text" - lines = transcript_raw.split("\n") if transcript_raw else [] - for line in lines: - line = line.strip() - if not line: - continue - if line.startswith("Agent:") or line.startswith("agent:"): - speaker_segments.append({ - "speaker": "Agent", - "text": line.split(":", 1)[1].strip() if ":" in line else line, - "start": 0, - "end": 0, - }) - elif line.startswith("User:") or line.startswith("user:"): - speaker_segments.append({ - "speaker": "User", - "text": line.split(":", 1)[1].strip() if ":" in line else line, - "start": 0, - "end": 0, - }) - elif isinstance(transcript_raw, list): - # Retell sometimes returns transcript as array of objects - for item in transcript_raw: - if isinstance(item, dict): - role = item.get("role", "") - content = item.get("content", "") or item.get("text", "") - - if not content: - continue - - speaker = "Agent" if role in ["agent", "assistant", "bot"] else "User" - speaker_segments.append({ - "speaker": speaker, - "text": content, - "start": item.get("start_time", 0) or item.get("timestamp", 0), - "end": item.get("end_time", 0), - }) - - # Build transcript text from segments - transcript_text = "\n".join([ - f"{seg['speaker']}: {seg['text']}" for seg in speaker_segments - ]) - - return transcript_text, speaker_segments - +from app.services.evaluators.call_data_transcript import extract_transcript_from_call_data +router = APIRouter(prefix="/playground", tags=["playground"]) def generate_unique_result_id(db: Session) -> str: """Generate a unique 6-digit result ID for EvaluatorResult.""" max_attempts = 100 @@ -411,7 +205,8 @@ def poll_call_metrics( or recording_urls.get("stereo_url") ) if audio_url: - resp = _http.get(audio_url, timeout=120) + vapi_headers = {"Authorization": f"Bearer {integration_api_key}"} + resp = _http.get(audio_url, headers=vapi_headers, timeout=120) if resp.status_code == 200: audio_bytes = resp.content @@ -656,13 +451,32 @@ async def create_web_call( # For now, we'll skip it for Retell. Other providers can handle it in their implementation. if integration.platform != "retell" and web_call_data.custom_sip_headers: call_params["custom_sip_headers"] = web_call_data.custom_sip_headers - - web_call_response = provider.create_web_call(**call_params) - - # Store call recording in database + + platform_value = ( + integration.platform.value + if hasattr(integration.platform, "value") + else integration.platform + ) + plat_lower = str(platform_value).lower() + + # Vapi Web SDK creates the call in the browser; server-side /call/web + # would spawn a second call that never receives the user's microphone. + if plat_lower == "vapi": + from app.services.voice_providers.vapi import VAPI_SAMPLE_RATE + + web_call_response = { + "call_type": "web_call", + "agent_id": agent.voice_ai_agent_id, + "metadata": web_call_data.metadata or {}, + "sample_rate": VAPI_SAMPLE_RATE, + "client_sdk_creates_call": True, + } + provider_call_id = None + else: + web_call_response = provider.create_web_call(**call_params) + provider_call_id = web_call_response.get("call_id") + call_short_id = generate_unique_call_short_id(db) - provider_call_id = web_call_response.get("call_id") - call_recording = CallRecording( organization_id=organization_id, workspace_id=workspace_id, @@ -702,17 +516,13 @@ async def create_web_call( # Add call_short_id to response for frontend response = web_call_response.copy() response["call_short_id"] = call_short_id - - platform_value = integration.platform.value if hasattr(integration.platform, 'value') else integration.platform - - # For Vapi, include the public key in the response (needed for frontend SDK) - if platform_value.lower() == "vapi" and integration.public_key: + + if plat_lower == "vapi" and integration.public_key: response["public_key"] = integration.public_key - - # For ElevenLabs, pass through the signed_url (frontend SDK connects directly) - if platform_value.lower() == "elevenlabs": + + if plat_lower == "elevenlabs": response["signed_url"] = web_call_response.get("signed_url") - + return response except Exception as e: raise HTTPException( @@ -1850,7 +1660,10 @@ async def summarize_transcript( 1. The voice bundle of ``agent_id`` (or the agent on ``call_short_id``). 2. Any configured AIProvider matching the fallback preference list. """ + from contextlib import nullcontext + from app.services.ai.llm_service import llm_service + from app.services.usage.context import llm_usage_context, usage_context_for_agent transcript_text = (payload.transcript or "").strip() if not transcript_text and payload.entries: @@ -1947,15 +1760,27 @@ async def summarize_transcript( ] try: - result = llm_service.generate_response( - messages=messages, - llm_provider=llm_provider, - llm_model=llm_model, - organization_id=organization_id, - db=db, - temperature=0.3, - max_tokens=400, - ) + usage_ctx = nullcontext() + if agent_uuid: + agent_row = db.query(Agent).filter( + Agent.id == agent_uuid, + Agent.organization_id == organization_id, + ).first() + if agent_row: + usage_ctx = llm_usage_context( + usage_context_for_agent(agent_row, workspace_id=workspace_id) + ) + + with usage_ctx: + result = llm_service.generate_response( + messages=messages, + llm_provider=llm_provider, + llm_model=llm_model, + organization_id=organization_id, + db=db, + temperature=0.3, + max_tokens=400, + ) except Exception as e: logger.error(f"[summarize-transcript] LLM call failed: {e}") raise HTTPException( diff --git a/app/api/v1/routes/vobiz_telephony.py b/app/api/v1/routes/vobiz_telephony.py index 084d03b7..052d03b9 100644 --- a/app/api/v1/routes/vobiz_telephony.py +++ b/app/api/v1/routes/vobiz_telephony.py @@ -698,6 +698,7 @@ async def vobiz_media_websocket(websocket: WebSocket): websocket, context.system_instruction, str(context.organization_id), + str(context.workspace_id) if context.workspace_id else None, agent_id, persona_id, scenario_id, diff --git a/app/api/v1/routes/voice_agent.py b/app/api/v1/routes/voice_agent.py index 47f7350e..e6d553b5 100644 --- a/app/api/v1/routes/voice_agent.py +++ b/app/api/v1/routes/voice_agent.py @@ -517,6 +517,7 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: websocket, system_instruction, str(organization_id), + str(workspace_id) if workspace_id else None, agent_id, persona_id, scenario_id, diff --git a/app/migrations/063_usage_kind_stt_and_buffer.py b/app/migrations/063_usage_kind_stt_and_buffer.py new file mode 100644 index 00000000..1cbb51d9 --- /dev/null +++ b/app/migrations/063_usage_kind_stt_and_buffer.py @@ -0,0 +1,233 @@ +"""Migration: STT usage_kind/audio_seconds + Redis-fallback pending buffer.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Add usage_kind/audio_seconds to llm_usage_daily and usage_pending_buffer " + "for durable STT + LLM usage when Redis is unavailable" +) + +_ZERO_UUID = "00000000-0000-0000-0000-000000000000" + + +def _column_exists(db: Session, table: str, column: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table, "column_name": column}, + ).first() + is not None + ) + + +def _table_exists(db: Session, table: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.tables + WHERE table_name = :table_name + """ + ), + {"table_name": table}, + ).first() + is not None + ) + + +def _dedupe_llm_usage_daily(db: Session) -> int: + """Merge duplicate bucket rows so the unique index can be created. + + Keeps the oldest row (MIN id), sums metrics into it, deletes extras. + """ + db.execute( + text( + f""" + WITH dupes AS ( + SELECT + organization_id, + COALESCE( + workspace_id, + '{_ZERO_UUID}'::uuid + ) AS ws_key, + product_section, + model, + COALESCE( + resource_id, + '{_ZERO_UUID}'::uuid + ) AS rid_key, + usage_date, + COALESCE(usage_kind, 'llm') AS kind_key, + MIN(id::text)::uuid AS keep_id, + SUM(prompt_tokens)::bigint AS prompt_tokens, + SUM(completion_tokens)::bigint AS completion_tokens, + SUM(cache_read_tokens)::bigint AS cache_read_tokens, + SUM(cache_creation_tokens)::bigint AS cache_creation_tokens, + SUM(reasoning_tokens)::bigint AS reasoning_tokens, + SUM(COALESCE(audio_seconds, 0))::bigint AS audio_seconds, + SUM(call_count)::bigint AS call_count + FROM llm_usage_daily + GROUP BY 1, 2, 3, 4, 5, 6, 7 + HAVING COUNT(*) > 1 + ) + UPDATE llm_usage_daily AS u SET + prompt_tokens = d.prompt_tokens, + completion_tokens = d.completion_tokens, + cache_read_tokens = d.cache_read_tokens, + cache_creation_tokens = d.cache_creation_tokens, + reasoning_tokens = d.reasoning_tokens, + audio_seconds = d.audio_seconds, + call_count = d.call_count, + updated_at = now() + FROM dupes AS d + WHERE u.id = d.keep_id + """ + ) + ) + + result = db.execute( + text( + f""" + WITH keepers AS ( + SELECT MIN(id::text)::uuid AS keep_id + FROM llm_usage_daily + GROUP BY + organization_id, + COALESCE(workspace_id, '{_ZERO_UUID}'::uuid), + product_section, + model, + COALESCE(resource_id, '{_ZERO_UUID}'::uuid), + usage_date, + COALESCE(usage_kind, 'llm') + HAVING COUNT(*) > 1 + ) + DELETE FROM llm_usage_daily AS u + USING keepers AS k, + llm_usage_daily AS peer + WHERE peer.id = k.keep_id + AND u.id <> k.keep_id + AND u.organization_id = peer.organization_id + AND u.product_section = peer.product_section + AND u.model = peer.model + AND u.usage_date = peer.usage_date + AND COALESCE(u.usage_kind, 'llm') = COALESCE(peer.usage_kind, 'llm') + AND u.workspace_id IS NOT DISTINCT FROM peer.workspace_id + AND u.resource_id IS NOT DISTINCT FROM peer.resource_id + """ + ) + ) + return int(result.rowcount or 0) + + +def _ensure_unique_bucket_index(db: Session) -> None: + """Drop legacy/broken unique index and recreate with usage_kind.""" + db.execute(text("DROP INDEX IF EXISTS uq_llm_usage_daily_bucket")) + db.execute( + text( + f""" + CREATE UNIQUE INDEX uq_llm_usage_daily_bucket + ON llm_usage_daily ( + organization_id, + COALESCE(workspace_id, '{_ZERO_UUID}'::uuid), + product_section, + model, + COALESCE(resource_id, '{_ZERO_UUID}'::uuid), + usage_date, + usage_kind + ) + """ + ) + ) + + +def upgrade(db: Session): + if not _table_exists(db, "llm_usage_daily"): + print("llm_usage_daily missing; run 062 first — skipping 063") + db.commit() + return + + if not _column_exists(db, "llm_usage_daily", "usage_kind"): + db.execute( + text( + """ + ALTER TABLE llm_usage_daily + ADD COLUMN usage_kind VARCHAR(16) NOT NULL DEFAULT 'llm' + """ + ) + ) + + if not _column_exists(db, "llm_usage_daily", "audio_seconds"): + db.execute( + text( + """ + ALTER TABLE llm_usage_daily + ADD COLUMN audio_seconds BIGINT NOT NULL DEFAULT 0 + """ + ) + ) + + removed = _dedupe_llm_usage_daily(db) + if removed: + print(f"Merged/removed {removed} duplicate llm_usage_daily row(s)") + + _ensure_unique_bucket_index(db) + + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_llm_usage_daily_org_kind_date + ON llm_usage_daily (organization_id, usage_kind, usage_date) + """ + ) + ) + print("Ensured usage_kind + audio_seconds + unique bucket index") + + if not _table_exists(db, "usage_pending_buffer"): + db.execute( + text( + """ + CREATE TABLE usage_pending_buffer ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id UUID NOT NULL, + workspace_id UUID, + product_section VARCHAR(64) NOT NULL, + model VARCHAR(255) NOT NULL, + context JSONB NOT NULL DEFAULT '{}'::jsonb, + usage_date DATE NOT NULL, + usage_kind VARCHAR(16) NOT NULL DEFAULT 'llm', + prompt_tokens BIGINT NOT NULL DEFAULT 0, + completion_tokens BIGINT NOT NULL DEFAULT 0, + cache_read_tokens BIGINT NOT NULL DEFAULT 0, + cache_creation_tokens BIGINT NOT NULL DEFAULT 0, + reasoning_tokens BIGINT NOT NULL DEFAULT 0, + audio_seconds BIGINT NOT NULL DEFAULT 0, + call_count BIGINT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_usage_pending_buffer_org_created + ON usage_pending_buffer (organization_id, created_at) + """ + ) + ) + print("Created usage_pending_buffer") + + db.commit() + + +def downgrade(db: Session): + db.execute(text("DROP TABLE IF EXISTS usage_pending_buffer")) + db.execute(text("DROP INDEX IF EXISTS ix_llm_usage_daily_org_kind_date")) + # Keep usage_kind/audio_seconds columns on downgrade to avoid data loss. + db.commit() diff --git a/app/migrations/064_usage_kind_tts_characters.py b/app/migrations/064_usage_kind_tts_characters.py new file mode 100644 index 00000000..57d42da0 --- /dev/null +++ b/app/migrations/064_usage_kind_tts_characters.py @@ -0,0 +1,74 @@ +"""Migration: TTS usage_kind + tts_characters metric.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add tts_characters to llm_usage_daily and usage_pending_buffer for TTS usage" + + +def _column_exists(db: Session, table: str, column: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table, "column_name": column}, + ).first() + is not None + ) + + +def _table_exists(db: Session, table: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.tables + WHERE table_name = :table_name + """ + ), + {"table_name": table}, + ).first() + is not None + ) + + +def upgrade(db: Session): + if _table_exists(db, "llm_usage_daily") and not _column_exists( + db, "llm_usage_daily", "tts_characters" + ): + db.execute( + text( + """ + ALTER TABLE llm_usage_daily + ADD COLUMN tts_characters BIGINT NOT NULL DEFAULT 0 + """ + ) + ) + print("Added tts_characters to llm_usage_daily") + + if _table_exists(db, "usage_pending_buffer") and not _column_exists( + db, "usage_pending_buffer", "tts_characters" + ): + db.execute( + text( + """ + ALTER TABLE usage_pending_buffer + ADD COLUMN tts_characters BIGINT NOT NULL DEFAULT 0 + """ + ) + ) + print("Added tts_characters to usage_pending_buffer") + + db.commit() + + +def downgrade(db: Session): + if _column_exists(db, "llm_usage_daily", "tts_characters"): + db.execute(text("ALTER TABLE llm_usage_daily DROP COLUMN tts_characters")) + if _column_exists(db, "usage_pending_buffer", "tts_characters"): + db.execute(text("ALTER TABLE usage_pending_buffer DROP COLUMN tts_characters")) + db.commit() diff --git a/app/migrations/065_usage_context_jsonb.py b/app/migrations/065_usage_context_jsonb.py new file mode 100644 index 00000000..4dd62378 --- /dev/null +++ b/app/migrations/065_usage_context_jsonb.py @@ -0,0 +1,216 @@ +"""Migration: JSONB context column for usage attribution metadata.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Move resource_id/resource_type into llm_usage_daily.context JSONB and " + "recreate bucket uniqueness on context keys" +) + +_ZERO_UUID = "00000000-0000-0000-0000-000000000000" + + +def _column_exists(db: Session, table: str, column: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table, "column_name": column}, + ).first() + is not None + ) + + +def _table_exists(db: Session, table: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.tables + WHERE table_name = :table_name + """ + ), + {"table_name": table}, + ).first() + is not None + ) + + +def _ensure_unique_bucket_index(db: Session) -> None: + db.execute(text("DROP INDEX IF EXISTS uq_llm_usage_daily_bucket")) + db.execute( + text( + f""" + CREATE UNIQUE INDEX uq_llm_usage_daily_bucket + ON llm_usage_daily ( + organization_id, + COALESCE(workspace_id, '{_ZERO_UUID}'::uuid), + product_section, + model, + usage_date, + usage_kind, + context + ) + """ + ) + ) + + +def _migrate_table_context(db: Session, table: str) -> None: + if not _table_exists(db, table): + return + if not _column_exists(db, table, "context"): + db.execute( + text( + f""" + ALTER TABLE {table} + ADD COLUMN context JSONB NOT NULL DEFAULT '{{}}'::jsonb + """ + ) + ) + + if _column_exists(db, table, "resource_id") or _column_exists(db, table, "resource_type"): + db.execute( + text( + f""" + UPDATE {table} + SET context = COALESCE(context, '{{}}'::jsonb) + || CASE + WHEN resource_id IS NOT NULL THEN + jsonb_build_object('resource_id', resource_id::text) + ELSE '{{}}'::jsonb + END + || CASE + WHEN resource_type IS NOT NULL AND resource_type <> '' THEN + jsonb_build_object('resource_type', resource_type) + ELSE '{{}}'::jsonb + END + WHERE resource_id IS NOT NULL + OR (resource_type IS NOT NULL AND resource_type <> '') + """ + ) + ) + if _column_exists(db, table, "resource_id"): + db.execute(text(f"DROP INDEX IF EXISTS ix_llm_usage_daily_org_resource_date")) + db.execute(text(f"ALTER TABLE {table} DROP COLUMN resource_id")) + if _column_exists(db, table, "resource_type"): + db.execute(text(f"ALTER TABLE {table} DROP COLUMN resource_type")) + + +def upgrade(db: Session): + if not _table_exists(db, "llm_usage_daily"): + print("llm_usage_daily missing; run 062 first — skipping 065") + db.commit() + return + + _migrate_table_context(db, "llm_usage_daily") + _migrate_table_context(db, "usage_pending_buffer") + + _ensure_unique_bucket_index(db) + + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_llm_usage_daily_context_gin + ON llm_usage_daily USING gin (context) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_llm_usage_daily_context_resource_id + ON llm_usage_daily ((context->>'resource_id')) + WHERE context ? 'resource_id' + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_llm_usage_daily_context_call_import_id + ON llm_usage_daily ((context->>'call_import_id')) + WHERE context ? 'call_import_id' + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_llm_usage_daily_context_evaluation_id + ON llm_usage_daily ((context->>'evaluation_id')) + WHERE context ? 'evaluation_id' + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_llm_usage_daily_context_evaluation_row_id + ON llm_usage_daily ((context->>'evaluation_row_id')) + WHERE context ? 'evaluation_row_id' + """ + ) + ) + print("Added context JSONB + migrated resource attribution") + db.commit() + + +def downgrade(db: Session): + if not _table_exists(db, "llm_usage_daily"): + db.commit() + return + + if not _column_exists(db, "llm_usage_daily", "resource_id"): + db.execute( + text( + """ + ALTER TABLE llm_usage_daily + ADD COLUMN resource_id UUID, + ADD COLUMN resource_type VARCHAR(64) + """ + ) + ) + db.execute( + text( + """ + UPDATE llm_usage_daily + SET resource_id = NULLIF(context->>'resource_id', '')::uuid, + resource_type = NULLIF(context->>'resource_type', '') + WHERE context IS NOT NULL + """ + ) + ) + + db.execute(text("DROP INDEX IF EXISTS ix_llm_usage_daily_context_evaluation_row_id")) + db.execute(text("DROP INDEX IF EXISTS ix_llm_usage_daily_context_evaluation_id")) + db.execute(text("DROP INDEX IF EXISTS ix_llm_usage_daily_context_call_import_id")) + db.execute(text("DROP INDEX IF EXISTS ix_llm_usage_daily_context_resource_id")) + db.execute(text("DROP INDEX IF EXISTS ix_llm_usage_daily_context_gin")) + db.execute(text("DROP INDEX IF EXISTS uq_llm_usage_daily_bucket")) + + if _column_exists(db, "llm_usage_daily", "context"): + db.execute(text("ALTER TABLE llm_usage_daily DROP COLUMN context")) + + db.execute( + text( + f""" + CREATE UNIQUE INDEX uq_llm_usage_daily_bucket + ON llm_usage_daily ( + organization_id, + COALESCE(workspace_id, '{_ZERO_UUID}'::uuid), + product_section, + model, + COALESCE(resource_id, '{_ZERO_UUID}'::uuid), + usage_date, + usage_kind + ) + """ + ) + ) + db.commit() diff --git a/app/migrations/066_backfill_llm_usage_workspace_from_call_import.py b/app/migrations/066_backfill_llm_usage_workspace_from_call_import.py new file mode 100644 index 00000000..478afd4f --- /dev/null +++ b/app/migrations/066_backfill_llm_usage_workspace_from_call_import.py @@ -0,0 +1,279 @@ +"""Migration: Backfill llm_usage_daily.workspace_id from call-import attribution.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Backfill workspace_id on llm_usage_daily and usage_pending_buffer rows " + "that have call-import context but were recorded with NULL workspace_id" +) + +_ZERO_UUID = "00000000-0000-0000-0000-000000000000" +_UUID_RE = "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" + + +def _column_exists(db: Session, table: str, column: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table, "column_name": column}, + ).first() + is not None + ) + + +def _table_exists(db: Session, table: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.tables + WHERE table_name = :table_name + """ + ), + {"table_name": table}, + ).first() + is not None + ) + + +def _dedupe_llm_usage_daily(db: Session) -> int: + """Merge duplicate buckets after workspace backfill.""" + tts_col = ( + "SUM(COALESCE(tts_characters, 0))::bigint AS tts_characters," + if _column_exists(db, "llm_usage_daily", "tts_characters") + else "" + ) + tts_set = ( + "tts_characters = d.tts_characters," + if _column_exists(db, "llm_usage_daily", "tts_characters") + else "" + ) + + db.execute( + text( + f""" + WITH dupes AS ( + SELECT + organization_id, + COALESCE(workspace_id, '{_ZERO_UUID}'::uuid) AS ws_key, + product_section, + model, + usage_date, + COALESCE(usage_kind, 'llm') AS kind_key, + context, + MIN(id::text)::uuid AS keep_id, + SUM(prompt_tokens)::bigint AS prompt_tokens, + SUM(completion_tokens)::bigint AS completion_tokens, + SUM(cache_read_tokens)::bigint AS cache_read_tokens, + SUM(cache_creation_tokens)::bigint AS cache_creation_tokens, + SUM(reasoning_tokens)::bigint AS reasoning_tokens, + SUM(COALESCE(audio_seconds, 0))::bigint AS audio_seconds, + {tts_col} + SUM(call_count)::bigint AS call_count + FROM llm_usage_daily + GROUP BY 1, 2, 3, 4, 5, 6, 7 + HAVING COUNT(*) > 1 + ) + UPDATE llm_usage_daily AS u SET + prompt_tokens = d.prompt_tokens, + completion_tokens = d.completion_tokens, + cache_read_tokens = d.cache_read_tokens, + cache_creation_tokens = d.cache_creation_tokens, + reasoning_tokens = d.reasoning_tokens, + audio_seconds = d.audio_seconds, + {tts_set} + call_count = d.call_count, + updated_at = now() + FROM dupes AS d + WHERE u.id = d.keep_id + """ + ) + ) + + result = db.execute( + text( + f""" + WITH keepers AS ( + SELECT MIN(id::text)::uuid AS keep_id + FROM llm_usage_daily + GROUP BY + organization_id, + COALESCE(workspace_id, '{_ZERO_UUID}'::uuid), + product_section, + model, + usage_date, + COALESCE(usage_kind, 'llm'), + context + HAVING COUNT(*) > 1 + ) + DELETE FROM llm_usage_daily AS u + USING keepers AS k, + llm_usage_daily AS peer + WHERE peer.id = k.keep_id + AND u.id <> k.keep_id + AND u.organization_id = peer.organization_id + AND u.product_section = peer.product_section + AND u.model = peer.model + AND u.usage_date = peer.usage_date + AND COALESCE(u.usage_kind, 'llm') = COALESCE(peer.usage_kind, 'llm') + AND u.workspace_id IS NOT DISTINCT FROM peer.workspace_id + AND u.context IS NOT DISTINCT FROM peer.context + """ + ) + ) + return int(result.rowcount or 0) + + +def _backfill_table_workspace(db: Session, table: str) -> int: + if not _table_exists(db, table): + return 0 + if not _column_exists(db, table, "workspace_id"): + return 0 + if not _column_exists(db, table, "context"): + return 0 + + total = 0 + + # context.call_import_id + result = db.execute( + text( + f""" + UPDATE {table} AS u + SET workspace_id = ci.workspace_id + FROM call_imports AS ci + WHERE u.workspace_id IS NULL + AND u.context ? 'call_import_id' + AND u.context->>'call_import_id' ~ :uuid_re + AND ci.id = (u.context->>'call_import_id')::uuid + AND ci.organization_id = u.organization_id + """ + ), + {"uuid_re": _UUID_RE}, + ) + total += int(result.rowcount or 0) + + # resource_type = call_import + result = db.execute( + text( + f""" + UPDATE {table} AS u + SET workspace_id = ci.workspace_id + FROM call_imports AS ci + WHERE u.workspace_id IS NULL + AND u.context->>'resource_type' = 'call_import' + AND u.context->>'resource_id' ~ :uuid_re + AND ci.id = (u.context->>'resource_id')::uuid + AND ci.organization_id = u.organization_id + """ + ), + {"uuid_re": _UUID_RE}, + ) + total += int(result.rowcount or 0) + + # context.evaluation_id + result = db.execute( + text( + f""" + UPDATE {table} AS u + SET workspace_id = e.workspace_id + FROM call_import_evaluations AS e + WHERE u.workspace_id IS NULL + AND u.context ? 'evaluation_id' + AND u.context->>'evaluation_id' ~ :uuid_re + AND e.id = (u.context->>'evaluation_id')::uuid + AND e.organization_id = u.organization_id + """ + ), + {"uuid_re": _UUID_RE}, + ) + total += int(result.rowcount or 0) + + # resource_type = call_import_evaluation + result = db.execute( + text( + f""" + UPDATE {table} AS u + SET workspace_id = e.workspace_id + FROM call_import_evaluations AS e + WHERE u.workspace_id IS NULL + AND u.context->>'resource_type' = 'call_import_evaluation' + AND u.context->>'resource_id' ~ :uuid_re + AND e.id = (u.context->>'resource_id')::uuid + AND e.organization_id = u.organization_id + """ + ), + {"uuid_re": _UUID_RE}, + ) + total += int(result.rowcount or 0) + + # context.call_import_row_id + result = db.execute( + text( + f""" + UPDATE {table} AS u + SET workspace_id = cir.workspace_id + FROM call_import_rows AS cir + WHERE u.workspace_id IS NULL + AND u.context ? 'call_import_row_id' + AND u.context->>'call_import_row_id' ~ :uuid_re + AND cir.id = (u.context->>'call_import_row_id')::uuid + AND cir.organization_id = u.organization_id + """ + ), + {"uuid_re": _UUID_RE}, + ) + total += int(result.rowcount or 0) + + # context.evaluation_row_id + result = db.execute( + text( + f""" + UPDATE {table} AS u + SET workspace_id = e.workspace_id + FROM call_import_evaluation_rows AS er + JOIN call_import_evaluations AS e ON e.id = er.evaluation_id + WHERE u.workspace_id IS NULL + AND u.context ? 'evaluation_row_id' + AND u.context->>'evaluation_row_id' ~ :uuid_re + AND er.id = (u.context->>'evaluation_row_id')::uuid + AND e.organization_id = u.organization_id + """ + ), + {"uuid_re": _UUID_RE}, + ) + total += int(result.rowcount or 0) + + return total + + +def upgrade(db: Session): + if not _table_exists(db, "llm_usage_daily"): + print("llm_usage_daily missing; skipping 066") + db.commit() + return + + daily_updated = _backfill_table_workspace(db, "llm_usage_daily") + print(f"Backfilled workspace_id on {daily_updated} llm_usage_daily row(s)") + + removed = _dedupe_llm_usage_daily(db) + if removed: + print(f"Removed {removed} duplicate llm_usage_daily row(s) after merge") + + buffer_updated = _backfill_table_workspace(db, "usage_pending_buffer") + if buffer_updated: + print( + f"Backfilled workspace_id on {buffer_updated} usage_pending_buffer row(s)" + ) + + db.commit() + + +def downgrade(db: Session): + # No-op: cannot distinguish backfilled workspace_id from originally recorded values. + db.commit() diff --git a/app/migrations/067_fix_llm_usage_bucket_unique_index.py b/app/migrations/067_fix_llm_usage_bucket_unique_index.py new file mode 100644 index 00000000..98a78e5d --- /dev/null +++ b/app/migrations/067_fix_llm_usage_bucket_unique_index.py @@ -0,0 +1,264 @@ +"""Migration: Reconcile llm_usage_daily bucket unique index with full JSONB context.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Merge legacy resource-scoped usage buckets and recreate uq_llm_usage_daily_bucket " + "on full context JSONB (per-row attribution)" +) + +_ZERO_UUID = "00000000-0000-0000-0000-000000000000" + + +def _table_exists(db: Session, table: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.tables + WHERE table_name = :table_name + """ + ), + {"table_name": table}, + ).first() + is not None + ) + + +def _column_exists(db: Session, table: str, column: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table, "column_name": column}, + ).first() + is not None + ) + + +def _dedupe_by_context_resource_keys(db: Session) -> int: + """Merge rows that share context resource_id/resource_type (legacy unique index).""" + tts_col = ( + "SUM(COALESCE(tts_characters, 0))::bigint AS tts_characters," + if _column_exists(db, "llm_usage_daily", "tts_characters") + else "" + ) + tts_set = ( + "tts_characters = d.tts_characters," + if _column_exists(db, "llm_usage_daily", "tts_characters") + else "" + ) + db.execute( + text( + f""" + WITH dupes AS ( + SELECT + organization_id, + COALESCE(workspace_id, '{_ZERO_UUID}'::uuid) AS ws_key, + product_section, + model, + usage_date, + COALESCE(usage_kind, 'llm') AS kind_key, + COALESCE(context->>'resource_id', '') AS res_id_key, + COALESCE(context->>'resource_type', '') AS res_type_key, + MIN(id::text)::uuid AS keep_id, + SUM(prompt_tokens)::bigint AS prompt_tokens, + SUM(completion_tokens)::bigint AS completion_tokens, + SUM(cache_read_tokens)::bigint AS cache_read_tokens, + SUM(cache_creation_tokens)::bigint AS cache_creation_tokens, + SUM(reasoning_tokens)::bigint AS reasoning_tokens, + SUM(COALESCE(audio_seconds, 0))::bigint AS audio_seconds, + {tts_col} + SUM(call_count)::bigint AS call_count + FROM llm_usage_daily + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8 + HAVING COUNT(*) > 1 + ) + UPDATE llm_usage_daily AS u SET + prompt_tokens = d.prompt_tokens, + completion_tokens = d.completion_tokens, + cache_read_tokens = d.cache_read_tokens, + cache_creation_tokens = d.cache_creation_tokens, + reasoning_tokens = d.reasoning_tokens, + audio_seconds = d.audio_seconds, + {tts_set} + call_count = d.call_count, + updated_at = now() + FROM dupes AS d + WHERE u.id = d.keep_id + """ + ) + ) + result = db.execute( + text( + f""" + WITH keepers AS ( + SELECT MIN(id::text)::uuid AS keep_id + FROM llm_usage_daily + GROUP BY + organization_id, + COALESCE(workspace_id, '{_ZERO_UUID}'::uuid), + product_section, + model, + usage_date, + COALESCE(usage_kind, 'llm'), + COALESCE(context->>'resource_id', ''), + COALESCE(context->>'resource_type', '') + HAVING COUNT(*) > 1 + ) + DELETE FROM llm_usage_daily AS u + USING keepers AS k, + llm_usage_daily AS peer + WHERE peer.id = k.keep_id + AND u.id <> k.keep_id + AND u.organization_id = peer.organization_id + AND u.product_section = peer.product_section + AND u.model = peer.model + AND u.usage_date = peer.usage_date + AND COALESCE(u.usage_kind, 'llm') = COALESCE(peer.usage_kind, 'llm') + AND u.workspace_id IS NOT DISTINCT FROM peer.workspace_id + AND COALESCE(u.context->>'resource_id', '') = + COALESCE(peer.context->>'resource_id', '') + AND COALESCE(u.context->>'resource_type', '') = + COALESCE(peer.context->>'resource_type', '') + """ + ) + ) + return int(result.rowcount or 0) + + +def _dedupe_by_full_context(db: Session) -> int: + tts_col = ( + "SUM(COALESCE(tts_characters, 0))::bigint AS tts_characters," + if _column_exists(db, "llm_usage_daily", "tts_characters") + else "" + ) + tts_set = ( + "tts_characters = d.tts_characters," + if _column_exists(db, "llm_usage_daily", "tts_characters") + else "" + ) + db.execute( + text( + f""" + WITH dupes AS ( + SELECT + organization_id, + COALESCE(workspace_id, '{_ZERO_UUID}'::uuid) AS ws_key, + product_section, + model, + usage_date, + COALESCE(usage_kind, 'llm') AS kind_key, + context, + MIN(id::text)::uuid AS keep_id, + SUM(prompt_tokens)::bigint AS prompt_tokens, + SUM(completion_tokens)::bigint AS completion_tokens, + SUM(cache_read_tokens)::bigint AS cache_read_tokens, + SUM(cache_creation_tokens)::bigint AS cache_creation_tokens, + SUM(reasoning_tokens)::bigint AS reasoning_tokens, + SUM(COALESCE(audio_seconds, 0))::bigint AS audio_seconds, + {tts_col} + SUM(call_count)::bigint AS call_count + FROM llm_usage_daily + GROUP BY 1, 2, 3, 4, 5, 6, 7 + HAVING COUNT(*) > 1 + ) + UPDATE llm_usage_daily AS u SET + prompt_tokens = d.prompt_tokens, + completion_tokens = d.completion_tokens, + cache_read_tokens = d.cache_read_tokens, + cache_creation_tokens = d.cache_creation_tokens, + reasoning_tokens = d.reasoning_tokens, + audio_seconds = d.audio_seconds, + {tts_set} + call_count = d.call_count, + updated_at = now() + FROM dupes AS d + WHERE u.id = d.keep_id + """ + ) + ) + result = db.execute( + text( + f""" + WITH keepers AS ( + SELECT MIN(id::text)::uuid AS keep_id + FROM llm_usage_daily + GROUP BY + organization_id, + COALESCE(workspace_id, '{_ZERO_UUID}'::uuid), + product_section, + model, + usage_date, + COALESCE(usage_kind, 'llm'), + context + HAVING COUNT(*) > 1 + ) + DELETE FROM llm_usage_daily AS u + USING keepers AS k, + llm_usage_daily AS peer + WHERE peer.id = k.keep_id + AND u.id <> k.keep_id + AND u.organization_id = peer.organization_id + AND u.product_section = peer.product_section + AND u.model = peer.model + AND u.usage_date = peer.usage_date + AND COALESCE(u.usage_kind, 'llm') = COALESCE(peer.usage_kind, 'llm') + AND u.workspace_id IS NOT DISTINCT FROM peer.workspace_id + AND u.context IS NOT DISTINCT FROM peer.context + """ + ) + ) + return int(result.rowcount or 0) + + +def _ensure_context_bucket_index(db: Session) -> None: + db.execute(text("DROP INDEX IF EXISTS uq_llm_usage_daily_bucket")) + db.execute( + text( + f""" + CREATE UNIQUE INDEX uq_llm_usage_daily_bucket + ON llm_usage_daily ( + organization_id, + COALESCE(workspace_id, '{_ZERO_UUID}'::uuid), + product_section, + model, + usage_date, + usage_kind, + context + ) + """ + ) + ) + + +def upgrade(db: Session): + if not _table_exists(db, "llm_usage_daily"): + print("llm_usage_daily missing; skipping 067") + db.commit() + return + if not _column_exists(db, "llm_usage_daily", "context"): + print("llm_usage_daily.context missing; run 065 first — skipping 067") + db.commit() + return + + legacy_removed = _dedupe_by_context_resource_keys(db) + if legacy_removed: + print(f"Merged {legacy_removed} legacy resource-scoped duplicate row(s)") + + context_removed = _dedupe_by_full_context(db) + if context_removed: + print(f"Merged {context_removed} duplicate full-context row(s)") + + _ensure_context_bucket_index(db) + print("Recreated uq_llm_usage_daily_bucket on full context JSONB") + db.commit() + + +def downgrade(db: Session): + db.commit() diff --git a/app/models/database.py b/app/models/database.py index 9f32f212..00b99fdb 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -19,7 +19,7 @@ select, text, ) -from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.orm import relationship from sqlalchemy.sql import func import uuid @@ -2780,7 +2780,7 @@ class JudgeRun(Base): class LLMUsageDaily(Base): - """Daily LLM usage rollups for org-scoped Usage reporting.""" + """Daily LLM/STT usage rollups for org-scoped Usage reporting.""" __tablename__ = "llm_usage_daily" @@ -2799,9 +2799,9 @@ class LLMUsageDaily(Base): ) product_section = Column(String(64), nullable=False, index=True) model = Column(String(255), nullable=False, index=True) - resource_id = Column(UUID(as_uuid=True), nullable=True, index=True) - resource_type = Column(String(64), nullable=True) + context = Column(JSONB, nullable=False, server_default="{}", default=dict) usage_date = Column(Date, nullable=False, index=True) + usage_kind = Column(String(16), nullable=False, default="llm", server_default="llm") prompt_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") completion_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") cache_read_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") @@ -2809,6 +2809,8 @@ class LLMUsageDaily(Base): BigInteger, nullable=False, default=0, server_default="0" ) reasoning_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") + audio_seconds = Column(BigInteger, nullable=False, default=0, server_default="0") + tts_characters = Column(BigInteger, nullable=False, default=0, server_default="0") call_count = Column(BigInteger, nullable=False, default=0, server_default="0") created_at = Column(DateTime(timezone=True), server_default=func.now()) updated_at = Column( diff --git a/app/services/ai/llm_gateway.py b/app/services/ai/llm_gateway.py index 750b1641..4b1a08dc 100644 --- a/app/services/ai/llm_gateway.py +++ b/app/services/ai/llm_gateway.py @@ -10,6 +10,7 @@ from __future__ import annotations import os +from contextlib import contextmanager from dataclasses import dataclass from typing import Any, Dict, Literal, Optional, Tuple from urllib.parse import urlparse @@ -723,10 +724,63 @@ def litellm_completion( model_name = str(kwargs.get("model") or "unknown") if "/" in model_name: model_name = model_name.rsplit("/", 1)[-1] - record_llm_usage(model_name, normalize_llm_usage(raw_response=response)) + record_llm_usage( + model_name, + normalize_llm_usage(raw_response=response), + organization_id=organization_id, + ) finally: if usage_token is not None: reset_usage_context(usage_token) except Exception as exc: logger.debug("litellm_completion usage record skipped: {}", exc) return response + + +@contextmanager +def litellm_batch_completion_recording( + *, + organization_id: UUID, + db: Session, + model: Optional[str] = None, + credential: Optional[CredentialRoutingContext] = None, +): + """Temporarily wrap litellm.batch_completion to record usage for each response.""" + import litellm + + from app.services.usage.llm_usage import record_llm_usage + from app.services.usage.normalize import normalize_llm_usage + + original = litellm.batch_completion + + def _recording_batch(**kwargs: Any): + kwargs = apply_llm_gateway( + kwargs, + organization_id=organization_id, + db=db, + model=model or kwargs.get("model"), + credential=credential, + ) + responses = original(**kwargs) + model_name = str(kwargs.get("model") or model or "unknown") + if "/" in model_name: + model_name = model_name.rsplit("/", 1)[-1] + items = responses if isinstance(responses, list) else [responses] + for resp in items: + if resp is None: + continue + try: + record_llm_usage( + model_name, + normalize_llm_usage(raw_response=resp), + organization_id=organization_id, + ) + except Exception as exc: + logger.debug("litellm batch usage record skipped: {}", exc) + return responses + + litellm.batch_completion = _recording_batch + try: + yield + finally: + litellm.batch_completion = original diff --git a/app/services/ai/llm_service.py b/app/services/ai/llm_service.py index a0b2090a..4bf5b61f 100644 --- a/app/services/ai/llm_service.py +++ b/app/services/ai/llm_service.py @@ -567,7 +567,9 @@ def generate_response( result["usage"]["cache_read_tokens"] = snapshot.cache_read_tokens result["usage"]["cache_creation_tokens"] = snapshot.cache_creation_tokens result["usage"]["reasoning_tokens"] = snapshot.reasoning_tokens - record_llm_usage(llm_model, snapshot) + record_llm_usage( + llm_model, snapshot, organization_id=organization_id + ) finally: if usage_token is not None: reset_usage_context(usage_token) diff --git a/app/services/ai/stt_clients/google.py b/app/services/ai/stt_clients/google.py index 6d86ec30..2bc100a1 100644 --- a/app/services/ai/stt_clients/google.py +++ b/app/services/ai/stt_clients/google.py @@ -152,6 +152,30 @@ def transcribe_google( f"Gemini transcription failed for {litellm_model}: {e}" ) + if organization_id is not None: + try: + from app.services.usage.llm_usage import record_llm_usage + from app.services.usage.normalize import normalize_llm_usage + + gemini_model = _strip_stt_suffix(model) + record_llm_usage( + gemini_model, + normalize_llm_usage(raw_response=response), + organization_id=organization_id, + ) + from app.services.usage.llm_usage import probe_audio_seconds, record_stt_usage + + audio_seconds = probe_audio_seconds(audio_file_path) + if audio_seconds > 0: + record_stt_usage( + gemini_model, + audio_seconds=audio_seconds, + organization_id=organization_id, + count_call=False, + ) + except Exception as exc: + logger.debug("[transcribe_google] llm usage record skipped: %s", exc) + text = "" try: text = (response.choices[0].message.content or "").strip() diff --git a/app/services/ai/transcription_service.py b/app/services/ai/transcription_service.py index fbe87c7c..9620e1ca 100644 --- a/app/services/ai/transcription_service.py +++ b/app/services/ai/transcription_service.py @@ -579,6 +579,24 @@ def transcribe_text_only( return None text = (result.get("text") or "").strip() + try: + from app.services.usage.llm_usage import ( + probe_audio_seconds, + record_stt_usage, + ) + + audio_seconds = probe_audio_seconds(audio_file_path) + if stt_provider != ModelProvider.GOOGLE: + record_stt_usage( + stt_model or "unknown", + audio_seconds=audio_seconds, + organization_id=organization_id, + ) + except Exception as exc: + logger.debug( + "[TranscriptionService] text-only stt usage record skipped: %s", + exc, + ) return text or None except Exception as e: logger.error(f"[TranscriptionService] text-only transcription failed ({stt_provider}/{stt_model}): {e}") @@ -661,6 +679,33 @@ def transcribe( else: result = self._transcribe_with_whisper_local(temp_file_path, "base") + try: + from app.services.usage.llm_usage import ( + probe_audio_seconds, + record_stt_usage, + ) + + audio_seconds = 0 + if isinstance(result, dict): + raw_duration = result.get("duration") + if raw_duration is not None: + try: + audio_seconds = int(float(raw_duration)) + except (TypeError, ValueError): + audio_seconds = 0 + if audio_seconds <= 0 and temp_file_path: + audio_seconds = probe_audio_seconds(temp_file_path) + if stt_provider != ModelProvider.GOOGLE: + record_stt_usage( + stt_model or "unknown", + audio_seconds=audio_seconds, + organization_id=organization_id, + ) + except Exception as exc: + logger.debug( + "[TranscriptionService] stt usage record skipped: %s", exc + ) + # Apply speaker diarization if enabled speaker_segments = None if enable_speaker_diarization: diff --git a/app/services/ai/tts_service.py b/app/services/ai/tts_service.py index 640c682c..4d1162f4 100644 --- a/app/services/ai/tts_service.py +++ b/app/services/ai/tts_service.py @@ -262,6 +262,11 @@ def synthesize( api_key = self._get_api_key_for_provider(tts_provider, db, organization_id) handler = self._get_tts_handler(tts_provider) audio_bytes, _ttfb_ms = handler(text, tts_model, api_key, voice, config) + self._record_tts_usage( + text=text, + tts_model=tts_model, + organization_id=organization_id, + ) return audio_bytes def synthesize_timed( @@ -280,8 +285,40 @@ def synthesize_timed( start = time.time() audio_bytes, ttfb_ms = handler(text, tts_model, api_key, voice, config) total_latency_ms = (time.time() - start) * 1000 + self._record_tts_usage( + text=text, + tts_model=tts_model, + organization_id=organization_id, + ) return audio_bytes, total_latency_ms, ttfb_ms + def _record_tts_usage( + self, + *, + text: str, + tts_model: str, + organization_id: UUID, + ) -> None: + try: + from app.services.usage.context import ( + ensure_usage_context, + reset_usage_context, + ) + from app.services.usage.llm_usage import record_tts_usage + + usage_token = ensure_usage_context(organization_id) + try: + record_tts_usage( + tts_model, + characters=len(text or ""), + organization_id=organization_id, + ) + finally: + if usage_token is not None: + reset_usage_context(usage_token) + except Exception: + pass + def synthesize_and_upload( self, text: str, diff --git a/app/services/evaluators/call_data_transcript.py b/app/services/evaluators/call_data_transcript.py new file mode 100644 index 00000000..4c7afe2e --- /dev/null +++ b/app/services/evaluators/call_data_transcript.py @@ -0,0 +1,195 @@ +"""Extract transcript text from voice-provider call payloads.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Tuple + + +def extract_transcript_from_call_data( + call_data: Dict[str, Any], + provider_platform: str, +) -> Tuple[str, List[dict]]: + """Return plain-text transcript and speaker segments from provider call_data.""" + transcript_text = "" + speaker_segments: List[dict] = [] + + if not call_data: + return transcript_text, speaker_segments + + provider_platform_lower = provider_platform.lower() if provider_platform else "" + + if provider_platform_lower == "vapi": + transcript_text = call_data.get("transcript", "") or "" + transcript_object = call_data.get("transcript_object", []) + if not transcript_object: + artifact = call_data.get("artifact", {}) if isinstance(call_data, dict) else {} + messages = call_data.get("messages", []) or artifact.get("messages", []) + for msg in messages: + role = msg.get("role", "unknown") + content = msg.get("message", "") or msg.get("content", "") + if not content or role == "system": + continue + if role in ("bot", "assistant"): + normalized_role = "agent" + elif role == "user": + normalized_role = "user" + else: + continue + speaker_segments.append( + { + "speaker": "Agent" if normalized_role == "agent" else "User", + "text": content, + "start": msg.get("secondsFromStart", 0), + "end": msg.get("secondsFromStart", 0) + + (msg.get("duration", 0) / 1000), + } + ) + else: + for entry in transcript_object: + role = entry.get("role", "unknown") + content = entry.get("content", "") + if not content: + continue + speaker_segments.append( + { + "speaker": "Agent" if role == "agent" else "User", + "text": content, + "start": entry.get("seconds_from_start", 0), + "end": entry.get("seconds_from_start", 0) + + (entry.get("duration_ms", 0) / 1000), + } + ) + if not transcript_text and speaker_segments: + transcript_text = "\n".join( + f"{seg['speaker']}: {seg['text']}" for seg in speaker_segments + ) + + elif provider_platform_lower == "elevenlabs": + raw_transcript = call_data.get("transcript") + transcript_obj = call_data.get("transcript_object", []) + if isinstance(raw_transcript, str) and raw_transcript: + transcript_text = raw_transcript + if isinstance(transcript_obj, list): + for seg in transcript_obj: + speaker_segments.append( + { + "speaker": seg.get("speaker", "Unknown"), + "text": seg.get("text", ""), + "start": seg.get("start", 0), + "end": seg.get("end", 0), + } + ) + elif isinstance(raw_transcript, list): + for entry in raw_transcript: + role = entry.get("role", "unknown") + content = entry.get("message", "") or entry.get("text", "") + if not content: + continue + speaker = "Agent" if role in ("agent", "assistant", "ai") else "User" + speaker_segments.append( + { + "speaker": speaker, + "text": content, + "start": entry.get("time_in_call_secs", 0) or entry.get("start", 0), + "end": entry.get("time_in_call_secs", 0) or entry.get("end", 0), + } + ) + transcript_text = "\n".join( + f"{seg['speaker']}: {seg['text']}" for seg in speaker_segments + ) + + elif provider_platform_lower == "smallest": + transcript_raw = call_data.get("transcript") + transcript_object = call_data.get("transcript_object", []) + if isinstance(transcript_object, list) and transcript_object: + for entry in transcript_object: + if not isinstance(entry, dict): + continue + text = entry.get("text", "") + if not text: + continue + speaker_segments.append( + { + "speaker": entry.get("speaker", "Unknown"), + "text": text, + "start": entry.get("start", 0), + "end": entry.get("end", entry.get("start", 0)), + } + ) + if not transcript_text: + transcript_text = "\n".join( + f"{seg['speaker']}: {seg['text']}" for seg in speaker_segments + ) + elif isinstance(transcript_raw, list): + for entry in transcript_raw: + if not isinstance(entry, dict): + continue + role = str(entry.get("speaker") or entry.get("role") or "").lower() + speaker = "Agent" if role in ("agent", "assistant", "ai", "bot") else "User" + text = entry.get("text", "") or entry.get("message", "") or entry.get("content", "") + if not text: + continue + ts = entry.get("timeInCallSecs", 0) or entry.get("start", 0) or entry.get("timestamp", 0) + speaker_segments.append( + { + "speaker": speaker, + "text": text, + "start": ts, + "end": entry.get("end", ts), + } + ) + transcript_text = "\n".join( + f"{seg['speaker']}: {seg['text']}" for seg in speaker_segments + ) + elif isinstance(transcript_raw, str): + transcript_text = transcript_raw + + elif provider_platform_lower == "retell": + transcript_raw = call_data.get("transcript", "") + if isinstance(transcript_raw, str): + transcript_text = transcript_raw + lines = transcript_raw.split("\n") if transcript_raw else [] + for line in lines: + line = line.strip() + if not line: + continue + if line.startswith("Agent:") or line.startswith("agent:"): + speaker_segments.append( + { + "speaker": "Agent", + "text": line.split(":", 1)[1].strip() if ":" in line else line, + "start": 0, + "end": 0, + } + ) + elif line.startswith("User:") or line.startswith("user:"): + speaker_segments.append( + { + "speaker": "User", + "text": line.split(":", 1)[1].strip() if ":" in line else line, + "start": 0, + "end": 0, + } + ) + elif isinstance(transcript_raw, list): + for item in transcript_raw: + if not isinstance(item, dict): + continue + role = item.get("role", "") + content = item.get("content", "") or item.get("text", "") + if not content: + continue + speaker = "Agent" if role in ["agent", "assistant", "bot"] else "User" + speaker_segments.append( + { + "speaker": speaker, + "text": content, + "start": item.get("start_time", 0) or item.get("timestamp", 0), + "end": item.get("end_time", 0), + } + ) + transcript_text = "\n".join( + f"{seg['speaker']}: {seg['text']}" for seg in speaker_segments + ) + + return transcript_text, speaker_segments diff --git a/app/services/judge_alignment/gepa_bridge.py b/app/services/judge_alignment/gepa_bridge.py index 37926fa1..6a12506b 100644 --- a/app/services/judge_alignment/gepa_bridge.py +++ b/app/services/judge_alignment/gepa_bridge.py @@ -219,6 +219,31 @@ def execute_judge_gepa(run_id: str, db: Session) -> Dict[str, Any]: if not evaluator: raise RuntimeError("Evaluator vanished between dispatch and execution") + from app.services.usage.context import ( + reset_usage_context, + set_usage_context, + usage_context_for_prompt_optimization_run, + ) + + usage_token = set_usage_context(usage_context_for_prompt_optimization_run(run)) + try: + return _execute_judge_gepa_with_context( + run=run, + cfg=cfg, + evaluator=evaluator, + db=db, + ) + finally: + reset_usage_context(usage_token) + + +def _execute_judge_gepa_with_context( + *, + run: PromptOptimizationRun, + cfg: Dict[str, Any], + evaluator: Evaluator, + db: Session, +) -> Dict[str, Any]: dev_ids: List[str] = cfg.get("dev_sample_ids", []) if not dev_ids: raise RuntimeError("Optimisation run has no dev_sample_ids in config") @@ -370,19 +395,27 @@ def reflection_lm(prompt: str) -> str: resp = litellm_completion(**reflection_kwargs, credential=credential_ctx) return resp.choices[0].message.content + from app.services.ai.llm_gateway import litellm_batch_completion_recording + run.status = PromptOptimizationStatus.RUNNING.value db.commit() try: - result = gepa_optimize( - seed_candidate={"system_prompt": evaluator.custom_prompt}, - trainset=trainset, - adapter=adapter, - reflection_lm=reflection_lm, - max_metric_calls=int(cfg.get("max_metric_calls", 20)), - reflection_minibatch_size=1, - candidate_selection_strategy="pareto", - ) + with litellm_batch_completion_recording( + organization_id=run.organization_id, + db=db, + model=lm_identifier, + credential=credential_ctx, + ): + result = gepa_optimize( + seed_candidate={"system_prompt": evaluator.custom_prompt}, + trainset=trainset, + adapter=adapter, + reflection_lm=reflection_lm, + max_metric_calls=int(cfg.get("max_metric_calls", 20)), + reflection_minibatch_size=1, + candidate_selection_strategy="pareto", + ) except Exception as exc: run.status = PromptOptimizationStatus.FAILED.value run.error_message = str(exc) diff --git a/app/services/optimization/gepa_service.py b/app/services/optimization/gepa_service.py index 5328dba9..d92895ac 100644 --- a/app/services/optimization/gepa_service.py +++ b/app/services/optimization/gepa_service.py @@ -149,15 +149,23 @@ def reflection_lm(prompt: str) -> str: resp = litellm_completion(**reflection_kwargs, credential=credential_ctx) return resp.choices[0].message.content - result = gepa_optimize( - seed_candidate={"system_prompt": seed_prompt}, - trainset=trainset, - adapter=adapter, - reflection_lm=reflection_lm, - max_metric_calls=max_metric_calls, - reflection_minibatch_size=min(minibatch_size, len(trainset)), - candidate_selection_strategy="pareto", - ) + from app.services.ai.llm_gateway import litellm_batch_completion_recording + + with litellm_batch_completion_recording( + organization_id=organization_id, + db=db, + model=lm_identifier, + credential=credential_ctx, + ): + result = gepa_optimize( + seed_candidate={"system_prompt": seed_prompt}, + trainset=trainset, + adapter=adapter, + reflection_lm=reflection_lm, + max_metric_calls=max_metric_calls, + reflection_minibatch_size=min(minibatch_size, len(trainset)), + candidate_selection_strategy="pareto", + ) return _format_result(result, seed_prompt) diff --git a/app/services/testing/test_agent_bridge_service.py b/app/services/testing/test_agent_bridge_service.py index f8e8d41a..4a05cdb0 100644 --- a/app/services/testing/test_agent_bridge_service.py +++ b/app/services/testing/test_agent_bridge_service.py @@ -613,6 +613,8 @@ def resolve_api_key_for_provider( ) test_agent_config = TestAgentConfig( + organization_id=organization_id, + workspace_id=getattr(agent, "workspace_id", None), agent_name=agent.name or "Voice AI Agent", agent_description=agent.description or "A voice AI assistant", test_agent_simulation_prompt=simulation_prompt, diff --git a/app/services/usage/__init__.py b/app/services/usage/__init__.py index 646e9adb..4ebc75ca 100644 --- a/app/services/usage/__init__.py +++ b/app/services/usage/__init__.py @@ -1,4 +1,4 @@ -"""LLM usage tracking (tokens, calls) with Redis buffer and catalog rollups.""" +"""LLM/STT/TTS usage tracking with Redis buffer, PG fallback, and catalog rollups.""" from app.services.usage.context import ( LLMUsageContext, @@ -10,13 +10,27 @@ reset_usage_hints, set_usage_context, set_usage_hints, + usage_context_for_judge_run, + usage_context_for_prompt_optimization_run, + usage_context_for_prompt_partial, +) +from app.services.usage.llm_usage import ( + flush_all_usage_to_catalog, + flush_usage_to_catalog, + probe_audio_seconds, + record_call_usage, + record_llm_usage, + record_stt_usage, + record_tts_usage, ) -from app.services.usage.llm_usage import flush_all_usage_to_catalog, flush_usage_to_catalog, record_llm_usage from app.services.usage.normalize import UsageSnapshot, normalize_llm_usage __all__ = [ "LLMUsageContext", "LLMUsageProductSection", + "usage_context_for_judge_run", + "usage_context_for_prompt_optimization_run", + "usage_context_for_prompt_partial", "UsageSnapshot", "ensure_usage_context", "flush_all_usage_to_catalog", @@ -24,7 +38,11 @@ "infer_product_section_from_path", "llm_usage_context", "normalize_llm_usage", + "probe_audio_seconds", + "record_call_usage", "record_llm_usage", + "record_stt_usage", + "record_tts_usage", "reset_usage_context", "reset_usage_hints", "set_usage_context", diff --git a/app/services/usage/bucket_context.py b/app/services/usage/bucket_context.py new file mode 100644 index 00000000..ad6202d9 --- /dev/null +++ b/app/services/usage/bucket_context.py @@ -0,0 +1,114 @@ +"""Canonical usage bucket context (JSONB) helpers.""" + +from __future__ import annotations + +import json +from typing import Any, Dict, Optional +from uuid import UUID + +_NONE = "__none__" + +# Stable keys stored in llm_usage_daily.context (extend without new columns). +KNOWN_CONTEXT_KEYS = frozenset( + { + "resource_id", + "resource_type", + "call_import_id", + "evaluation_id", + "evaluation_row_id", + "call_import_row_id", + "credential_id", + "agent_id", + "job_id", + "user_id", + "trace_id", + } +) + +# Never store roll-up counters in JSONB — they stay as BIGINT columns for SUM(). +_FORBIDDEN_CONTEXT_KEYS = frozenset( + { + "prompt_tokens", + "completion_tokens", + "cache_read_tokens", + "cache_creation_tokens", + "reasoning_tokens", + "audio_seconds", + "tts_characters", + "call_count", + "total_tokens", + } +) + + +def build_bucket_context( + *, + resource_id: Optional[UUID] = None, + resource_type: Optional[str] = None, + extra: Optional[Dict[str, Any]] = None, +) -> Dict[str, str]: + """Build a normalized string-keyed context dict for rollup buckets.""" + ctx: Dict[str, str] = {} + if resource_id is not None: + ctx["resource_id"] = str(resource_id) + if resource_type: + ctx["resource_type"] = resource_type + if extra: + for key, value in extra.items(): + if value is None or key in ctx: + continue + if key in _FORBIDDEN_CONTEXT_KEYS: + continue + ctx[key] = str(value) + return ctx + + +def context_bucket_token(context: Optional[Dict[str, Any]]) -> str: + """Stable Redis bucket token for a context dict.""" + if not context: + return _NONE + normalized = {k: str(v) for k, v in sorted(context.items()) if v is not None} + if not normalized: + return _NONE + return json.dumps(normalized, sort_keys=True, separators=(",", ":")) + + +def parse_context_bucket_token(token: str) -> Dict[str, str]: + if not token or token == _NONE: + return {} + try: + parsed = json.loads(token) + except json.JSONDecodeError: + return {} + if not isinstance(parsed, dict): + return {} + return {str(k): str(v) for k, v in parsed.items() if v is not None} + + +def legacy_resource_context( + resource_id: Optional[UUID], + resource_type: Optional[str], +) -> Dict[str, str]: + return build_bucket_context( + resource_id=resource_id, + resource_type=resource_type, + ) + + +def resource_id_from_context(context: Optional[Dict[str, Any]]) -> Optional[UUID]: + if not context: + return None + raw = context.get("resource_id") + if not raw: + return None + try: + return UUID(str(raw)) + except (ValueError, TypeError): + return None + + +def resource_type_from_context(context: Optional[Dict[str, Any]]) -> Optional[str]: + if not context: + return None + raw = context.get("resource_type") + return str(raw) if raw else None diff --git a/app/services/usage/call_import_context.py b/app/services/usage/call_import_context.py new file mode 100644 index 00000000..e23f46cf --- /dev/null +++ b/app/services/usage/call_import_context.py @@ -0,0 +1,200 @@ +"""Usage context builders for call-import evaluation pipelines.""" + +from __future__ import annotations + +from typing import Any, Optional +from uuid import UUID + +from app.services.usage.context import LLMUsageContext, LLMUsageProductSection + + +def _parse_uuid(raw: Any) -> Optional[UUID]: + try: + return UUID(str(raw)) + except (TypeError, ValueError): + return None + + +def call_import_ids_from_usage_context(ctx: LLMUsageContext) -> dict[str, Optional[UUID]]: + """Extract call-import linkage ids from a usage context (no DB).""" + extra = ctx.extra or {} + ids: dict[str, Optional[UUID]] = { + "call_import_id": _parse_uuid(extra.get("call_import_id")), + "evaluation_id": _parse_uuid(extra.get("evaluation_id")), + "call_import_row_id": _parse_uuid(extra.get("call_import_row_id")), + "evaluation_row_id": _parse_uuid(extra.get("evaluation_row_id")), + } + if ctx.resource_type == "call_import" and ctx.resource_id: + ids["call_import_id"] = ids["call_import_id"] or ctx.resource_id + if ctx.resource_type == "call_import_evaluation" and ctx.resource_id: + ids["evaluation_id"] = ids["evaluation_id"] or ctx.resource_id + return ids + + +def resolve_workspace_id_for_usage_context(ctx: LLMUsageContext) -> Optional[UUID]: + """Resolve workspace from call-import entities when context omitted workspace_id.""" + if ctx.workspace_id is not None: + return ctx.workspace_id + + ids = call_import_ids_from_usage_context(ctx) + if not any(ids.values()): + return None + + from app.database import SessionLocal + from app.models.database import ( + CallImport, + CallImportEvaluation, + CallImportEvaluationRow, + CallImportRow, + ) + + org_id = ctx.organization_id + db = SessionLocal() + try: + if ids["call_import_id"]: + ws = ( + db.query(CallImport.workspace_id) + .filter( + CallImport.id == ids["call_import_id"], + CallImport.organization_id == org_id, + ) + .scalar() + ) + if ws: + return ws + + if ids["evaluation_id"]: + ws = ( + db.query(CallImportEvaluation.workspace_id) + .filter( + CallImportEvaluation.id == ids["evaluation_id"], + CallImportEvaluation.organization_id == org_id, + ) + .scalar() + ) + if ws: + return ws + + if ids["call_import_row_id"]: + ws = ( + db.query(CallImportRow.workspace_id) + .filter( + CallImportRow.id == ids["call_import_row_id"], + CallImportRow.organization_id == org_id, + ) + .scalar() + ) + if ws: + return ws + + if ids["evaluation_row_id"]: + ws = ( + db.query(CallImportEvaluation.workspace_id) + .join( + CallImportEvaluationRow, + CallImportEvaluationRow.evaluation_id == CallImportEvaluation.id, + ) + .filter( + CallImportEvaluationRow.id == ids["evaluation_row_id"], + CallImportEvaluation.organization_id == org_id, + ) + .scalar() + ) + if ws: + return ws + + return None + finally: + db.close() + + +def enrich_usage_context_workspace(ctx: LLMUsageContext) -> LLMUsageContext: + """Fill workspace_id from call-import linkage when missing at record time.""" + if ctx.workspace_id is not None: + return ctx + try: + resolved = resolve_workspace_id_for_usage_context(ctx) + except Exception: + return ctx + if resolved is None: + return ctx + return LLMUsageContext( + organization_id=ctx.organization_id, + workspace_id=resolved, + product_section=ctx.product_section, + resource_id=ctx.resource_id, + resource_type=ctx.resource_type, + extra=ctx.extra, + ) + + +def call_import_evaluation_usage_context( + *, + organization_id: UUID, + workspace_id: Optional[UUID], + evaluation_id: UUID, + call_import_id: UUID, + evaluation_row_id: Optional[UUID] = None, + call_import_row_id: Optional[UUID] = None, +) -> LLMUsageContext: + """Full attribution for eval-run LLM/STT/TTS inside a call import evaluation.""" + extra: dict[str, str] = { + "call_import_id": str(call_import_id), + "evaluation_id": str(evaluation_id), + } + if evaluation_row_id is not None: + extra["evaluation_row_id"] = str(evaluation_row_id) + if call_import_row_id is not None: + extra["call_import_row_id"] = str(call_import_row_id) + return LLMUsageContext( + organization_id=organization_id, + workspace_id=workspace_id, + product_section=LLMUsageProductSection.CALL_IMPORT_EVALUATIONS, + resource_id=evaluation_id, + resource_type="call_import_evaluation", + extra=extra, + ) + + +def call_import_row_usage_context( + *, + organization_id: UUID, + workspace_id: Optional[UUID], + call_import_id: UUID, + call_import_row_id: UUID, + evaluation_id: Optional[UUID] = None, + evaluation_row_id: Optional[UUID] = None, +) -> LLMUsageContext: + """Attribution for diarisation / STT on a single call-import row.""" + if evaluation_id is not None: + return call_import_evaluation_usage_context( + organization_id=organization_id, + workspace_id=workspace_id, + evaluation_id=evaluation_id, + call_import_id=call_import_id, + evaluation_row_id=evaluation_row_id, + call_import_row_id=call_import_row_id, + ) + return LLMUsageContext( + organization_id=organization_id, + workspace_id=workspace_id, + product_section=LLMUsageProductSection.CALL_IMPORTS, + resource_id=call_import_id, + resource_type="call_import", + extra={ + "call_import_id": str(call_import_id), + "call_import_row_id": str(call_import_row_id), + }, + ) + + +def usage_context_for_evaluation( + evaluation: Any, +) -> LLMUsageContext: + """Usage context from a loaded CallImportEvaluation row.""" + return call_import_evaluation_usage_context( + organization_id=evaluation.organization_id, + workspace_id=evaluation.workspace_id, + evaluation_id=evaluation.id, + call_import_id=evaluation.call_import_id, + ) diff --git a/app/services/usage/context.py b/app/services/usage/context.py index 80e84fac..8bd895e1 100644 --- a/app/services/usage/context.py +++ b/app/services/usage/context.py @@ -6,7 +6,7 @@ from contextvars import ContextVar, Token from dataclasses import dataclass from enum import Enum -from typing import Iterator, Optional +from typing import Iterator, Optional, Any from uuid import UUID @@ -36,6 +36,7 @@ class LLMUsageContext: product_section: LLMUsageProductSection = LLMUsageProductSection.OTHER resource_id: Optional[UUID] = None resource_type: Optional[str] = None + extra: Optional[dict[str, str]] = None _usage_context_var: ContextVar[Optional[LLMUsageContext]] = ContextVar( @@ -128,6 +129,7 @@ def ensure_usage_context( product_section: LLMUsageProductSection = LLMUsageProductSection.OTHER, resource_id: Optional[UUID] = None, resource_type: Optional[str] = None, + extra: Optional[dict[str, str]] = None, ) -> Token | None: """Set or enrich usage context. Returns token to reset, or None if unchanged.""" resolved_workspace = workspace_id or get_usage_workspace_hint() @@ -146,6 +148,7 @@ def ensure_usage_context( product_section=resolved_section, resource_id=resource_id, resource_type=resource_type, + extra=extra, ) ) @@ -158,11 +161,13 @@ def ensure_usage_context( ) upgraded_resource_id = current.resource_id or resource_id upgraded_resource_type = current.resource_type or resource_type + upgraded_extra = {**(current.extra or {}), **(extra or {})} or None if ( upgraded_workspace == current.workspace_id and upgraded_section == current.product_section and upgraded_resource_id == current.resource_id and upgraded_resource_type == current.resource_type + and upgraded_extra == current.extra ): return None @@ -173,5 +178,109 @@ def ensure_usage_context( product_section=upgraded_section, resource_id=upgraded_resource_id, resource_type=upgraded_resource_type, + extra=upgraded_extra, ) ) + + +def usage_context_for_agent( + agent: Any, + *, + workspace_id: Optional[UUID] = None, + extra: Optional[dict[str, str]] = None, +) -> LLMUsageContext: + """Usage context for agent-scoped LLM work (simulations, setup, summaries).""" + merged: dict[str, str] = dict(extra or {}) + merged.setdefault("agent_id", str(agent.id)) + short = getattr(agent, "agent_id", None) + if short: + merged.setdefault("agent_short_id", str(short)) + return LLMUsageContext( + organization_id=agent.organization_id, + workspace_id=workspace_id or agent.workspace_id, + product_section=LLMUsageProductSection.AGENTS, + resource_id=agent.id, + resource_type="agent", + extra=merged, + ) + + +def usage_context_for_evaluator_result(result: Any) -> LLMUsageContext: + """Usage context for processing an evaluator result (Vapi / playground runs).""" + if result.agent_id: + return LLMUsageContext( + organization_id=result.organization_id, + workspace_id=result.workspace_id, + product_section=LLMUsageProductSection.AGENTS, + resource_id=result.agent_id, + resource_type="agent", + extra={"agent_id": str(result.agent_id)}, + ) + extra: dict[str, str] = {"evaluator_result_id": str(result.id)} + if getattr(result, "result_id", None): + extra["result_short_id"] = str(result.result_id) + if result.evaluator_id: + extra["evaluator_id"] = str(result.evaluator_id) + return LLMUsageContext( + organization_id=result.organization_id, + workspace_id=result.workspace_id, + product_section=LLMUsageProductSection.EVALUATORS, + resource_id=result.id, + resource_type="evaluator_result", + extra=extra, + ) + + +def usage_context_for_prompt_optimization_run(run: Any) -> LLMUsageContext: + """Usage context for a GEPA prompt optimization run.""" + cfg = run.config if isinstance(run.config, dict) else {} + is_judge = cfg.get("source") == "judge_alignment" + extra: dict[str, str] = { + "optimization_run_id": str(run.id), + "agent_id": str(run.agent_id), + } + if run.evaluator_id: + extra["evaluator_id"] = str(run.evaluator_id) + if is_judge: + extra["source"] = "judge_alignment" + if cfg.get("judge_dataset_id"): + extra["judge_dataset_id"] = str(cfg["judge_dataset_id"]) + return LLMUsageContext( + organization_id=run.organization_id, + workspace_id=run.workspace_id, + product_section=( + LLMUsageProductSection.JUDGE_ALIGNMENT + if is_judge + else LLMUsageProductSection.PROMPT_OPTIMIZATION + ), + resource_id=run.agent_id, + resource_type="agent", + extra=extra, + ) + + +def usage_context_for_judge_run(run: Any) -> LLMUsageContext: + """Usage context for a judge alignment scoring run.""" + return LLMUsageContext( + organization_id=run.organization_id, + workspace_id=run.workspace_id, + product_section=LLMUsageProductSection.JUDGE_ALIGNMENT, + resource_id=run.evaluator_id, + resource_type="evaluator", + extra={ + "judge_run_id": str(run.id), + "judge_dataset_id": str(run.dataset_id), + }, + ) + + +def usage_context_for_prompt_partial(partial: Any) -> LLMUsageContext: + """Usage context for prompt partial / agent flowchart LLM work.""" + return LLMUsageContext( + organization_id=partial.organization_id, + workspace_id=partial.workspace_id, + product_section=LLMUsageProductSection.PROMPT_PARTIALS, + resource_id=partial.id, + resource_type="prompt_partial", + extra={"prompt_partial_id": str(partial.id)}, + ) diff --git a/app/services/usage/dates.py b/app/services/usage/dates.py new file mode 100644 index 00000000..340ba9eb --- /dev/null +++ b/app/services/usage/dates.py @@ -0,0 +1,38 @@ +"""Local calendar dates vs UTC usage_date bucket bounds.""" + +from __future__ import annotations + +from datetime import date, datetime, time, timedelta, timezone +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + + +def resolve_usage_timezone(tz: str | None) -> ZoneInfo: + if not tz: + return ZoneInfo("UTC") + try: + return ZoneInfo(tz) + except ZoneInfoNotFoundError: + return ZoneInfo("UTC") + + +def usage_local_today(tz: str | None) -> date: + return datetime.now(resolve_usage_timezone(tz)).date() + + +def usage_date_filter_bounds( + start: date, + end: date, + tz: str | None, +) -> tuple[date, date]: + """Map inclusive local calendar days to usage_date (UTC-day) filter bounds.""" + if not tz: + return start, end + + zone = resolve_usage_timezone(tz) + start_local = datetime.combine(start, time.min, tzinfo=zone) + end_exclusive = datetime.combine(end + timedelta(days=1), time.min, tzinfo=zone) + filter_start = start_local.astimezone(timezone.utc).date() + filter_end = ( + end_exclusive - timedelta(seconds=1) + ).astimezone(timezone.utc).date() + return filter_start, filter_end diff --git a/app/services/usage/llm_usage.py b/app/services/usage/llm_usage.py index 4c1fcecb..33fda254 100644 --- a/app/services/usage/llm_usage.py +++ b/app/services/usage/llm_usage.py @@ -1,9 +1,11 @@ -"""Redis-buffered LLM usage counters with catalog rollup flush.""" +"""Redis-buffered LLM/STT usage counters with catalog rollup flush.""" from __future__ import annotations +import math import time import uuid +import json from datetime import date, datetime, timezone from typing import Any, Dict, Iterable, List, Optional, Tuple from uuid import UUID @@ -11,11 +13,19 @@ import redis from loguru import logger from sqlalchemy import text +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from app.config import settings +from app.services.usage.bucket_context import ( + build_bucket_context, + context_bucket_token, + legacy_resource_context, + parse_context_bucket_token, +) from app.services.usage.context import ( LLMUsageContext, + LLMUsageProductSection, get_usage_context, ) from app.services.usage.normalize import UsageSnapshot @@ -26,16 +36,20 @@ _PENDING_TTL_SECONDS = 14 * 24 * 60 * 60 _FLUSH_LOCK_TTL_SECONDS = 45 _FLUSH_LOCK_WAIT_SECONDS = 3.0 +USAGE_KIND_LLM = "llm" +USAGE_KIND_STT = "stt" +USAGE_KIND_TTS = "tts" _METRIC_FIELDS = ( "prompt_tokens", "completion_tokens", "cache_read_tokens", "cache_creation_tokens", "reasoning_tokens", + "audio_seconds", + "tts_characters", "call_count", ) -# Atomically move pending hash → claim key so only one flusher owns the deltas. _CLAIM_LUA = """ if redis.call('EXISTS', KEYS[1]) == 0 then return 0 @@ -73,51 +87,106 @@ def _bucket_prefix( workspace_id: Optional[UUID], product_section: str, model: str, - resource_id: Optional[UUID], - resource_type: Optional[str], + context: Optional[Dict[str, Any]], usage_date: date, + usage_kind: str, ) -> str: return "|".join( [ _token(workspace_id), product_section, model, - _token(resource_id), - resource_type or _NONE, + context_bucket_token(context), usage_date.isoformat(), + usage_kind or USAGE_KIND_LLM, ] ) def _parse_bucket_prefix(prefix: str) -> Optional[Dict[str, Any]]: parts = prefix.split("|") - if len(parts) != 6: + # New: 6 parts with JSON context token + usage_kind. + if len(parts) == 6: + ws_token, section, model, context_token, day_str, usage_kind = parts + context = parse_context_bucket_token(context_token) + elif len(parts) == 7: + # Legacy Redis keys: resource_id + resource_type before date/kind. + ( + ws_token, + section, + model, + resource_token, + resource_type, + day_str, + usage_kind, + ) = parts + resource_id = None if resource_token == _NONE else UUID(resource_token) + resolved_resource_type = None if resource_type == _NONE else resource_type + context = legacy_resource_context(resource_id, resolved_resource_type) + elif len(parts) == 5: + ws_token, section, model, context_token, day_str = parts + usage_kind = USAGE_KIND_LLM + context = parse_context_bucket_token(context_token) + else: return None - ws_token, section, model, resource_token, resource_type, day_str = parts try: usage_date = date.fromisoformat(day_str) except ValueError: return None workspace_id = None if ws_token == _NONE else UUID(ws_token) - resource_id = None if resource_token == _NONE else UUID(resource_token) - resolved_resource_type = None if resource_type == _NONE else resource_type return { "workspace_id": workspace_id, "product_section": section, "model": model, - "resource_id": resource_id, - "resource_type": resolved_resource_type, + "context": context, "usage_date": usage_date, + "usage_kind": usage_kind or USAGE_KIND_LLM, } -def _resolve_context(ctx: Optional[LLMUsageContext]) -> LLMUsageContext: +def _bucket_from_context( + context: LLMUsageContext, + *, + model: str, + usage_date: date, + usage_kind: str, +) -> Dict[str, Any]: + return { + "workspace_id": context.workspace_id, + "product_section": context.product_section.value, + "model": model, + "context": build_bucket_context( + resource_id=context.resource_id, + resource_type=context.resource_type, + extra=context.extra, + ), + "usage_date": usage_date, + "usage_kind": usage_kind, + } + + +def _resolve_context( + *, + organization_id: Optional[UUID] = None, + ctx: Optional[LLMUsageContext] = None, +) -> Optional[LLMUsageContext]: if ctx is not None: return ctx current = get_usage_context() if current is not None: return current - raise ValueError("LLM usage context is not set") + if organization_id is not None: + from app.services.usage.context import ( + get_usage_section_hint, + get_usage_workspace_hint, + ) + + return LLMUsageContext( + organization_id=organization_id, + workspace_id=get_usage_workspace_hint(), + product_section=get_usage_section_hint(), + ) + return None def _deltas_from_usage(usage: UsageSnapshot) -> Dict[str, int]: @@ -127,24 +196,147 @@ def _deltas_from_usage(usage: UsageSnapshot) -> Dict[str, int]: "cache_read_tokens": usage.cache_read_tokens, "cache_creation_tokens": usage.cache_creation_tokens, "reasoning_tokens": usage.reasoning_tokens, + "audio_seconds": 0, + "tts_characters": 0, + "call_count": 1, + } + + +def _deltas_from_stt(audio_seconds: int, *, count_call: bool = True) -> Dict[str, int]: + return { + "prompt_tokens": 0, + "completion_tokens": 0, + "cache_read_tokens": 0, + "cache_creation_tokens": 0, + "reasoning_tokens": 0, + "audio_seconds": max(0, int(audio_seconds)), + "tts_characters": 0, + "call_count": 1 if count_call else 0, + } + + +def _deltas_from_tts(characters: int) -> Dict[str, int]: + return { + "prompt_tokens": 0, + "completion_tokens": 0, + "cache_read_tokens": 0, + "cache_creation_tokens": 0, + "reasoning_tokens": 0, + "audio_seconds": 0, + "tts_characters": max(0, int(characters)), "call_count": 1, } +def _buffer_to_postgres( + organization_id: UUID, + bucket: Dict[str, Any], + deltas: Dict[str, int], +) -> None: + """Durable fallback when Redis is unavailable.""" + try: + from app.database import SessionLocal + except Exception as exc: + logger.warning("usage postgres fallback unavailable: {}", exc) + return + + params = { + "organization_id": str(organization_id), + "workspace_id": str(bucket["workspace_id"]) if bucket.get("workspace_id") else None, + "product_section": bucket["product_section"], + "model": bucket["model"], + "context": json.dumps(bucket.get("context") or {}), + "usage_date": bucket["usage_date"].isoformat(), + "usage_kind": bucket.get("usage_kind") or USAGE_KIND_LLM, + "prompt_tokens": int(deltas.get("prompt_tokens", 0)), + "completion_tokens": int(deltas.get("completion_tokens", 0)), + "cache_read_tokens": int(deltas.get("cache_read_tokens", 0)), + "cache_creation_tokens": int(deltas.get("cache_creation_tokens", 0)), + "reasoning_tokens": int(deltas.get("reasoning_tokens", 0)), + "audio_seconds": int(deltas.get("audio_seconds", 0)), + "tts_characters": int(deltas.get("tts_characters", 0)), + "call_count": int(deltas.get("call_count", 0)), + } + db = SessionLocal() + try: + db.execute( + text( + """ + INSERT INTO usage_pending_buffer ( + id, organization_id, workspace_id, product_section, model, + context, usage_date, usage_kind, + prompt_tokens, completion_tokens, cache_read_tokens, + cache_creation_tokens, reasoning_tokens, audio_seconds, + tts_characters, call_count, created_at + ) VALUES ( + gen_random_uuid(), CAST(:organization_id AS uuid), + CAST(:workspace_id AS uuid), :product_section, :model, + CAST(:context AS jsonb), + CAST(:usage_date AS date), :usage_kind, + :prompt_tokens, :completion_tokens, :cache_read_tokens, + :cache_creation_tokens, :reasoning_tokens, :audio_seconds, + :tts_characters, :call_count, now() + ) + """ + ), + params, + ) + db.commit() + except Exception as exc: + db.rollback() + logger.warning("usage postgres fallback insert failed: {}", exc) + finally: + db.close() + + +def _incr_pending( + organization_id: UUID, + prefix: str, + deltas: Dict[str, int], + bucket: Dict[str, Any], +) -> None: + hash_key = _pending_hash_key(organization_id) + try: + client = _client() + pipe = client.pipeline() + for metric, delta in deltas.items(): + if delta: + pipe.hincrby(hash_key, f"{prefix}|{metric}", int(delta)) + pipe.sadd("usage:pending:orgs", str(organization_id)) + pipe.expire(hash_key, _PENDING_TTL_SECONDS) + pipe.execute() + except redis.RedisError as exc: + logger.warning("usage redis counter failed, buffering to postgres: {}", exc) + _buffer_to_postgres(organization_id, bucket, deltas) + + +def _context_for_record( + *, + organization_id: Optional[UUID] = None, + ctx: Optional[LLMUsageContext] = None, +) -> Optional[LLMUsageContext]: + context = _resolve_context(organization_id=organization_id, ctx=ctx) + if context is None: + return None + from app.services.usage.call_import_context import enrich_usage_context_workspace + + return enrich_usage_context_workspace(context) + + def record_llm_usage( model: str, usage: UsageSnapshot, *, + organization_id: Optional[UUID] = None, ctx: Optional[LLMUsageContext] = None, usage_date: Optional[date] = None, ) -> None: - """Increment Redis counters for one LLM call (best-effort, never raises).""" + """Increment counters for one LLM call (best-effort, never raises).""" if not model: model = "unknown" - try: - context = _resolve_context(ctx) - except ValueError: - logger.debug("llm usage record skipped: missing context") + context = _context_for_record(organization_id=organization_id, ctx=ctx) + if context is None: + logger.warning("llm usage record skipped: missing organization_id") return deltas = _deltas_from_usage(usage) @@ -152,27 +344,133 @@ def record_llm_usage( return day = usage_date or datetime.now(timezone.utc).date() - prefix = _bucket_prefix( - workspace_id=context.workspace_id, - product_section=context.product_section.value, + bucket = _bucket_from_context( + context, model=model, - resource_id=context.resource_id, - resource_type=context.resource_type, usage_date=day, + usage_kind=USAGE_KIND_LLM, ) - hash_key = _pending_hash_key(context.organization_id) + prefix = _bucket_prefix(**bucket) + _incr_pending(context.organization_id, prefix, deltas, bucket) + + +def record_stt_usage( + model: str, + *, + audio_seconds: float | int = 0, + organization_id: Optional[UUID] = None, + ctx: Optional[LLMUsageContext] = None, + usage_date: Optional[date] = None, + count_call: bool = True, +) -> None: + """Increment counters for one STT call (audio seconds + optional call_count).""" + if not model: + model = "unknown" + context = _context_for_record(organization_id=organization_id, ctx=ctx) + if context is None: + logger.warning("stt usage record skipped: missing organization_id") + return + seconds = int(max(0, math.ceil(float(audio_seconds or 0)))) + deltas = _deltas_from_stt(seconds, count_call=count_call) + day = usage_date or datetime.now(timezone.utc).date() + bucket = _bucket_from_context( + context, + model=model, + usage_date=day, + usage_kind=USAGE_KIND_STT, + ) + prefix = _bucket_prefix(**bucket) + _incr_pending(context.organization_id, prefix, deltas, bucket) + + +def record_call_usage( + model: str = "voice-call", + *, + organization_id: Optional[UUID] = None, + ctx: Optional[LLMUsageContext] = None, + usage_date: Optional[date] = None, + audio_seconds: int = 0, +) -> None: + """Record one completed call session (call_count + optional duration). + + Best-effort, never raises. Uses the same Redis-buffered path as other + usage counters (one pipelined HINCRBY batch per call). + """ + if not model: + model = "voice-call" + context = _context_for_record(organization_id=organization_id, ctx=ctx) + if context is None: + logger.warning("call usage record skipped: missing organization_id") + return + + seconds = max(0, int(audio_seconds or 0)) + deltas = { + "prompt_tokens": 0, + "completion_tokens": 0, + "cache_read_tokens": 0, + "cache_creation_tokens": 0, + "reasoning_tokens": 0, + "audio_seconds": seconds, + "tts_characters": 0, + "call_count": 1, + } + day = usage_date or datetime.now(timezone.utc).date() + bucket = _bucket_from_context( + context, + model=model, + usage_date=day, + usage_kind=USAGE_KIND_LLM, + ) + prefix = _bucket_prefix(**bucket) + _incr_pending(context.organization_id, prefix, deltas, bucket) + + +def record_tts_usage( + model: str, + *, + characters: int = 0, + organization_id: Optional[UUID] = None, + ctx: Optional[LLMUsageContext] = None, + usage_date: Optional[date] = None, +) -> None: + """Increment counters for one TTS call (characters + call_count).""" + if not model: + model = "unknown" + context = _context_for_record(organization_id=organization_id, ctx=ctx) + if context is None: + logger.warning("tts usage record skipped: missing organization_id") + return + + chars = max(0, int(characters or 0)) + deltas = _deltas_from_tts(chars) + day = usage_date or datetime.now(timezone.utc).date() + bucket = _bucket_from_context( + context, + model=model, + usage_date=day, + usage_kind=USAGE_KIND_TTS, + ) + prefix = _bucket_prefix(**bucket) + _incr_pending(context.organization_id, prefix, deltas, bucket) + + +def probe_audio_seconds(audio_file_path: str) -> int: + """Best-effort audio duration in whole seconds.""" + if not audio_file_path: + return 0 try: - client = _client() - pipe = client.pipeline() - for metric, delta in deltas.items(): - if delta: - pipe.hincrby(hash_key, f"{prefix}|{metric}", int(delta)) - pipe.sadd("usage:pending:orgs", str(context.organization_id)) - pipe.expire(hash_key, _PENDING_TTL_SECONDS) - pipe.execute() - except redis.RedisError as exc: - logger.warning("llm usage counter skipped: {}", exc) + from pydub import AudioSegment + + return max(0, int(math.ceil(AudioSegment.from_file(audio_file_path).duration_seconds))) + except Exception: + pass + try: + import librosa + + return max(0, int(math.ceil(librosa.get_duration(path=audio_file_path)))) + except Exception: + return 0 def _parse_hash_to_buckets(raw: Dict[str, str]) -> Dict[str, Dict[str, int]]: @@ -216,7 +514,42 @@ def _restore_buckets_to_pending( pipe.expire(hash_key, _PENDING_TTL_SECONDS) pipe.execute() except redis.RedisError as exc: - logger.warning("llm usage redis restore failed: {}", exc) + logger.warning("llm usage redis restore failed, buffering: {}", exc) + for prefix, metrics in buckets.items(): + parsed = _parse_bucket_prefix(prefix) + if parsed: + _buffer_to_postgres(organization_id, parsed, metrics) + + +_CLAIM_COMMITTED_TTL_SECONDS = 24 * 60 * 60 + + +def _claim_committed_key(claim_key: str) -> str: + return f"usage:claim_done:{claim_key}" + + +def _mark_claim_committed(claim_key: str) -> None: + try: + _client().set(_claim_committed_key(claim_key), "1", ex=_CLAIM_COMMITTED_TTL_SECONDS) + except redis.RedisError: + pass + + +def _is_claim_committed(claim_key: str) -> bool: + try: + return bool(_client().exists(_claim_committed_key(claim_key))) + except redis.RedisError: + return False + + +def _has_pending_usage(organization_id: UUID) -> bool: + try: + client = _client() + if client.exists(_pending_hash_key(organization_id)): + return True + return bool(client.sismember("usage:pending:orgs", str(organization_id))) + except redis.RedisError: + return False def _acquire_flush_lock(organization_id: UUID) -> bool: @@ -248,7 +581,6 @@ def _release_flush_lock(organization_id: UUID) -> None: def _claim_pending( organization_id: UUID, ) -> Tuple[Optional[str], Dict[str, Dict[str, int]]]: - """Rename pending → claim key. Returns (claim_key, buckets) or (None, {}).""" claim_id = str(uuid.uuid4()) pending_key = _pending_hash_key(organization_id) claim_key = _claim_hash_key(organization_id, claim_id) @@ -283,74 +615,137 @@ def _upsert_bucket( organization_id: UUID, bucket: Dict[str, Any], deltas: Dict[str, int], - *, - resource_type: Optional[str], ) -> None: + context = bucket.get("context") or {} params = { "organization_id": str(organization_id), "workspace_id": str(bucket["workspace_id"]) if bucket["workspace_id"] else None, "product_section": bucket["product_section"], "model": bucket["model"], - "resource_id": str(bucket["resource_id"]) if bucket["resource_id"] else None, - "resource_type": resource_type, + "context": json.dumps(context), + "context_resource_id": str(context.get("resource_id") or ""), + "context_resource_type": str(context.get("resource_type") or ""), "usage_date": bucket["usage_date"].isoformat(), + "usage_kind": bucket.get("usage_kind") or USAGE_KIND_LLM, "prompt_tokens": int(deltas.get("prompt_tokens", 0)), "completion_tokens": int(deltas.get("completion_tokens", 0)), "cache_read_tokens": int(deltas.get("cache_read_tokens", 0)), "cache_creation_tokens": int(deltas.get("cache_creation_tokens", 0)), "reasoning_tokens": int(deltas.get("reasoning_tokens", 0)), + "audio_seconds": int(deltas.get("audio_seconds", 0)), + "tts_characters": int(deltas.get("tts_characters", 0)), "call_count": int(deltas.get("call_count", 0)), } - result = db.execute( - text( - """ - UPDATE llm_usage_daily SET + update_set = """ prompt_tokens = prompt_tokens + :prompt_tokens, completion_tokens = completion_tokens + :completion_tokens, cache_read_tokens = cache_read_tokens + :cache_read_tokens, cache_creation_tokens = cache_creation_tokens + :cache_creation_tokens, reasoning_tokens = reasoning_tokens + :reasoning_tokens, + audio_seconds = audio_seconds + :audio_seconds, + tts_characters = tts_characters + :tts_characters, call_count = call_count + :call_count, - resource_type = COALESCE(resource_type, :resource_type), + context = CASE + WHEN context = '{}'::jsonb THEN CAST(:context AS jsonb) + ELSE context || CAST(:context AS jsonb) + END, updated_at = now() + """ + where_base = """ WHERE organization_id = CAST(:organization_id AS uuid) AND product_section = :product_section AND model = :model AND usage_date = CAST(:usage_date AS date) + AND usage_kind = :usage_kind AND workspace_id IS NOT DISTINCT FROM CAST(:workspace_id AS uuid) - AND resource_id IS NOT DISTINCT FROM CAST(:resource_id AS uuid) - """ - ), + """ + exact_context_where = where_base + " AND context = CAST(:context AS jsonb)" + legacy_context_where = ( + where_base + + """ + AND COALESCE(context->>'resource_id', '') = :context_resource_id + AND COALESCE(context->>'resource_type', '') = :context_resource_type + """ + ) + + result = db.execute( + text(f"UPDATE llm_usage_daily SET {update_set} {exact_context_where}"), params, ) if result.rowcount: return - db.execute( - text( - """ + result = db.execute( + text(f"UPDATE llm_usage_daily SET {update_set} {legacy_context_where}"), + params, + ) + if result.rowcount: + return + + db.execute(text("SAVEPOINT llm_usage_bucket_insert")) + try: + db.execute( + text( + """ INSERT INTO llm_usage_daily ( id, organization_id, workspace_id, product_section, model, - resource_id, resource_type, usage_date, + context, usage_date, usage_kind, prompt_tokens, completion_tokens, cache_read_tokens, - cache_creation_tokens, reasoning_tokens, call_count, + cache_creation_tokens, reasoning_tokens, audio_seconds, + tts_characters, call_count, created_at, updated_at ) VALUES ( gen_random_uuid(), CAST(:organization_id AS uuid), CAST(:workspace_id AS uuid), :product_section, :model, - CAST(:resource_id AS uuid), :resource_type, CAST(:usage_date AS date), + CAST(:context AS jsonb), CAST(:usage_date AS date), + :usage_kind, :prompt_tokens, :completion_tokens, :cache_read_tokens, - :cache_creation_tokens, :reasoning_tokens, :call_count, + :cache_creation_tokens, :reasoning_tokens, :audio_seconds, + :tts_characters, :call_count, now(), now() ) """ - ), - params, - ) + ), + params, + ) + db.execute(text("RELEASE SAVEPOINT llm_usage_bucket_insert")) + except IntegrityError as exc: + if not _is_unique_violation(exc): + db.execute(text("ROLLBACK TO SAVEPOINT llm_usage_bucket_insert")) + db.execute(text("RELEASE SAVEPOINT llm_usage_bucket_insert")) + raise + db.execute(text("ROLLBACK TO SAVEPOINT llm_usage_bucket_insert")) + result = db.execute( + text(f"UPDATE llm_usage_daily SET {update_set} {legacy_context_where}"), + params, + ) + if not result.rowcount: + result = db.execute( + text(f"UPDATE llm_usage_daily SET {update_set} {exact_context_where}"), + params, + ) + db.execute(text("RELEASE SAVEPOINT llm_usage_bucket_insert")) + if not result.rowcount: + logger.warning( + "llm usage upsert unique conflict but no matching bucket for org {}", + organization_id, + ) + + +def _is_unique_violation(exc: BaseException) -> bool: + text_blob = " ".join( + str(part) + for part in ( + exc, + getattr(exc, "orig", None), + getattr(getattr(exc, "orig", None), "pgcode", None), + ) + if part is not None + ).lower() + return "uniqueviolation" in text_blob or "duplicate key" in text_blob def _is_missing_organization_fk(exc: BaseException) -> bool: - """True when insert failed because organization_id is not in organizations.""" text_blob = " ".join( str(part) for part in (exc, getattr(exc, "orig", None), getattr(exc, "args", None)) @@ -359,62 +754,193 @@ def _is_missing_organization_fk(exc: BaseException) -> bool: return "llm_usage_daily_organization_id_fkey" in text_blob -def flush_usage_to_catalog(db: Session, organization_id: UUID) -> int: - """Claim Redis deltas, commit to llm_usage_daily, then ack the claim.""" - if not _acquire_flush_lock(organization_id): +def _flush_pending_buffer(db: Session, organization_id: UUID) -> int: + """Drain Postgres write-ahead rows into llm_usage_daily.""" + try: + rows = db.execute( + text( + """ + SELECT id, workspace_id, product_section, model, context, + usage_date, usage_kind, + prompt_tokens, completion_tokens, cache_read_tokens, + cache_creation_tokens, reasoning_tokens, audio_seconds, + tts_characters, call_count + FROM usage_pending_buffer + WHERE organization_id = CAST(:organization_id AS uuid) + ORDER BY created_at ASC + LIMIT 2000 + """ + ), + {"organization_id": str(organization_id)}, + ).mappings().all() + except Exception as exc: + db.rollback() + logger.debug("usage buffer read skipped: {}", exc) + return 0 + if not rows: return 0 - claim_key = None - buckets: Dict[str, Dict[str, int]] = {} + flushed = 0 + ids: List[str] = [] try: - claim_key, buckets = _claim_pending(organization_id) - if not claim_key or not buckets: - return 0 + for row in rows: + row_context = row["context"] or {} + if isinstance(row_context, str): + row_context = json.loads(row_context) + bucket = { + "workspace_id": row["workspace_id"], + "product_section": row["product_section"], + "model": row["model"], + "context": row_context, + "usage_date": row["usage_date"], + "usage_kind": row["usage_kind"] or USAGE_KIND_LLM, + } + deltas = { + "prompt_tokens": int(row["prompt_tokens"] or 0), + "completion_tokens": int(row["completion_tokens"] or 0), + "cache_read_tokens": int(row["cache_read_tokens"] or 0), + "cache_creation_tokens": int(row["cache_creation_tokens"] or 0), + "reasoning_tokens": int(row["reasoning_tokens"] or 0), + "audio_seconds": int(row["audio_seconds"] or 0), + "tts_characters": int(row.get("tts_characters") or 0), + "call_count": int(row["call_count"] or 0), + } + _upsert_bucket( + db, + organization_id, + bucket, + deltas, + ) + ids.append(str(row["id"])) + flushed += 1 + if ids: + from sqlalchemy import bindparam - flushed = 0 - try: - for prefix, deltas in buckets.items(): - parsed = _parse_bucket_prefix(prefix) - if not parsed: - continue - _upsert_bucket( - db, - organization_id, - parsed, - deltas, - resource_type=parsed.get("resource_type"), - ) - flushed += 1 - db.commit() - except Exception as exc: - db.rollback() - if _is_missing_organization_fk(exc): - # Stale/test org ids must not be restored — that loops forever on beat. - logger.warning( - "llm usage flush dropped for unknown organization {}: {}", - organization_id, - exc, - ) - if claim_key: - _ack_claim(claim_key, organization_id) - return 0 - logger.warning("llm usage catalog flush failed, restoring redis: {}", exc) - _restore_buckets_to_pending(organization_id, buckets) - if claim_key: + db.execute( + text( + "DELETE FROM usage_pending_buffer WHERE id IN :ids" + ).bindparams(bindparam("ids", expanding=True)), + {"ids": ids}, + ) + db.commit() + return flushed + except Exception as exc: + db.rollback() + if _is_missing_organization_fk(exc): + logger.warning( + "usage buffer dropped for unknown organization {}: {}", + organization_id, + exc, + ) + if ids: try: - _client().delete(claim_key) - except redis.RedisError: - pass + db.execute( + text( + """ + DELETE FROM usage_pending_buffer + WHERE organization_id = CAST(:organization_id AS uuid) + """ + ), + {"organization_id": str(organization_id)}, + ) + db.commit() + except Exception: + db.rollback() return 0 + logger.warning("usage buffer flush failed: {}", exc) + return 0 - _ack_claim(claim_key, organization_id) - return flushed - finally: - _release_flush_lock(organization_id) + +_CATALOG_FLUSH_COOLDOWN_SEC = 20 + + +def _catalog_flush_recently(organization_id: UUID) -> bool: + """Skip flush if another request flushed this org within the cooldown window.""" + try: + client = _client() + key = f"usage:catalog_flush:{organization_id}" + return not client.set(key, "1", nx=True, ex=_CATALOG_FLUSH_COOLDOWN_SEC) + except redis.RedisError: + return False + + +def flush_usage_to_catalog(db: Session, organization_id: UUID, *, force: bool = False) -> int: + """Claim Redis deltas + drain PG buffer into llm_usage_daily.""" + _recover_orphaned_claims() + skip_redis_flush = not force and _catalog_flush_recently(organization_id) + if skip_redis_flush and _has_pending_usage(organization_id): + skip_redis_flush = False + flushed = 0 + if not skip_redis_flush: + redis_locked = _acquire_flush_lock(organization_id) + claim_key = None + buckets: Dict[str, Dict[str, int]] = {} + try: + if redis_locked: + claim_key, buckets = _claim_pending(organization_id) + if claim_key and buckets: + skipped: Dict[str, Dict[str, int]] = {} + try: + for prefix, deltas in buckets.items(): + parsed = _parse_bucket_prefix(prefix) + if not parsed: + skipped[prefix] = deltas + logger.warning( + "llm usage skipped unparseable bucket prefix for org {}", + organization_id, + ) + continue + _upsert_bucket( + db, + organization_id, + parsed, + deltas, + ) + flushed += 1 + db.commit() + except Exception as exc: + db.rollback() + if _is_missing_organization_fk(exc): + logger.warning( + "llm usage flush dropped for unknown organization {}: {}", + organization_id, + exc, + ) + if claim_key: + _mark_claim_committed(claim_key) + _ack_claim(claim_key, organization_id) + claim_key = None + buckets = {} + else: + logger.warning( + "llm usage catalog flush failed, restoring redis: {}", exc + ) + restore_buckets = dict(buckets) + if skipped: + restore_buckets.update(skipped) + _restore_buckets_to_pending(organization_id, restore_buckets) + if claim_key: + try: + _client().delete(claim_key) + except redis.RedisError: + pass + claim_key = None + return flushed + _flush_pending_buffer(db, organization_id) + if skipped: + _restore_buckets_to_pending(organization_id, skipped) + if claim_key: + _mark_claim_committed(claim_key) + _ack_claim(claim_key, organization_id) + claim_key = None + finally: + if redis_locked: + _release_flush_lock(organization_id) + + flushed += _flush_pending_buffer(db, organization_id) + return flushed def _recover_orphaned_claims() -> None: - """Re-queue claim hashes left behind by a crashed flusher.""" try: client = _client() for claim_key in client.scan_iter(match="usage:flushing:*", count=100): @@ -427,6 +953,9 @@ def _recover_orphaned_claims() -> None: continue if client.exists(_flush_lock_key(org_id)): continue + if _is_claim_committed(claim_key): + client.delete(claim_key) + continue buckets = _read_hash_buckets(claim_key) if buckets: _restore_buckets_to_pending(org_id, buckets) @@ -436,10 +965,11 @@ def _recover_orphaned_claims() -> None: def list_pending_organization_ids() -> List[UUID]: + result: List[UUID] = [] + seen: set[UUID] = set() try: client = _client() raw_ids = client.smembers("usage:pending:orgs") - result: List[UUID] = [] stale: List[str] = [] for value in raw_ids: try: @@ -449,17 +979,44 @@ def list_pending_organization_ids() -> List[UUID]: continue if client.exists(_pending_hash_key(org_id)): result.append(org_id) + seen.add(org_id) else: stale.append(value) if stale: client.srem("usage:pending:orgs", *stale) - return result except (redis.RedisError, ValueError): - return [] + pass + + try: + from app.database import SessionLocal + + db = SessionLocal() + try: + rows = db.execute( + text( + """ + SELECT DISTINCT organization_id + FROM usage_pending_buffer + LIMIT 5000 + """ + ) + ).all() + for row in rows: + org_id = row[0] if not isinstance(row, dict) else row["organization_id"] + if isinstance(org_id, str): + org_id = UUID(org_id) + if org_id not in seen: + result.append(org_id) + seen.add(org_id) + finally: + db.close() + except Exception as exc: + logger.debug("usage buffer org scan skipped: {}", exc) + + return result def flush_all_usage_to_catalog(db_factory) -> int: - """Flush all orgs with pending usage (Celery beat).""" _recover_orphaned_claims() total = 0 for org_id in list_pending_organization_ids(): @@ -486,6 +1043,8 @@ def merge_usage_totals( getattr(row, "cache_creation_tokens", 0) or 0 ) totals["reasoning_tokens"] += int(getattr(row, "reasoning_tokens", 0) or 0) + totals["audio_seconds"] += int(getattr(row, "audio_seconds", 0) or 0) + totals["tts_characters"] += int(getattr(row, "tts_characters", 0) or 0) totals["call_count"] += int(getattr(row, "call_count", 0) or 0) totals["total_tokens"] = totals["prompt_tokens"] + totals["completion_tokens"] return totals diff --git a/app/services/usage/usage_labels.py b/app/services/usage/usage_labels.py new file mode 100644 index 00000000..4cee5167 --- /dev/null +++ b/app/services/usage/usage_labels.py @@ -0,0 +1,424 @@ +"""Human-readable labels for usage attribution context (no raw UUIDs in UI).""" + +from __future__ import annotations + +from typing import Any, Dict, Optional, Set +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models.database import ( + Agent, + CallImport, + CallImportEvaluation, + CallImportRow, + CallImportTag, + CallImportTagAssignment, + EvaluatorResult, + TTSComparison, +) + +RESOURCE_TYPE_LABELS = { + "call_import_evaluation": "Evaluation", + "call_import": "Import", + "tts_comparison": "Simulation", + "evaluator_result": "Evaluator result", + "agent": "Agent", + "metric": "Metric", +} + +_USAGE_KIND_LABELS = { + "llm": "LLM", + "stt": "STT", + "tts": "TTS", +} + + +def usage_kind_label(kind: Optional[str]) -> str: + if not kind: + return "—" + return _USAGE_KIND_LABELS.get(kind, kind) + + +def short_entity_id(uid: UUID) -> str: + return str(uid)[:8] + + +def format_entity_label( + custom_name: Optional[str], + uid: UUID, + default_prefix: str, +) -> str: + """Human label: name-shortId (e.g. unauthenticated sheet.xlsx-3111d376).""" + short = short_entity_id(uid) + text = (custom_name or "").strip() + if text: + return f"{text}-{short}" + return f"{default_prefix}-{short}" + + +class UsageNameResolver: + """Batch-resolve entity names for usage context JSONB keys.""" + + def __init__(self, db: Session, organization_id: UUID) -> None: + self._db = db + self._organization_id = organization_id + self._evaluations: Dict[UUID, str] = {} + self._call_imports: Dict[UUID, str] = {} + self._call_import_rows: Dict[UUID, str] = {} + self._tts_comparisons: Dict[UUID, str] = {} + self._agents: Dict[UUID, str] = {} + + def preload(self, contexts: list[Dict[str, Any]]) -> None: + eval_ids: Set[UUID] = set() + import_ids: Set[UUID] = set() + row_ids: Set[UUID] = set() + comparison_ids: Set[UUID] = set() + agent_ids: Set[UUID] = set() + evaluator_result_ids: Set[UUID] = set() + + for ctx in contexts: + if not ctx: + continue + norm = _normalize_context(ctx) + rtype = norm.get("resource_type") + resource_id = norm.get("resource_id") + if resource_id: + uid = parse_uuid(resource_id) + if uid: + if rtype == "call_import": + import_ids.add(uid) + elif rtype == "call_import_evaluation": + eval_ids.add(uid) + elif rtype == "tts_comparison": + comparison_ids.add(uid) + elif rtype == "agent": + agent_ids.add(uid) + elif rtype == "evaluator_result": + eval_ids.add(uid) + else: + eval_ids.add(uid) + import_ids.add(uid) + for key, bucket in ( + ("evaluation_id", eval_ids), + ("call_import_id", import_ids), + ("call_import_row_id", row_ids), + ("agent_id", agent_ids), + ("evaluator_result_id", evaluator_result_ids), + ): + raw = norm.get(key) + if not raw: + continue + uid = parse_uuid(raw) + if uid: + bucket.add(uid) + + if eval_ids: + for row in ( + self._db.query(CallImportEvaluation) + .filter( + CallImportEvaluation.organization_id == self._organization_id, + CallImportEvaluation.id.in_(eval_ids), + ) + .all() + ): + self._evaluations[row.id] = format_entity_label( + row.name, row.id, "Evaluation" + ) + + tag_names_by_import: Dict[UUID, list[str]] = {} + if import_ids: + for cid, tag_name in ( + self._db.query( + CallImportTagAssignment.call_import_id, + CallImportTag.name, + ) + .join( + CallImportTag, + CallImportTag.id == CallImportTagAssignment.tag_id, + ) + .filter(CallImportTagAssignment.call_import_id.in_(import_ids)) + .all() + ): + tag_names_by_import.setdefault(cid, []).append(tag_name) + + if import_ids: + for row in ( + self._db.query(CallImport) + .filter( + CallImport.organization_id == self._organization_id, + CallImport.id.in_(import_ids), + ) + .all() + ): + display_name, prefix = _call_import_label_parts( + row, + sorted(tag_names_by_import.get(row.id, [])), + ) + self._call_imports[row.id] = format_entity_label( + display_name, row.id, prefix + ) + + if comparison_ids: + for row in ( + self._db.query(TTSComparison) + .filter( + TTSComparison.organization_id == self._organization_id, + TTSComparison.id.in_(comparison_ids), + ) + .all() + ): + self._tts_comparisons[row.id] = _tts_comparison_display_name(row) + + if evaluator_result_ids: + for row in ( + self._db.query(EvaluatorResult) + .filter( + EvaluatorResult.organization_id == self._organization_id, + EvaluatorResult.id.in_(evaluator_result_ids), + ) + .all() + ): + if row.agent_id: + agent_ids.add(row.agent_id) + + missing_agent_ids = [uid for uid in agent_ids if uid not in self._agents] + if missing_agent_ids: + for row in ( + self._db.query(Agent) + .filter( + Agent.organization_id == self._organization_id, + Agent.id.in_(missing_agent_ids), + ) + .all() + ): + self._agents[row.id] = _agent_display_name(row) + + if row_ids: + for row in ( + self._db.query(CallImportRow) + .filter( + CallImportRow.organization_id == self._organization_id, + CallImportRow.id.in_(row_ids), + ) + .all() + ): + self._call_import_rows[row.id] = _clean_name( + row.conversation_id, "Conversation" + ) + + def evaluation_name(self, raw_id: str) -> str: + uid = parse_uuid(raw_id) + if uid and uid in self._evaluations: + return self._evaluations[uid] + if uid: + return format_entity_label(None, uid, "Evaluation") + return "Evaluation" + + def call_import_name(self, raw_id: str) -> str: + uid = parse_uuid(raw_id) + if uid and uid in self._call_imports: + return self._call_imports[uid] + if uid: + return format_entity_label(None, uid, "Import") + return "Import" + + def tts_comparison_name(self, raw_id: str) -> str: + uid = parse_uuid(raw_id) + if uid and uid in self._tts_comparisons: + return self._tts_comparisons[uid] + if uid: + return format_entity_label(None, uid, "Simulation") + return "Simulation" + + def agent_name(self, raw_id: str) -> str: + uid = parse_uuid(raw_id) + if uid and uid in self._agents: + return self._agents[uid] + if uid: + return format_entity_label(None, uid, "Agent") + return "Agent" + + def resource_name(self, raw_id: str, resource_type: Optional[str]) -> str: + if resource_type == "call_import_evaluation": + return self.evaluation_name(raw_id) + if resource_type == "call_import": + return self.call_import_name(raw_id) + if resource_type == "tts_comparison": + return self.tts_comparison_name(raw_id) + if resource_type == "agent": + return self.agent_name(raw_id) + uid = parse_uuid(raw_id) + if uid: + prefix = RESOURCE_TYPE_LABELS.get(resource_type or "", "Resource") + return format_entity_label(None, uid, prefix) + return RESOURCE_TYPE_LABELS.get(resource_type or "", "Unscoped") + + def call_import_row_name(self, raw_id: str) -> str: + uid = parse_uuid(raw_id) + if uid and uid in self._call_import_rows: + return self._call_import_rows[uid] + return "Conversation" + + +def parse_uuid(raw: Any) -> Optional[UUID]: + try: + return UUID(str(raw)) + except (ValueError, TypeError): + return None + + +def _clean_name(value: Optional[str], fallback: str) -> str: + text = (value or "").strip() + return text or fallback + + +def _call_import_title(row: CallImport) -> Optional[str]: + filename = (row.original_filename or "").strip() + if filename: + return filename + dataset = (row.dataset or "").strip() + if dataset: + return dataset + return None + + +def _call_import_meta_suffix(row: CallImport, tag_names: list[str]) -> str: + parts: list[str] = [] + dataset = (row.dataset or "").strip() + filename = (row.original_filename or "").strip() + if dataset and dataset != filename: + parts.append(dataset) + for name in tag_names: + if name and name not in parts: + parts.append(name) + if not parts: + return "" + return f" ({' · '.join(parts)})" + + +def _call_import_label_parts( + row: CallImport, + tag_names: list[str], +) -> tuple[Optional[str], str]: + base = _call_import_title(row) + suffix = _call_import_meta_suffix(row, tag_names) + if base: + return f"{base}{suffix}", "Import" + if suffix: + return suffix.strip(" ()"), "Import" + return None, "Import" + + +def _tts_comparison_display_name(row: TTSComparison) -> str: + name = (row.name or "").strip() or "Simulation" + sim = (row.simulation_id or "").strip() + if sim: + return f"{name} #{sim}" + return format_entity_label(name, row.id, "Simulation") + + +def _agent_display_name(row: Agent) -> str: + name = (row.name or "").strip() or "Agent" + short = (row.agent_id or "").strip() + if short: + return f"{name} #{short}" + return format_entity_label(name, row.id, "Agent") + + +def _normalize_context(raw: Any) -> Dict[str, str]: + if not raw or not isinstance(raw, dict): + return {} + return {str(k): str(v) for k, v in raw.items() if v is not None} + + +def _richest_context(contexts: list[Dict[str, str]]) -> Dict[str, str]: + if not contexts: + return {} + return max(contexts, key=lambda c: len(c)) + + +def build_usage_resource_label( + context: Optional[Dict[str, Any]], + resource_type: Optional[str], + resolver: UsageNameResolver, +) -> str: + """Hierarchical label: call import · evaluation · conversation.""" + ctx = _normalize_context(context) + parts: list[str] = [] + + call_import_id = ctx.get("call_import_id") + if call_import_id: + parts.append(resolver.call_import_name(call_import_id)) + elif resource_type == "call_import" and ctx.get("resource_id"): + parts.append(resolver.call_import_name(ctx["resource_id"])) + + evaluation_id = ctx.get("evaluation_id") + agent_id = ctx.get("agent_id") + resource_id = ctx.get("resource_id") + if evaluation_id: + parts.append(resolver.evaluation_name(evaluation_id)) + elif resource_type == "call_import_evaluation" and resource_id: + parts.append(resolver.evaluation_name(resource_id)) + elif resource_type == "tts_comparison" and resource_id: + parts.append(resolver.tts_comparison_name(resource_id)) + elif resource_type == "agent" and resource_id: + parts.append(resolver.agent_name(resource_id)) + elif agent_id: + parts.append(resolver.agent_name(agent_id)) + elif resource_type == "evaluator_result" and resource_id: + parts.append(resolver.resource_name(resource_id, resource_type)) + elif resource_type and resource_id: + parts.append(resolver.resource_name(resource_id, resource_type)) + + row_id = ctx.get("call_import_row_id") + if row_id: + parts.append(resolver.call_import_row_name(row_id)) + + if not parts: + rid = resource_id or agent_id + if rid and (resource_type == "agent" or agent_id): + return resolver.agent_name(rid) + if resource_type: + prefix = RESOURCE_TYPE_LABELS.get(resource_type, resource_type) + uid = parse_uuid(rid) + if uid: + return format_entity_label(None, uid, prefix) + return prefix + return "Unscoped" + + return " / ".join(parts) + + +def labels_for_resource_buckets( + buckets: list[tuple[Optional[str], Optional[str], list[Dict[str, Any]]]], + resolver: UsageNameResolver, +) -> Dict[str, str]: + """Map resource_id string -> label; buckets are (resource_id, resource_type, contexts).""" + labels: Dict[str, str] = {} + for raw_id, resource_type, contexts in buckets: + if not raw_id: + continue + ctx = _richest_context([_normalize_context(c) for c in contexts]) + merged = dict(ctx) + merged.setdefault("resource_id", raw_id) + if resource_type: + merged.setdefault("resource_type", resource_type) + labels[str(raw_id)] = build_usage_resource_label( + merged, resource_type, resolver + ) + return labels + + +def labels_for_call_import_ids( + import_ids: list[UUID], + resolver: UsageNameResolver, +) -> Dict[str, str]: + labels: Dict[str, str] = {} + for uid in import_ids: + labels[str(uid)] = resolver.call_import_name(str(uid)) + return labels + + +def collect_contexts_from_rows(rows: list[Any]) -> list[Dict[str, Any]]: + return [_normalize_context(r[0]) for r in rows if r and r[0]] diff --git a/app/services/usage/voice_usage_processor.py b/app/services/usage/voice_usage_processor.py index 56d65f57..d1529507 100644 --- a/app/services/usage/voice_usage_processor.py +++ b/app/services/usage/voice_usage_processor.py @@ -1,7 +1,8 @@ -"""Voice pipeline processor that records LLM token usage from MetricsFrames.""" +"""Voice pipeline processor that records LLM/TTS usage from MetricsFrames.""" from __future__ import annotations +import math from typing import Optional from uuid import UUID @@ -16,7 +17,7 @@ def create_llm_usage_recorder( resource_id: UUID | str | None = None, resource_type: Optional[str] = None, ): - """Build a FrameProcessor that records LLM usage from MetricsFrames. + """Build a FrameProcessor that records LLM/TTS usage from MetricsFrames. Returns None when organization_id is missing or efficientai is unavailable. """ @@ -25,15 +26,22 @@ def create_llm_usage_recorder( try: from efficientai.frames.frames import Frame, MetricsFrame - from efficientai.metrics.metrics import LLMUsageMetricsData + from efficientai.metrics.metrics import ( + LLMUsageMetricsData, + ProcessingMetricsData, + TTSUsageMetricsData, + ) from efficientai.processors.frame_processor import FrameDirection, FrameProcessor from app.services.usage.context import ( LLMUsageContext, LLMUsageProductSection, - set_usage_context, ) - from app.services.usage.llm_usage import record_llm_usage + from app.services.usage.llm_usage import ( + record_llm_usage, + record_stt_usage, + record_tts_usage, + ) from app.services.usage.normalize import UsageSnapshot except Exception as exc: logger.debug("voice usage recorder unavailable: {}", exc) @@ -48,14 +56,12 @@ def create_llm_usage_recorder( ws_uuid = UUID(str(workspace_id)) if workspace_id else None res_uuid = UUID(str(resource_id)) if resource_id else None - set_usage_context( - LLMUsageContext( - organization_id=org_uuid, - workspace_id=ws_uuid, - product_section=section, - resource_id=res_uuid, - resource_type=resource_type, - ) + usage_ctx = LLMUsageContext( + organization_id=org_uuid, + workspace_id=ws_uuid, + product_section=section, + resource_id=res_uuid, + resource_type=resource_type, ) class LLMUsageRecorderProcessor(FrameProcessor): @@ -74,7 +80,29 @@ async def process_frame(self, frame: Frame, direction: FrameDirection): ), reasoning_tokens=int(tokens.reasoning_tokens or 0), ) - record_llm_usage(item.model or "unknown", snapshot) + record_llm_usage( + item.model or "unknown", + snapshot, + organization_id=org_uuid, + ctx=usage_ctx, + ) + elif isinstance(item, TTSUsageMetricsData): + record_tts_usage( + item.model or "unknown", + characters=int(item.value or 0), + organization_id=org_uuid, + ctx=usage_ctx, + ) + elif isinstance(item, ProcessingMetricsData): + processor_name = (item.processor or "").lower() + if "stt" in processor_name and item.value is not None: + seconds = max(1, int(math.ceil(float(item.value)))) + record_stt_usage( + item.model or "unknown", + audio_seconds=seconds, + organization_id=org_uuid, + ctx=usage_ctx, + ) await self.push_frame(frame, direction) return LLMUsageRecorderProcessor() diff --git a/app/services/voice_agent/bot_fast_api.py b/app/services/voice_agent/bot_fast_api.py index 3e2d25e4..3bcd92a7 100644 --- a/app/services/voice_agent/bot_fast_api.py +++ b/app/services/voice_agent/bot_fast_api.py @@ -251,7 +251,9 @@ async def on_silence_hangup(): usage_recorder = create_llm_usage_recorder( organization_id=organization_id, workspace_id=workspace_id, - product_section="telephony" if telephony_mode else "playground", + product_section="agents" if agent_id else ("telephony" if telephony_mode else "playground"), + resource_id=agent_id, + resource_type="agent" if agent_id else None, ) if usage_recorder: pipeline_processors.append(usage_recorder) @@ -292,7 +294,9 @@ async def on_client_disconnected(transport, client): usage_recorder = create_llm_usage_recorder( organization_id=organization_id, workspace_id=workspace_id, - product_section="playground", + product_section="agents" if agent_id else "playground", + resource_id=agent_id, + resource_type="agent" if agent_id else None, ) pipeline_steps = [ ws_transport.input(), diff --git a/app/services/voice_agent/voice_bundle.py b/app/services/voice_agent/voice_bundle.py index 296bf032..edebc4d0 100644 --- a/app/services/voice_agent/voice_bundle.py +++ b/app/services/voice_agent/voice_bundle.py @@ -510,6 +510,7 @@ async def run_voice_bundle_fastapi( websocket_client, system_instruction: str | None = None, organization_id: str | None = None, + workspace_id: str | None = None, agent_id: str | None = None, persona_id: str | None = None, scenario_id: str | None = None, @@ -835,8 +836,27 @@ async def on_silence_hangup(): if telephony_mode and call_short_id and agent_transcript_processor: pipeline_processors.append(agent_transcript_processor) + from app.services.usage.voice_usage_processor import create_llm_usage_recorder + + agent_usage_section = ( + "agents" + if agent_id + else ("telephony" if telephony_mode else "voice_playground") + ) + usage_recorder = create_llm_usage_recorder( + organization_id=organization_id, + workspace_id=workspace_id, + product_section=agent_usage_section, + resource_id=agent_id, + resource_type="agent" if agent_id else None, + ) + pipeline_processors.extend([ tts, + ]) + if usage_recorder: + pipeline_processors.append(usage_recorder) + pipeline_processors.extend([ bot_recorder if use_aligned_recorders else audio_buffer_output, ws_transport.output(), context_aggregator.assistant(), @@ -865,6 +885,7 @@ async def on_client_disconnected(transport, client): await task.cancel() else: rtvi = imports["RTVIProcessor"](config=imports["RTVIConfig"](config=[])) + rtvi_processors = [ws_transport.input()] if silence_hangup_processor: rtvi_processors.append(silence_hangup_processor) @@ -875,6 +896,17 @@ async def on_client_disconnected(transport, client): rtvi, llm, tts, + ]) + usage_recorder = create_llm_usage_recorder( + organization_id=organization_id, + workspace_id=workspace_id, + product_section="agents" if agent_id else "voice_playground", + resource_id=agent_id, + resource_type="agent" if agent_id else None, + ) + if usage_recorder: + rtvi_processors.append(usage_recorder) + rtvi_processors.extend([ audio_buffer_output, ws_transport.output(), context_aggregator.assistant(), diff --git a/app/services/webrtc_bridge/test_agent_processor.py b/app/services/webrtc_bridge/test_agent_processor.py index 6d5ea658..990fa6bd 100644 --- a/app/services/webrtc_bridge/test_agent_processor.py +++ b/app/services/webrtc_bridge/test_agent_processor.py @@ -9,8 +9,9 @@ import asyncio import io import os -from typing import Optional, Callable, Awaitable, List, Dict, Any +from typing import Optional, Callable, Awaitable, List, Dict, Any, Union from dataclasses import dataclass, field +from uuid import UUID from loguru import logger # TTS service imports @@ -90,6 +91,9 @@ class TestAgentConfig: response_delay_ms: int = 500 # Delay before responding (more natural) allow_interruptions: bool = False + organization_id: Optional[Union[UUID, str]] = None + workspace_id: Optional[Union[UUID, str]] = None + class TestAgentProcessor: """ @@ -366,7 +370,9 @@ async def _generate_llm_response(self) -> Optional[str]: max_tokens=self.config.llm_max_tokens if self.config.llm_max_tokens is not None else 150, temperature=self.config.llm_temperature if self.config.llm_temperature is not None else 0.7, ) - + + self._record_llm_usage(response=response) + return response.choices[0].message.content.strip() except Exception as e: @@ -392,6 +398,8 @@ async def _text_to_speech(self, text: str) -> Optional[bytes]: else: audio = await self._tts_cartesia(text) + if audio: + self._record_tts_usage(text=text) return audio except Exception as e: logger.error(f"[TestAgent] TTS ({provider}) error: {e}") @@ -400,6 +408,82 @@ async def _text_to_speech(self, text: str) -> Optional[bytes]: def _tts_settings(self) -> Dict[str, Any]: return dict(self.config.tts_config or {}) + def _record_tts_usage(self, *, text: str) -> None: + if not self.config.organization_id: + return + try: + from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + llm_usage_context, + ) + from app.services.usage.llm_usage import record_tts_usage + + model = self.config.tts_model or TTS_DEFAULT_MODELS.get( + self.config.tts_provider.lower(), "unknown" + ) + org_id = UUID(str(self.config.organization_id)) + ws_id = ( + UUID(str(self.config.workspace_id)) + if self.config.workspace_id + else None + ) + with llm_usage_context( + LLMUsageContext( + organization_id=org_id, + workspace_id=ws_id, + product_section=LLMUsageProductSection.TEST_AGENT, + ) + ): + record_tts_usage( + model, + characters=len(text or ""), + organization_id=org_id, + ) + except Exception as exc: + logger.debug("test agent tts usage record skipped: {}", exc) + + def _record_llm_usage(self, *, response: Any) -> None: + if not self.config.organization_id: + return + try: + from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + llm_usage_context, + ) + from app.services.usage.llm_usage import record_llm_usage + from app.services.usage.normalize import UsageSnapshot + + usage = getattr(response, "usage", None) + if usage is None: + return + org_id = UUID(str(self.config.organization_id)) + ws_id = ( + UUID(str(self.config.workspace_id)) + if self.config.workspace_id + else None + ) + model = self.config.llm_model or "unknown" + snapshot = UsageSnapshot( + prompt_tokens=int(getattr(usage, "prompt_tokens", 0) or 0), + completion_tokens=int(getattr(usage, "completion_tokens", 0) or 0), + ) + with llm_usage_context( + LLMUsageContext( + organization_id=org_id, + workspace_id=ws_id, + product_section=LLMUsageProductSection.TEST_AGENT, + ) + ): + record_llm_usage( + model, + snapshot, + organization_id=org_id, + ) + except Exception as exc: + logger.debug("test agent llm usage record skipped: {}", exc) + async def _tts_cartesia(self, text: str) -> Optional[bytes]: """Synthesize speech via Cartesia.""" import httpx diff --git a/app/workers/tasks/agent_flowchart_jobs.py b/app/workers/tasks/agent_flowchart_jobs.py index 8704fd6c..02431794 100644 --- a/app/workers/tasks/agent_flowchart_jobs.py +++ b/app/workers/tasks/agent_flowchart_jobs.py @@ -52,13 +52,19 @@ def generate_agent_flowchart_task( ) return - graph, provider_enum, model_str = generate_agent_flowchart( - prompt_text=partial.content, - organization_id=partial.organization_id, - db=db, - provider=provider, - model=model, + from app.services.usage.context import ( + llm_usage_context, + usage_context_for_prompt_partial, ) + + with llm_usage_context(usage_context_for_prompt_partial(partial)): + graph, provider_enum, model_str = generate_agent_flowchart( + prompt_text=partial.content, + organization_id=partial.organization_id, + db=db, + provider=provider, + model=model, + ) partial.agent_flowchart = graph.model_dump(mode="json") if isinstance(partial.agent_flowchart, dict): generated_at = graph.generated_at @@ -130,18 +136,24 @@ def map_agent_flowchart_prompt_sections_task( ): raise ValueError("Generate a flowchart before mapping prompt sections") + from app.services.usage.context import ( + llm_usage_context, + usage_context_for_prompt_partial, + ) + graph = apply_prompt_hash_staleness( AgentFlowGraph.model_validate(partial.agent_flowchart), partial.content, ) - mapped_graph = map_all_flow_nodes_to_prompt( - prompt_text=partial.content, - graph=graph, - organization_id=partial.organization_id, - db=db, - provider=provider, - model=model, - ) + with llm_usage_context(usage_context_for_prompt_partial(partial)): + mapped_graph = map_all_flow_nodes_to_prompt( + prompt_text=partial.content, + graph=graph, + organization_id=partial.organization_id, + db=db, + provider=provider, + model=model, + ) partial.agent_flowchart = mapped_graph.model_dump(mode="json") if isinstance(partial.agent_flowchart, dict): if mapped_graph.generated_at is not None: diff --git a/app/workers/tasks/evaluate_call_import_row.py b/app/workers/tasks/evaluate_call_import_row.py index e87b3285..dafba330 100644 --- a/app/workers/tasks/evaluate_call_import_row.py +++ b/app/workers/tasks/evaluate_call_import_row.py @@ -364,19 +364,19 @@ def evaluate_call_import_row_task( row_db.commit() return {"status": "failed", "reason": "evaluation_missing"} - from app.services.usage.context import ( - LLMUsageContext, - LLMUsageProductSection, - set_usage_context, + from app.services.usage.call_import_context import ( + call_import_evaluation_usage_context, ) + from app.services.usage.context import set_usage_context usage_ctx_token = set_usage_context( - LLMUsageContext( + call_import_evaluation_usage_context( organization_id=evaluation.organization_id, workspace_id=evaluation.workspace_id, - product_section=LLMUsageProductSection.CALL_IMPORT_EVALUATIONS, - resource_id=evaluation.id, - resource_type="call_import_evaluation", + evaluation_id=evaluation.id, + call_import_id=evaluation.call_import_id, + evaluation_row_id=eval_row.id, + call_import_row_id=source_row.id, ) ) diff --git a/app/workers/tasks/generate_evaluation_metric_clusters.py b/app/workers/tasks/generate_evaluation_metric_clusters.py index e91dc108..8cb59775 100644 --- a/app/workers/tasks/generate_evaluation_metric_clusters.py +++ b/app/workers/tasks/generate_evaluation_metric_clusters.py @@ -120,19 +120,23 @@ def on_progress(completed: int, total: int) -> None: flag_modified(evaluation, "metric_clusters") db.commit() - state = generate_metric_clusters( - db, - evaluation, - evaluation.organization_id, - provider_enum, - model_str, - completed_row_pairs=completed_pairs, - metrics=metrics, - policies=policies, - on_progress=on_progress, - max_llm_calls=max_llm_calls, - is_cancelled=_reload_cancelled, - ) + from app.services.usage.call_import_context import usage_context_for_evaluation + from app.services.usage.context import llm_usage_context + + with llm_usage_context(usage_context_for_evaluation(evaluation)): + state = generate_metric_clusters( + db, + evaluation, + evaluation.organization_id, + provider_enum, + model_str, + completed_row_pairs=completed_pairs, + metrics=metrics, + policies=policies, + on_progress=on_progress, + max_llm_calls=max_llm_calls, + is_cancelled=_reload_cancelled, + ) if _reload_cancelled(): return diff --git a/app/workers/tasks/generate_evaluation_prompt_improvements.py b/app/workers/tasks/generate_evaluation_prompt_improvements.py index 0ba282a2..fd22b1fb 100644 --- a/app/workers/tasks/generate_evaluation_prompt_improvements.py +++ b/app/workers/tasks/generate_evaluation_prompt_improvements.py @@ -107,25 +107,29 @@ def generate_evaluation_prompt_improvements_task( eval_rows, ) - state = generate_prompt_improvements( - evaluation=evaluation, - imported_agent=imported_agent, - clusters_state=clusters_state, - organization_id=evaluation.organization_id, - db=db, - provider=provider, - model=model, - credential_id=UUID(credential_id) if credential_id else None, - period_deltas=period_deltas, - ) - evaluation.prompt_improvements = prompt_improvements_state_to_db(state) - flag_modified(evaluation, "prompt_improvements") - db.commit() - logger.info( - "Prompt improvements completed for evaluation {} ({} suggestions)", - evaluation_id, - len(state.suggestions), - ) + from app.services.usage.call_import_context import usage_context_for_evaluation + from app.services.usage.context import llm_usage_context + + with llm_usage_context(usage_context_for_evaluation(evaluation)): + state = generate_prompt_improvements( + evaluation=evaluation, + imported_agent=imported_agent, + clusters_state=clusters_state, + organization_id=evaluation.organization_id, + db=db, + provider=provider, + model=model, + credential_id=UUID(credential_id) if credential_id else None, + period_deltas=period_deltas, + ) + evaluation.prompt_improvements = prompt_improvements_state_to_db(state) + flag_modified(evaluation, "prompt_improvements") + db.commit() + logger.info( + "Prompt improvements completed for evaluation {} ({} suggestions)", + evaluation_id, + len(state.suggestions), + ) except Exception as exc: logger.exception( "Prompt improvements failed for evaluation {}: {}", diff --git a/app/workers/tasks/generate_evaluation_tldr_insights.py b/app/workers/tasks/generate_evaluation_tldr_insights.py index b236d0f7..6f9e96a1 100644 --- a/app/workers/tasks/generate_evaluation_tldr_insights.py +++ b/app/workers/tasks/generate_evaluation_tldr_insights.py @@ -40,20 +40,18 @@ def generate_evaluation_tldr_insights_task( if evaluation is None: return {"error": "evaluation_not_found", "status_code": 404} - from app.services.usage.context import ( - LLMUsageContext, - LLMUsageProductSection, - llm_usage_context, + from app.services.usage.call_import_context import ( + call_import_evaluation_usage_context, ) + from app.services.usage.context import llm_usage_context try: with llm_usage_context( - LLMUsageContext( + call_import_evaluation_usage_context( organization_id=evaluation.organization_id, workspace_id=evaluation.workspace_id, - product_section=LLMUsageProductSection.CALL_IMPORT_EVALUATIONS, - resource_id=evaluation.id, - resource_type="call_import_evaluation", + evaluation_id=evaluation.id, + call_import_id=evaluation.call_import_id, ) ): summary = _generate_and_persist_tldr_summary( diff --git a/app/workers/tasks/generate_evaluation_user_insights.py b/app/workers/tasks/generate_evaluation_user_insights.py index 6029ce4a..71f666df 100644 --- a/app/workers/tasks/generate_evaluation_user_insights.py +++ b/app/workers/tasks/generate_evaluation_user_insights.py @@ -81,21 +81,25 @@ def on_progress(completed: int, total: int) -> None: flag_modified(evaluation, "user_insights") db.commit() - state = generate_user_insights( - db, - evaluation, - evaluation.organization_id, - provider_enum, - model_str, - completed_row_pairs=completed_pairs, - metrics=metrics, - aggregate=aggregate, - on_progress=on_progress, - max_llm_calls=max_llm_calls, - ) - evaluation.user_insights = user_insights_state_to_db(state) - flag_modified(evaluation, "user_insights") - db.commit() + from app.services.usage.call_import_context import usage_context_for_evaluation + from app.services.usage.context import llm_usage_context + + with llm_usage_context(usage_context_for_evaluation(evaluation)): + state = generate_user_insights( + db, + evaluation, + evaluation.organization_id, + provider_enum, + model_str, + completed_row_pairs=completed_pairs, + metrics=metrics, + aggregate=aggregate, + on_progress=on_progress, + max_llm_calls=max_llm_calls, + ) + evaluation.user_insights = user_insights_state_to_db(state) + flag_modified(evaluation, "user_insights") + db.commit() except Exception as exc: # noqa: BLE001 logger.exception( "User insights generation failed for evaluation {}: {}", diff --git a/app/workers/tasks/process_evaluator_result.py b/app/workers/tasks/process_evaluator_result.py index dc0e1b64..a86a4503 100644 --- a/app/workers/tasks/process_evaluator_result.py +++ b/app/workers/tasks/process_evaluator_result.py @@ -1,5 +1,6 @@ """Celery task: process evaluator result (transcribe and evaluate metrics).""" +import math import time import uuid as _uuid from uuid import UUID @@ -24,6 +25,11 @@ handle_llm_evaluation_error, ) from app.services.evaluators.evaluator_result_call_data import slim_call_data_for_evaluator_result +from app.services.evaluators.call_data_transcript import extract_transcript_from_call_data + + +class EvaluatorInputUnavailableError(ValueError): + """Permanent failure: no audio or transcript available for evaluation.""" def _commit_evaluator_result(db, result) -> None: @@ -288,6 +294,98 @@ def _run_bucket(bucket, parent_metric=None): return metric_scores, evaluation_time +_PERMANENT_VAPI_ENDED_REASONS = frozenset( + { + "call.in-progress.error-assistant-did-not-receive-customer-audio", + "call.in-progress.error-assistant-did-not-receive-customer-media", + "customer-did-not-give-microphone-permission", + "customer-did-not-answer", + } +) + + +def _call_data_for_transcript_extraction(result, db) -> tuple[dict, str]: + """Resolve provider payload and platform for transcript extraction.""" + platform = _normalize_platform(result.provider_platform) + recording = _playground_call_recording(db, result) if db is not None else None + if recording: + if not result.provider_call_id and recording.provider_call_id: + result.provider_call_id = recording.provider_call_id + if not result.provider_platform and recording.provider_platform: + result.provider_platform = recording.provider_platform + platform = _normalize_platform(result.provider_platform or recording.provider_platform) + + result_data = result.call_data if isinstance(result.call_data, dict) else {} + recording_data = ( + recording.call_data + if recording and isinstance(recording.call_data, dict) + else {} + ) + + if not result_data and not recording_data: + return {}, platform + + merged = dict(recording_data) + merged.update(result_data) + for key in ("endedReason", "messages", "transcript", "transcript_object"): + if not merged.get(key) and recording_data.get(key): + merged[key] = recording_data[key] + return merged, platform + + +def _hydrate_transcript_from_call_data(result, db) -> bool: + """Copy transcript from provider call_data onto the result row when missing.""" + if (result.transcription or "").strip(): + return True + + call_data, platform = _call_data_for_transcript_extraction(result, db) + if not call_data or not platform: + return False + + transcript_text, speaker_segments = extract_transcript_from_call_data( + call_data, platform + ) + if not (transcript_text or "").strip(): + return False + + result.transcription = transcript_text.strip() + if speaker_segments: + result.speaker_segments = speaker_segments + + if isinstance(result.call_data, dict) and result.call_data: + result.call_data = slim_call_data_for_evaluator_result(result.call_data) + + _commit_evaluator_result(db, result) + logger.info( + f"[EvaluatorResult {result.result_id}] Hydrated transcript from provider call_data" + ) + return True + + +def _permanent_input_failure_message(result, db) -> str: + """Return a user-facing error when retrying cannot succeed.""" + call_data, platform = _call_data_for_transcript_extraction(result, db) + if not isinstance(call_data, dict): + call_data = {} + + ended_reason = str( + call_data.get("endedReason") or call_data.get("ended_reason") or "" + ).strip() + if not platform and ended_reason.startswith("call."): + platform = "vapi" + + if platform == "vapi": + if ended_reason in _PERMANENT_VAPI_ENDED_REASONS: + return ( + "Call ended before customer audio was available " + f"({ended_reason}). Check microphone permissions and try again." + ) + if ended_reason: + return f"Call ended without usable audio or transcript ({ended_reason})." + + return "No audio or transcript available for evaluation." + + def _normalize_platform(platform: object) -> str: """Normalize provider platform enum/string into lowercase string.""" if not platform: @@ -394,7 +492,11 @@ def _recover_missing_audio_for_result(result, db, refresh_call_data: bool = True ) return False - headers = {"xi-api-key": decrypted_key} if platform == "elevenlabs" and decrypted_key else None + headers = None + if platform == "elevenlabs" and decrypted_key: + headers = {"xi-api-key": decrypted_key} + elif platform == "vapi" and decrypted_key: + headers = {"Authorization": f"Bearer {decrypted_key}"} try: response = _http.get(audio_url, headers=headers, timeout=120) except Exception as download_err: @@ -458,6 +560,71 @@ def _playground_call_recording(db, result): ) +_EXTERNAL_VOICE_PROVIDER_PLATFORMS = frozenset( + {"vapi", "retell", "elevenlabs", "smallest"} +) + + +def _should_record_external_agent_call_usage(result) -> bool: + """Only bill completed calls that ran on an external voice provider. + + Internal voice-bundle / WebSocket calls already emit LLM/STT/TTS usage + from the live pipeline; recording again here would double-count. + """ + if not result.agent_id: + return False + platform = (getattr(result, "provider_platform", None) or "").strip().lower() + return platform in _EXTERNAL_VOICE_PROVIDER_PLATFORMS + + +def _resolve_call_duration_seconds(result) -> int: + if result.duration_seconds: + try: + return max(0, int(math.ceil(float(result.duration_seconds)))) + except (TypeError, ValueError): + pass + call_data = result.call_data if isinstance(result.call_data, dict) else {} + for key in ("duration_seconds", "duration"): + raw = call_data.get(key) + if raw is not None: + try: + return max(0, int(math.ceil(float(raw)))) + except (TypeError, ValueError): + continue + started_at = call_data.get("startedAt") or call_data.get("start_timestamp") + ended_at = call_data.get("endedAt") or call_data.get("end_timestamp") + if started_at and ended_at: + try: + from datetime import datetime + + start = datetime.fromisoformat(str(started_at).replace("Z", "+00:00")) + end = datetime.fromisoformat(str(ended_at).replace("Z", "+00:00")) + return max(0, int(math.ceil((end - start).total_seconds()))) + except Exception: + return 0 + return 0 + + +def _record_agent_call_usage(result, *, usage_ctx) -> None: + if not _should_record_external_agent_call_usage(result): + return + try: + from app.services.usage.llm_usage import record_call_usage + + record_call_usage( + "voice-agent-call", + organization_id=result.organization_id, + ctx=usage_ctx, + audio_seconds=_resolve_call_duration_seconds(result), + ) + except Exception as exc: + logger.debug( + "[EvaluatorResult {}] agent call usage record skipped: {}", + result.result_id, + exc, + ) + + @celery_app.task(name="process_evaluator_result", bind=True, max_retries=3) def process_evaluator_result_task(self, result_id: str): """ @@ -489,283 +656,295 @@ def process_evaluator_result_task(self, result_id: str): return {"error": "Evaluator result not found"} from app.services.usage.context import ( - LLMUsageContext, - LLMUsageProductSection, - set_usage_context, - reset_usage_context, - ) - - usage_token = set_usage_context( - LLMUsageContext( - organization_id=result.organization_id, - workspace_id=result.workspace_id, - product_section=LLMUsageProductSection.EVALUATORS, - resource_id=result.id, - resource_type="evaluator_result", - ) + llm_usage_context, + usage_context_for_evaluator_result, ) - logger.info(f"[EvaluatorResult {result.result_id}] Starting processing task") + usage_ctx = usage_context_for_evaluator_result(result) + with llm_usage_context(usage_ctx): + logger.info(f"[EvaluatorResult {result.result_id}] Starting processing task") - result.celery_task_id = self.request.id - db.commit() + result.celery_task_id = self.request.id + db.commit() - try: - if not result.audio_s3_key: - _recover_missing_audio_for_result(result, db, refresh_call_data=True) - - has_existing_transcript = bool(result.transcription) - if not result.audio_s3_key and not has_existing_transcript: - raise ValueError("No audio S3 key or existing transcript found") + try: + if not result.audio_s3_key: + _recover_missing_audio_for_result(result, db, refresh_call_data=True) - evaluator, agent, persona, scenario = _load_related_entities(db, result) - is_custom_evaluator = evaluator and ( - bool(evaluator.custom_prompt) - or evaluator.agent_id is None - ) + _hydrate_transcript_from_call_data(result, db) - if not is_custom_evaluator and not agent: - raise ValueError("Agent not found and no custom prompt available") + has_existing_transcript = bool((result.transcription or "").strip()) + if not result.audio_s3_key and not has_existing_transcript: + raise EvaluatorInputUnavailableError( + _permanent_input_failure_message(result, db) + ) - ai_providers = db.query(AIProvider).filter( - AIProvider.organization_id == result.organization_id, - AIProvider.is_active == True, - ).all() + evaluator, agent, persona, scenario = _load_related_entities(db, result) + is_custom_evaluator = evaluator and ( + bool(evaluator.custom_prompt) + or evaluator.agent_id is None + ) - # Step 1: Transcription - if has_existing_transcript: - transcription = result.transcription - speaker_segments = result.speaker_segments or [] - transcription_time = 0.0 - else: - result.status = EvaluatorResultStatus.TRANSCRIBING.value - db.commit() + if not is_custom_evaluator and not agent: + raise ValueError("Agent not found and no custom prompt available") + + ai_providers = db.query(AIProvider).filter( + AIProvider.organization_id == result.organization_id, + AIProvider.is_active == True, + ).all() + + # Step 1: Transcription + if has_existing_transcript: + transcription = result.transcription + speaker_segments = result.speaker_segments or [] + transcription_time = 0.0 + else: + result.status = EvaluatorResultStatus.TRANSCRIBING.value + db.commit() - transcription, speaker_segments, transcription_time = _transcribe_audio( - result, ai_providers, db - ) - result.transcription = transcription - # Avoid duplicating transcript structure when provider call_data already carries it. - if not result.call_data: - result.speaker_segments = speaker_segments if speaker_segments else None - db.commit() + transcription, speaker_segments, transcription_time = _transcribe_audio( + result, ai_providers, db + ) + result.transcription = transcription + # Avoid duplicating transcript structure when provider call_data already carries it. + if not result.call_data: + result.speaker_segments = speaker_segments if speaker_segments else None + db.commit() - # Step 2: Load and categorize metrics - # Include metrics that have "agent" in their enabled_surfaces so users can - # restrict metrics to specific surfaces from the metrics page. Legacy rows - # (created before the surfaces column existed, or via fixtures that don't - # set the field) keep the original behavior: an enabled=True metric with - # an empty enabled_surfaces list is treated as agent-enabled. - enabled_metrics = db.query(Metric).filter( - Metric.organization_id == result.organization_id, - Metric.enabled == True, - ).all() - enabled_metrics = [ - m for m in enabled_metrics - if (m.name or "").strip().lower() not in REMOVED_EVALUATION_METRIC_NAMES - and ( - "agent" in (m.enabled_surfaces or []) - or not (m.enabled_surfaces or []) # legacy/unset → default to agent - ) - ] - - # Explicit metric selection on the evaluator/suite. Categorization - # parents are stored as a single ID and expanded to child labels here. - from app.services.evaluators.evaluator_helpers import expand_metric_ids_for_evaluation - - selected_metric_ids = { - str(mid) for mid in (getattr(evaluator, "metric_ids", None) or []) - } - if selected_metric_ids: - expanded_ids = expand_metric_ids_for_evaluation( - db, - result.organization_id, - list(selected_metric_ids), - ) or set() + # Step 2: Load and categorize metrics + # Include metrics that have "agent" in their enabled_surfaces so users can + # restrict metrics to specific surfaces from the metrics page. Legacy rows + # (created before the surfaces column existed, or via fixtures that don't + # set the field) keep the original behavior: an enabled=True metric with + # an empty enabled_surfaces list is treated as agent-enabled. + enabled_metrics = db.query(Metric).filter( + Metric.organization_id == result.organization_id, + Metric.enabled == True, + ).all() enabled_metrics = [ - m for m in enabled_metrics if str(m.id) in expanded_ids + m for m in enabled_metrics + if (m.name or "").strip().lower() not in REMOVED_EVALUATION_METRIC_NAMES + and ( + "agent" in (m.enabled_surfaces or []) + or not (m.enabled_surfaces or []) # legacy/unset → default to agent + ) ] - has_audio = bool(result.audio_s3_key) - llm_metrics, audio_metrics, metric_scores = _categorize_metrics(enabled_metrics, has_audio) - selected_metric_count = len(llm_metrics) + len(audio_metrics) + # Explicit metric selection on the evaluator/suite. Categorization + # parents are stored as a single ID and expanded to child labels here. + from app.services.evaluators.evaluator_helpers import expand_metric_ids_for_evaluation - call_recording = _playground_call_recording(db, result) - if call_recording: - from app.services.billing.flexprice_service import ( - record_playground_call_evaluated, - ) + selected_metric_ids = { + str(mid) for mid in (getattr(evaluator, "metric_ids", None) or []) + } + if selected_metric_ids: + expanded_ids = expand_metric_ids_for_evaluation( + db, + result.organization_id, + list(selected_metric_ids), + ) or set() + enabled_metrics = [ + m for m in enabled_metrics if str(m.id) in expanded_ids + ] + + has_audio = bool(result.audio_s3_key) + llm_metrics, audio_metrics, metric_scores = _categorize_metrics(enabled_metrics, has_audio) + selected_metric_count = len(llm_metrics) + len(audio_metrics) + + call_recording = _playground_call_recording(db, result) + if call_recording: + from app.services.billing.flexprice_service import ( + record_playground_call_evaluated, + ) - evaluation_attempt_id = f"{result.id}:{self.request.id}" - record_playground_call_evaluated( - result.organization_id, - evaluation_attempt_id, - evaluator_result_id=result.id, - workspace_id=result.workspace_id, - call_short_id=call_recording.call_short_id, - metric_count=selected_metric_count, - ) + evaluation_attempt_id = f"{result.id}:{self.request.id}" + record_playground_call_evaluated( + result.organization_id, + evaluation_attempt_id, + evaluator_result_id=result.id, + workspace_id=result.workspace_id, + call_short_id=call_recording.call_short_id, + metric_count=selected_metric_count, + ) - evaluation_time = None + evaluation_time = None - # Step 3: Audio metrics evaluation - if audio_metrics and has_audio: - try: - audio_scores = evaluate_audio_metrics( - audio_s3_key=result.audio_s3_key, - audio_metrics=audio_metrics, - result_id=result.result_id, - ) + # Step 3: Audio metrics evaluation + if audio_metrics and has_audio: + try: + audio_scores = evaluate_audio_metrics( + audio_s3_key=result.audio_s3_key, + audio_metrics=audio_metrics, + result_id=result.result_id, + ) - if _all_audio_scores_download_failed(audio_scores): + if _all_audio_scores_download_failed(audio_scores): + logger.warning( + f"[EvaluatorResult {result.result_id}] Existing S3 audio unavailable; " + "attempting provider audio recovery" + ) + recovered = _recover_missing_audio_for_result(result, db, refresh_call_data=True) + if recovered and result.audio_s3_key: + audio_scores = evaluate_audio_metrics( + audio_s3_key=result.audio_s3_key, + audio_metrics=audio_metrics, + result_id=result.result_id, + ) + + metric_scores.update(audio_scores) + except Exception as audio_err: + logger.error( + f"[EvaluatorResult {result.result_id}] Audio analysis failed: {audio_err}", + exc_info=True, + ) + metric_scores.update(handle_audio_evaluation_error(audio_metrics, audio_err)) + + # Step 4: LLM metrics evaluation + if llm_metrics and transcription: + result.status = EvaluatorResultStatus.EVALUATING.value + db.commit() + + try: + llm_scores, evaluation_time = _evaluate_llm_metrics_grouped( + transcription=transcription, + llm_metrics=llm_metrics, + ai_providers=ai_providers, + organization_id=result.organization_id, + result_id=result.result_id, + db=db, + evaluator=evaluator, + agent=agent, + persona=persona, + scenario=scenario, + ) + metric_scores.update(llm_scores) + except Exception as llm_err: + error_msg = str(llm_err).replace("{", "{{").replace("}", "}}") + logger.error( + f"[EvaluatorResult {result.result_id}] ✗ LLM evaluation failed: {error_msg}", + exc_info=True, + ) + metric_scores.update(handle_llm_evaluation_error(llm_metrics, llm_err)) + else: + if not llm_metrics: logger.warning( - f"[EvaluatorResult {result.result_id}] Existing S3 audio unavailable; " - "attempting provider audio recovery" + f"[EvaluatorResult {result.result_id}] No LLM-evaluable metrics found " + "(audio-only metrics were skipped), skipping evaluation" + ) + if not transcription: + logger.warning( + f"[EvaluatorResult {result.result_id}] No transcription available, " + "skipping evaluation" ) - recovered = _recover_missing_audio_for_result(result, db, refresh_call_data=True) - if recovered and result.audio_s3_key: - audio_scores = evaluate_audio_metrics( - audio_s3_key=result.audio_s3_key, - audio_metrics=audio_metrics, - result_id=result.result_id, - ) - metric_scores.update(audio_scores) - except Exception as audio_err: - logger.error( - f"[EvaluatorResult {result.result_id}] Audio analysis failed: {audio_err}", - exc_info=True, - ) - metric_scores.update(handle_audio_evaluation_error(audio_metrics, audio_err)) + # Step 5: Call Analysis + if transcription and not (result.call_data and result.call_data.get("call_analysis")): + try: + call_analysis = _generate_call_analysis( + transcription=transcription, + ai_providers=ai_providers, + organization_id=result.organization_id, + result_id=result.result_id, + db=db, + agent=agent, + scenario=scenario, + ) + if call_analysis: + existing_call_data = dict(result.call_data) if isinstance(result.call_data, dict) else {} + existing_call_data["call_analysis"] = call_analysis + result.call_data = slim_call_data_for_evaluator_result(existing_call_data) + except Exception as analysis_err: + logger.warning( + f"[EvaluatorResult {result.result_id}] Call analysis failed (non-fatal): {analysis_err}" + ) - # Step 4: LLM metrics evaluation - if llm_metrics and transcription: - result.status = EvaluatorResultStatus.EVALUATING.value - db.commit() + # Step 6: Complete + from sqlalchemy.orm.attributes import flag_modified + + result.metric_scores = _make_json_serializable(metric_scores) + flag_modified(result, "metric_scores") + if isinstance(result.call_data, dict): + result.call_data = slim_call_data_for_evaluator_result(result.call_data) + if isinstance(result.call_data, (dict, list)): + result.call_data = _make_json_serializable(result.call_data) + flag_modified(result, "call_data") + result.status = EvaluatorResultStatus.COMPLETED.value + result.error_message = None + _record_agent_call_usage(result, usage_ctx=usage_ctx) + _commit_evaluator_result(db, result) - try: - llm_scores, evaluation_time = _evaluate_llm_metrics_grouped( - transcription=transcription, - llm_metrics=llm_metrics, - ai_providers=ai_providers, - organization_id=result.organization_id, - result_id=result.result_id, - db=db, - evaluator=evaluator, - agent=agent, - persona=persona, - scenario=scenario, - ) - metric_scores.update(llm_scores) - except Exception as llm_err: - error_msg = str(llm_err).replace("{", "{{").replace("}", "}}") - logger.error( - f"[EvaluatorResult {result.result_id}] ✗ LLM evaluation failed: {error_msg}", - exc_info=True, - ) - metric_scores.update(handle_llm_evaluation_error(llm_metrics, llm_err)) - else: - if not llm_metrics: - logger.warning( - f"[EvaluatorResult {result.result_id}] No LLM-evaluable metrics found " - "(audio-only metrics were skipped), skipping evaluation" - ) - if not transcription: - logger.warning( - f"[EvaluatorResult {result.result_id}] No transcription available, " - "skipping evaluation" - ) + from app.services.billing.flexprice_service import ( + record_playground_evaluation_completed, + ) - # Step 5: Call Analysis - if transcription and not (result.call_data and result.call_data.get("call_analysis")): - try: - call_analysis = _generate_call_analysis( - transcription=transcription, - ai_providers=ai_providers, - organization_id=result.organization_id, - result_id=result.result_id, - db=db, - agent=agent, - scenario=scenario, - ) - if call_analysis: - existing_call_data = dict(result.call_data) if isinstance(result.call_data, dict) else {} - existing_call_data["call_analysis"] = call_analysis - result.call_data = slim_call_data_for_evaluator_result(existing_call_data) - except Exception as analysis_err: - logger.warning( - f"[EvaluatorResult {result.result_id}] Call analysis failed (non-fatal): {analysis_err}" + call_recording = _playground_call_recording(db, result) + if call_recording: + record_playground_evaluation_completed( + result.organization_id, + f"{result.id}:{self.request.id}", + evaluator_result_id=result.id, + workspace_id=result.workspace_id, + call_short_id=call_recording.call_short_id, + duration_seconds=result.duration_seconds, + metric_count=len(metric_scores) or selected_metric_count, ) - # Step 6: Complete - from sqlalchemy.orm.attributes import flag_modified - - result.metric_scores = _make_json_serializable(metric_scores) - flag_modified(result, "metric_scores") - if isinstance(result.call_data, dict): - result.call_data = slim_call_data_for_evaluator_result(result.call_data) - if isinstance(result.call_data, (dict, list)): - result.call_data = _make_json_serializable(result.call_data) - flag_modified(result, "call_data") - result.status = EvaluatorResultStatus.COMPLETED.value - result.error_message = None - _commit_evaluator_result(db, result) - - from app.services.billing.flexprice_service import ( - record_playground_evaluation_completed, - ) - - call_recording = _playground_call_recording(db, result) - if call_recording: - record_playground_evaluation_completed( - result.organization_id, - f"{result.id}:{self.request.id}", - evaluator_result_id=result.id, - workspace_id=result.workspace_id, - call_short_id=call_recording.call_short_id, - duration_seconds=result.duration_seconds, - metric_count=len(metric_scores) or selected_metric_count, + total_time = time.time() - task_start_time + logger.info( + f"[EvaluatorResult {result.result_id}] Completed in {total_time:.2f}s, " + f"{len(metric_scores)} metrics evaluated" ) - total_time = time.time() - task_start_time - logger.info( - f"[EvaluatorResult {result.result_id}] Completed in {total_time:.2f}s, " - f"{len(metric_scores)} metrics evaluated" - ) + return { + "result_id": result_id, + "status": "completed", + "transcription": transcription, + "metrics_evaluated": len(metric_scores), + "processing_time": total_time, + "transcription_time": transcription_time, + "evaluation_time": evaluation_time, + } - return { - "result_id": result_id, - "status": "completed", - "transcription": transcription, - "metrics_evaluated": len(metric_scores), - "processing_time": total_time, - "transcription_time": transcription_time, - "evaluation_time": evaluation_time, - } - - except Exception as e: - db.rollback() - logger.error(f"[EvaluatorResult {result_id}] Processing failed: {e}", exc_info=True) - try: - failed_result = db.query(EvaluatorResult).filter(EvaluatorResult.id == result_uuid).first() - if failed_result: - failed_result.status = EvaluatorResultStatus.FAILED.value - failed_result.error_message = str(e) - db.commit() - except Exception as persist_err: + except EvaluatorInputUnavailableError: db.rollback() - logger.error( - f"[EvaluatorResult {result_id}] Failed to persist FAILED status: {persist_err}", - exc_info=True, - ) - raise + raise + except Exception as e: + db.rollback() + logger.error(f"[EvaluatorResult {result_id}] Processing failed: {e}", exc_info=True) + try: + failed_result = db.query(EvaluatorResult).filter(EvaluatorResult.id == result_uuid).first() + if failed_result: + failed_result.status = EvaluatorResultStatus.FAILED.value + failed_result.error_message = str(e) + db.commit() + except Exception as persist_err: + db.rollback() + logger.error( + f"[EvaluatorResult {result_id}] Failed to persist FAILED status: {persist_err}", + exc_info=True, + ) + raise + except EvaluatorInputUnavailableError as exc: + logger.warning(f"[EvaluatorResult {result_id}] Input unavailable: {exc}") + try: + from app.models.database import EvaluatorResult, EvaluatorResultStatus + + failed_result = db.query(EvaluatorResult).filter( + EvaluatorResult.id == result_uuid + ).first() + if failed_result: + failed_result.status = EvaluatorResultStatus.FAILED.value + failed_result.error_message = str(exc) + db.commit() + except Exception as persist_err: + db.rollback() + logger.error( + f"[EvaluatorResult {result_id}] Failed to persist FAILED status: {persist_err}", + exc_info=True, + ) + return {"error": str(exc), "status": "failed"} except Exception as exc: raise self.retry(exc=exc, countdown=60) finally: - try: - if "usage_token" in locals() and usage_token is not None: - reset_usage_context(usage_token) - except Exception: - pass db.close() diff --git a/app/workers/tasks/run_judge_alignment.py b/app/workers/tasks/run_judge_alignment.py index 6ffbb655..053546cc 100644 --- a/app/workers/tasks/run_judge_alignment.py +++ b/app/workers/tasks/run_judge_alignment.py @@ -79,7 +79,13 @@ def run_judge_alignment_task( return {"error": "No samples"} try: - metrics = run_judge(run, dataset, evaluator, samples, db) + from app.services.usage.context import ( + llm_usage_context, + usage_context_for_judge_run, + ) + + with llm_usage_context(usage_context_for_judge_run(run)): + metrics = run_judge(run, dataset, evaluator, samples, db) except Exception as exc: logger.error( f"[JudgeAlignment] Run {judge_run_id} crashed: {exc}", diff --git a/app/workers/tasks/run_prompt_optimization.py b/app/workers/tasks/run_prompt_optimization.py index dbd9c869..b7efb247 100644 --- a/app/workers/tasks/run_prompt_optimization.py +++ b/app/workers/tasks/run_prompt_optimization.py @@ -106,19 +106,24 @@ def run_prompt_optimization_task(self, optimization_run_id: str): ) from app.services.optimization import run_optimization - - result = run_optimization( - agent=agent, - evaluator=evaluator, - voice_bundle=voice_bundle, - training_data=training_data, - metrics=enabled_metrics, - ai_providers=ai_providers, - organization_id=run.organization_id, - db=db, - config=run.config, + from app.services.usage.context import ( + llm_usage_context, + usage_context_for_prompt_optimization_run, ) + with llm_usage_context(usage_context_for_prompt_optimization_run(run)): + result = run_optimization( + agent=agent, + evaluator=evaluator, + voice_bundle=voice_bundle, + training_data=training_data, + metrics=enabled_metrics, + ai_providers=ai_providers, + organization_id=run.organization_id, + db=db, + config=run.config, + ) + run.best_prompt = result["best_candidate"] run.best_score = result["best_score"] run.metric_history = result["metric_history"] diff --git a/app/workers/tasks/transcribe_call_import_row.py b/app/workers/tasks/transcribe_call_import_row.py index 0f342ab3..bd09bf25 100644 --- a/app/workers/tasks/transcribe_call_import_row.py +++ b/app/workers/tasks/transcribe_call_import_row.py @@ -390,12 +390,11 @@ def _persist_diarization_failure( def _run_diarization_pipeline(ctx: dict[str, Any]) -> dict[str, Any]: """STT / S3 / LLM diarisation without a long-lived DB session.""" + from uuid import UUID + from app.models.enums import ModelProvider - from app.services.usage.context import ( - LLMUsageContext, - LLMUsageProductSection, - llm_usage_context, - ) + from app.services.usage.call_import_context import call_import_row_usage_context + from app.services.usage.context import llm_usage_context from app.workers.tasks.helpers.llm_diarisation import ( LLMDiarisationError, diarize_audio_with_llm, @@ -411,13 +410,20 @@ def _run_diarization_pipeline(ctx: dict[str, Any]) -> dict[str, Any]: llm_credential_uuid = ctx["llm_credential_uuid"] effective_prompt = ctx["effective_prompt"] + evaluation_id = ctx.get("evaluation_id") + evaluation_row_id = ctx.get("evaluation_row_id") + call_import_id = ctx.get("call_import_id") + if not call_import_id: + raise ValueError("call_import_id missing from transcribe pipeline context") + with llm_usage_context( - LLMUsageContext( + call_import_row_usage_context( organization_id=organization_id, workspace_id=ctx.get("workspace_id"), - product_section=LLMUsageProductSection.CALL_IMPORTS, - resource_id=ctx.get("call_import_id"), - resource_type="call_import", + call_import_id=call_import_id, + call_import_row_id=UUID(str(row_id)), + evaluation_id=evaluation_id, + evaluation_row_id=evaluation_row_id, ) ): return _run_diarization_pipeline_inner( @@ -960,6 +966,14 @@ def transcribe_call_import_row_task( "organization_id": row.organization_id, "workspace_id": getattr(row, "workspace_id", None), "call_import_id": row.call_import_id, + "evaluation_id": ( + UUID(evaluation_id_for_dispatch) + if evaluation_id_for_dispatch + else None + ), + "evaluation_row_id": ( + UUID(run_eval_row_id) if run_eval_row_id else None + ), "stt_provider": provider_enum.value if provider_enum else None, "stt_model": stt_model, "credential_uuid": credential_uuid, diff --git a/app/workers/tasks/tts_comparison.py b/app/workers/tasks/tts_comparison.py index 8ae42ea9..2c8968ac 100644 --- a/app/workers/tasks/tts_comparison.py +++ b/app/workers/tasks/tts_comparison.py @@ -180,6 +180,7 @@ def generate_tts_comparison_task(self, comparison_id: str): ) from app.services.ai.tts_service import tts_service, get_audio_file_extension from app.services.storage.s3_service import s3_service + from app.services.usage.context import LLMUsageContext, LLMUsageProductSection, llm_usage_context db = SessionLocal() try: @@ -258,15 +259,24 @@ def _resolve_voice_meta(sample_obj): f"provider={sample.provider} voice={sample.voice_id} " f"sample_rate_hz={sample_rate_hz} config={tts_config}" ) - audio_bytes, latency_ms, ttfb_ms = tts_service.synthesize_timed( - text=sample.text, - tts_provider=provider_enum, - tts_model=sample.model, - organization_id=comp.organization_id, - db=db, - voice=sample.voice_id, - config=tts_config or None, - ) + with llm_usage_context( + LLMUsageContext( + organization_id=comp.organization_id, + workspace_id=comp.workspace_id, + product_section=LLMUsageProductSection.VOICE_PLAYGROUND, + resource_id=comp.id, + resource_type="tts_comparison", + ) + ): + audio_bytes, latency_ms, ttfb_ms = tts_service.synthesize_timed( + text=sample.text, + tts_provider=provider_enum, + tts_model=sample.model, + organization_id=comp.organization_id, + db=db, + voice=sample.voice_id, + config=tts_config or None, + ) audio_ext = get_audio_file_extension( sample.provider, int(sample_rate_hz) if sample_rate_hz else None @@ -514,6 +524,20 @@ def evaluate_tts_comparison_task(self, comparison_id: str): db.commit() return {"evaluated": 0} + from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + llm_usage_context, + ) + + usage_ctx = LLMUsageContext( + organization_id=comp.organization_id, + workspace_id=comp.workspace_id, + product_section=LLMUsageProductSection.VOICE_PLAYGROUND, + resource_id=comp.id, + resource_type="tts_comparison", + ) + stt_provider_str, stt_model = _resolve_stt_config(comp, db) stt_available = bool(stt_provider_str and stt_model) if stt_available: @@ -535,86 +559,86 @@ def evaluate_tts_comparison_task(self, comparison_id: str): ) evaluated = 0 - for sample in samples: - tmp_path = None - try: - audio_bytes = s3_service.download_file_by_key(sample.audio_s3_key) - if not audio_bytes: - continue - - ext = ".mp3" - if sample.audio_s3_key: - key_ext = os.path.splitext(sample.audio_s3_key)[1].lower() - if key_ext in {".wav", ".mp3", ".flac", ".ogg", ".m4a"}: - ext = key_ext - tmp_fd, tmp_path = tempfile.mkstemp(suffix=ext) - os.close(tmp_fd) - with open(tmp_path, "wb") as f: - f.write(audio_bytes) - - if any_qualitative_enabled: - metrics = qualitative_voice_service.calculate_all_metrics(tmp_path) - metrics = _filter_qualitative_metrics(metrics, enabled_voice_metric_names) - else: - metrics = {} - - if stt_available and sample.text: - selected_stt_provider = ModelProvider(stt_provider_str) - selected_language = None - if selected_stt_provider == ModelProvider.SARVAM: - # Sarvam saarika models accept language_code; saaras models do not. - if "saarika" in (stt_model or "").lower(): - selected_language = "hi-IN" - asr_transcript = transcription_service.transcribe_text_only( - audio_file_path=tmp_path, - stt_provider=selected_stt_provider, - stt_model=stt_model, - organization_id=comp.organization_id, - db=db, - language=selected_language, - ) - if asr_transcript: - score_bundle = _compute_wer_cer(sample.text, asr_transcript) - metrics["WER Raw"] = score_bundle.get("raw_wer") - metrics["CER Raw"] = score_bundle.get("raw_cer") - metrics["WER Normalized"] = score_bundle.get("normalized_wer") - metrics["CER Normalized"] = score_bundle.get("normalized_cer") - metrics["WER"] = ( - score_bundle.get("normalized_wer") - if score_bundle.get("normalized_wer") is not None - else score_bundle.get("raw_wer") - ) - metrics["CER"] = ( - score_bundle.get("normalized_cer") - if score_bundle.get("normalized_cer") is not None - else score_bundle.get("raw_cer") - ) - metrics["ASR Transcript"] = asr_transcript + with llm_usage_context(usage_ctx): + for sample in samples: + tmp_path = None + try: + audio_bytes = s3_service.download_file_by_key(sample.audio_s3_key) + if not audio_bytes: + continue + + ext = ".mp3" + if sample.audio_s3_key: + key_ext = os.path.splitext(sample.audio_s3_key)[1].lower() + if key_ext in {".wav", ".mp3", ".flac", ".ogg", ".m4a"}: + ext = key_ext + tmp_fd, tmp_path = tempfile.mkstemp(suffix=ext) + os.close(tmp_fd) + with open(tmp_path, "wb") as f: + f.write(audio_bytes) + + if any_qualitative_enabled: + metrics = qualitative_voice_service.calculate_all_metrics(tmp_path) + metrics = _filter_qualitative_metrics(metrics, enabled_voice_metric_names) else: - metrics["WER"] = None - metrics["CER"] = None - metrics["WER Raw"] = None - metrics["CER Raw"] = None - metrics["WER Normalized"] = None - metrics["CER Normalized"] = None - metrics["ASR Transcript"] = None - - _evaluate_custom_voice_metrics(sample, metrics, comp, db) - - sample.evaluation_metrics = _sanitize_metrics(metrics) - db.commit() - evaluated += 1 - - logger.info(f"[TTS Eval] Sample {sample.id} metrics: {metrics}") - - except Exception as e: - logger.warning("[TTS Eval] Sample {} eval failed: {}", sample.id, e) - finally: - if tmp_path and os.path.exists(tmp_path): - try: - os.unlink(tmp_path) - except Exception: - pass + metrics = {} + + if stt_available and sample.text: + selected_stt_provider = ModelProvider(stt_provider_str) + selected_language = None + if selected_stt_provider == ModelProvider.SARVAM: + if "saarika" in (stt_model or "").lower(): + selected_language = "hi-IN" + asr_transcript = transcription_service.transcribe_text_only( + audio_file_path=tmp_path, + stt_provider=selected_stt_provider, + stt_model=stt_model, + organization_id=comp.organization_id, + db=db, + language=selected_language, + ) + if asr_transcript: + score_bundle = _compute_wer_cer(sample.text, asr_transcript) + metrics["WER Raw"] = score_bundle.get("raw_wer") + metrics["CER Raw"] = score_bundle.get("raw_cer") + metrics["WER Normalized"] = score_bundle.get("normalized_wer") + metrics["CER Normalized"] = score_bundle.get("normalized_cer") + metrics["WER"] = ( + score_bundle.get("normalized_wer") + if score_bundle.get("normalized_wer") is not None + else score_bundle.get("raw_wer") + ) + metrics["CER"] = ( + score_bundle.get("normalized_cer") + if score_bundle.get("normalized_cer") is not None + else score_bundle.get("raw_cer") + ) + metrics["ASR Transcript"] = asr_transcript + else: + metrics["WER"] = None + metrics["CER"] = None + metrics["WER Raw"] = None + metrics["CER Raw"] = None + metrics["WER Normalized"] = None + metrics["CER Normalized"] = None + metrics["ASR Transcript"] = None + + _evaluate_custom_voice_metrics(sample, metrics, comp, db) + + sample.evaluation_metrics = _sanitize_metrics(metrics) + db.commit() + evaluated += 1 + + logger.info(f"[TTS Eval] Sample {sample.id} metrics: {metrics}") + + except Exception as e: + logger.warning("[TTS Eval] Sample {} eval failed: {}", sample.id, e) + finally: + if tmp_path and os.path.exists(tmp_path): + try: + os.unlink(tmp_path) + except Exception: + pass from app.api.v1.routes.voice_playground import _recompute_summary diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index f8701723..7e2c4369 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -2994,10 +2994,16 @@ class ApiClient { async getOrgUsageSummary(params: { start?: string end?: string + tz?: string workspace_id?: string product_section?: string model?: string resource_id?: string + usage_kind?: string + call_import_id?: string + evaluation_id?: string + dataset?: string + tag_id?: string }): Promise<{ start: string end: string @@ -3008,6 +3014,8 @@ class ApiClient { cache_read_tokens: number cache_creation_tokens: number reasoning_tokens: number + audio_seconds: number + tts_characters: number call_count: number } last_updated_at?: string | null @@ -3021,11 +3029,17 @@ class ApiClient { async getOrgUsageBreakdown(params: { start?: string end?: string - group_by?: 'workspace' | 'product_section' | 'model' | 'resource' + tz?: string + group_by?: 'workspace' | 'product_section' | 'model' | 'resource' | 'usage_kind' | 'call_import' workspace_id?: string product_section?: string model?: string resource_id?: string + usage_kind?: string + call_import_id?: string + evaluation_id?: string + dataset?: string + tag_id?: string limit?: number offset?: number }): Promise<{ @@ -3041,15 +3055,21 @@ class ApiClient { resource_id?: string | null resource_type?: string | null resource_label?: string | null + call_import_id?: string | null + call_import_label?: string | null + usage_kind?: string | null prompt_tokens: number completion_tokens: number total_tokens: number cache_read_tokens: number cache_creation_tokens: number reasoning_tokens: number + audio_seconds: number + tts_characters: number call_count: number }> total_count: number + truncated_at_limit?: boolean last_updated_at?: string | null }> { const response = await this.client.get('/api/v1/organizations/usage/breakdown', { @@ -3061,11 +3081,27 @@ class ApiClient { async getOrgUsageFilters(params?: { start?: string end?: string + tz?: string + workspace_id?: string + product_section?: string + model?: string + resource_id?: string + evaluation_id?: string + usage_kind?: string + call_import_id?: string + q?: string + dataset?: string + tag_id?: string }): Promise<{ workspaces: Array<{ id: string; name: string }> product_sections: Array<{ id: string; label: string }> + call_imports: Array<{ id: string; label: string }> + evaluations: Array<{ id: string; label: string }> models: string[] - resources: Array<{ id: string; type?: string; label: string }> + resources: Array<{ id: string; type?: string; label: string; product_section?: string }> + usage_kinds: Array<{ id: string; label: string }> + datasets: string[] + tags: Array<{ id: string; label: string }> }> { const response = await this.client.get('/api/v1/organizations/usage/filters', { params, @@ -3237,7 +3273,7 @@ class ApiClient { }): Promise<{ call_type: string access_token?: string - call_id: string + call_id?: string agent_id: string agent_version?: number call_status?: string diff --git a/frontend/src/pages/agents/components/AgentTalkSidebar.tsx b/frontend/src/pages/agents/components/AgentTalkSidebar.tsx index be2231e7..1afd845d 100644 --- a/frontend/src/pages/agents/components/AgentTalkSidebar.tsx +++ b/frontend/src/pages/agents/components/AgentTalkSidebar.tsx @@ -59,6 +59,7 @@ export default function AgentTalkSidebar({ const userInitiatedDisconnectRef = useRef(false) const wasOpenRef = useRef(false) const userSpeakingTimeoutRef = useRef | null>(null) + const callShortIdRef = useRef(null) const pulseUserSpeaking = (durationMs = 1200) => { setActiveSpeaker('user') @@ -205,9 +206,16 @@ export default function AgentTalkSidebar({ }) } else if (isVapi) { const client = vapiClientRef.current! - client.on('call-start', () => { + client.on('call-start', async (call: any) => { setIsConnected(true) setIsConnecting(false) + if (callShortIdRef.current && call?.id) { + try { + await apiClient.updateCallRecording(callShortIdRef.current, call.id) + } catch (err) { + console.error('Failed to update Vapi call recording', err) + } + } }) client.on('speech-start', () => setActiveSpeaker('agent')) client.on('speech-end', () => setActiveSpeaker((prev) => (prev === 'agent' ? null : prev))) @@ -224,13 +232,26 @@ export default function AgentTalkSidebar({ } } }) - client.on('call-end', () => { + client.on('call-end', async () => { setIsConnected(false) setIsConnecting(false) setActiveSpeaker(null) + if (callShortIdRef.current) { + apiClient.refreshCallRecording(callShortIdRef.current).catch((err) => { + console.error('Failed to refresh Vapi call recording', err) + }) + } }) - await apiClient.createWebCall({ agent_id: agent.id, metadata: {} }) - await client.start(agent.voice_ai_agent_id!) + const webCall = await apiClient.createWebCall({ agent_id: agent.id, metadata: {} }) + callShortIdRef.current = webCall.call_short_id ?? null + const vapiCall = await client.start(agent.voice_ai_agent_id!) + if (callShortIdRef.current && vapiCall?.id) { + try { + await apiClient.updateCallRecording(callShortIdRef.current, vapiCall.id) + } catch (err) { + console.error('Failed to update Vapi call recording from start()', err) + } + } } else if (isElevenLabs) { const webCall = await apiClient.createWebCall({ agent_id: agent.id, metadata: {} }) if (!webCall.signed_url) throw new Error('No signed URL') diff --git a/frontend/src/pages/usage/SearchableSelect.tsx b/frontend/src/pages/usage/SearchableSelect.tsx new file mode 100644 index 00000000..66f625dd --- /dev/null +++ b/frontend/src/pages/usage/SearchableSelect.tsx @@ -0,0 +1,125 @@ +import { useEffect, useMemo, useRef, useState } from 'react' +import { ChevronDown, Search, X } from 'lucide-react' +import { usageTheme } from './usageTheme' + +export type SearchableOption = { id: string; label: string } + +type SearchableSelectProps = { + label: string + placeholder: string + value: string + options: SearchableOption[] + onChange: (id: string) => void + disabled?: boolean + emptyMessage?: string +} + +export default function SearchableSelect({ + label, + placeholder, + value, + options, + onChange, + disabled, + emptyMessage = 'No matches', +}: SearchableSelectProps) { + const [open, setOpen] = useState(false) + const [search, setSearch] = useState('') + const rootRef = useRef(null) + + const selected = options.find((o) => o.id === value) + + const filtered = useMemo(() => { + const q = search.trim().toLowerCase() + if (!q) return options + return options.filter((o) => o.label.toLowerCase().includes(q)) + }, [options, search]) + + useEffect(() => { + const onDoc = (e: MouseEvent) => { + if (!rootRef.current?.contains(e.target as Node)) setOpen(false) + } + document.addEventListener('mousedown', onDoc) + return () => document.removeEventListener('mousedown', onDoc) + }, []) + + return ( +
+ {label} + + + {open && !disabled ? ( +
+
+ + setSearch(e.target.value)} + placeholder="Search…" + className="flex-1 text-sm outline-none" + autoFocus + /> +
+
    + {filtered.length === 0 ? ( +
  • {emptyMessage}
    • + ) : ( + filtered.map((opt) => ( +
  • + +
    • + )) + )} +
+
+ ) : null} +
+ ) +} diff --git a/frontend/src/pages/usage/Usage.tsx b/frontend/src/pages/usage/Usage.tsx index 0f7cdcb9..c97b6f7c 100644 --- a/frontend/src/pages/usage/Usage.tsx +++ b/frontend/src/pages/usage/Usage.tsx @@ -1,100 +1,974 @@ -import { useMemo } from 'react' -import { useQuery } from '@tanstack/react-query' +import { useEffect, useMemo } from 'react' +import { keepPreviousData, useQuery } from '@tanstack/react-query' import { useSearchParams } from 'react-router-dom' import { Card, CardBody, Spinner } from '@heroui/react' -import { Activity } from 'lucide-react' +import { Activity, ChevronRight } from 'lucide-react' import { apiClient } from '../../lib/api' +import UsageFiltersBar from './UsageFiltersBar' +import UsageDrillPath from './UsageDrillPath' +import { defaultUsageDateRange } from './UsageDateRangePicker' +import { getUsageTimezone } from './usageTimezone' +import { + CALL_IMPORT_BATCH_HEADLINE, + CALL_IMPORT_HINT, + CALL_IMPORT_PRODUCT_SECTIONS, + PRODUCT_SECTION_HEADLINES, + PRODUCT_SECTION_HINTS, +} from './usageProductHints' -type GroupBy = 'workspace' | 'product_section' | 'model' | 'resource' +type DrillGroupBy = + | 'workspace' + | 'call_import' + | 'resource' + | 'model' + | 'usage_kind' + | 'product_section' +type Kind = '' | 'llm' | 'stt' | 'tts' + +type FilterOptions = { + workspaces: Array<{ id: string; name: string }> + call_imports: Array<{ id: string; label: string }> + evaluations: Array<{ id: string; label: string }> + resources?: Array<{ id: string; label: string; type?: string; product_section?: string }> + models: string[] + usage_kinds: Array<{ id: string; label: string }> + product_sections?: Array<{ id: string; label: string }> + datasets?: string[] + tags?: Array<{ id: string; label: string }> +} + +type BreakdownRow = { + workspace_id?: string | null + workspace_name?: string | null + call_import_id?: string | null + call_import_label?: string | null + resource_id?: string | null + resource_type?: string | null + resource_label?: string | null + model?: string | null + usage_kind?: string | null + product_section?: string | null + product_section_label?: string | null + prompt_tokens: number + completion_tokens: number + total_tokens: number + cache_read_tokens: number + cache_creation_tokens: number + reasoning_tokens: number + audio_seconds: number + tts_characters: number + call_count: number +} + +type WorkspaceSourceRow = BreakdownRow & { + rowKind: 'call_import' | 'workspace_resource' + hint: string +} + +const NON_COMPOSITE_RESOURCE_TYPES = new Set([ + 'call_import', + 'call_import_evaluation', +]) + +type TableRow = BreakdownRow | WorkspaceSourceRow + +const EMPTY_METRICS = { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + cache_read_tokens: 0, + cache_creation_tokens: 0, + reasoning_tokens: 0, + audio_seconds: 0, + tts_characters: 0, + call_count: 0, +} + +function compositeRowHeadline(row: WorkspaceSourceRow): string { + if (row.rowKind === 'call_import') return CALL_IMPORT_BATCH_HEADLINE + const section = row.product_section || '' + return ( + PRODUCT_SECTION_HEADLINES[section] || + row.product_section_label || + section || + 'Other' + ) +} + +function usableResourceLabel(label: string | null | undefined): string | undefined { + if (!label || label === 'Unscoped') return undefined + return label +} + +function compositeRowTitle(row: WorkspaceSourceRow, options?: FilterOptions): string { + if (row.rowKind === 'call_import') { + return row.call_import_label || 'Call import batch' + } + const fromFilters = row.resource_id + ? options?.resources?.find((r) => idKey(r.id) === idKey(row.resource_id))?.label + : undefined + return ( + usableResourceLabel(row.resource_label) || + fromFilters || + row.product_section_label || + 'Unscoped usage' + ) +} + +function sortRows(rows: BreakdownRow[]): BreakdownRow[] { + return [...rows].sort((a, b) => b.total_tokens - a.total_tokens) +} function formatNumber(value: number): string { return new Intl.NumberFormat().format(value || 0) } -function toDateInput(d: Date): string { - return d.toISOString().slice(0, 10) +function formatAudio(seconds: number): string { + const total = Math.max(0, Math.floor(seconds || 0)) + if (total < 60) return `${total}s` + const mins = Math.floor(total / 60) + const secs = total % 60 + if (mins < 60) return secs ? `${mins}m ${secs}s` : `${mins}m` + const hours = Math.floor(mins / 60) + const remMins = mins % 60 + return remMins ? `${hours}h ${remMins}m` : `${hours}h` } -function FilterField({ - label, - children, -}: { - label: string - children: React.ReactNode -}) { +function rowHasUsageForKind(row: BreakdownRow, kind: Kind): boolean { + if (!kind) return true + if (kind === 'llm') { + return (row.total_tokens ?? 0) > 0 || (row.call_count ?? 0) > 0 + } + if (kind === 'stt') { + return (row.audio_seconds ?? 0) > 0 || (row.call_count ?? 0) > 0 + } + if (kind === 'tts') { + return (row.tts_characters ?? 0) > 0 || (row.call_count ?? 0) > 0 + } + return true +} + +function filterRowsForUsageKind(rows: BreakdownRow[], kind: Kind): BreakdownRow[] { + if (!kind) return rows + return rows.filter((row) => rowHasUsageForKind(row, kind)) +} + +function rowHasAnyUsage(row: BreakdownRow): boolean { return ( - + (row.total_tokens ?? 0) > 0 || + (row.call_count ?? 0) > 0 || + (row.audio_seconds ?? 0) > 0 || + (row.tts_characters ?? 0) > 0 ) } -const fieldClassName = - 'h-9 rounded-lg border border-gray-200 bg-white px-3 text-sm text-gray-900 outline-none focus:border-indigo-400 focus:ring-1 focus:ring-indigo-200' +function isUsageScopeActive( + workspaceId: string, + callImportId: string, + evaluationId: string, + dataset: string, + tagId: string, + usageKind: Kind, + model: string, + productSection: string, +): boolean { + return Boolean( + workspaceId || + callImportId || + evaluationId || + dataset || + tagId || + usageKind || + model || + productSection, + ) +} + +/** Drop zero rows when any filter is active; kind filter uses kind-specific metrics. */ +function filterTableRows( + rows: BreakdownRow[], + usageKind: Kind, + scopeActive: boolean, +): BreakdownRow[] { + if (usageKind) return filterRowsForUsageKind(rows, usageKind) + if (scopeActive) return rows.filter(rowHasAnyUsage) + return rows +} + +function drillGroupBy( + workspaceId: string, + callImportId: string, + evaluationId: string, + model: string, + productSection: string, +): DrillGroupBy { + if (productSection && model) return 'usage_kind' + if (productSection) return 'model' + if (evaluationId && model) return 'usage_kind' + if (evaluationId) return 'model' + if (callImportId) return 'resource' + if (workspaceId) return 'call_import' + return 'workspace' +} + +function enrichCallImportRows( + rawRows: BreakdownRow[], + options?: FilterOptions, +): BreakdownRow[] { + const labelById = new Map( + (options?.call_imports ?? []).map((c) => [idKey(c.id), c.label]), + ) + return sortRows( + rawRows + .filter((r) => r.call_import_id) + .map((row) => { + const key = idKey(row.call_import_id) + return { + ...row, + call_import_label: + labelById.get(key) || row.call_import_label || 'Call import', + } + }), + ) +} + +function buildWorkspaceCompositeRows( + callImportRaw: BreakdownRow[], + resourceRaw: BreakdownRow[], + options?: FilterOptions, + padMissingRows = true, +): WorkspaceSourceRow[] { + const resourceLabelById = new Map( + (options?.resources ?? []).map((r) => [idKey(r.id), r.label]), + ) + const importRows = enrichCallImportRows(callImportRaw, options) + const rows: WorkspaceSourceRow[] = [] + const shownImportIds = new Set() + + for (const row of importRows) { + if (!row.call_import_id) continue + shownImportIds.add(idKey(row.call_import_id)) + rows.push({ + ...row, + rowKind: 'call_import', + hint: CALL_IMPORT_HINT, + }) + } + + if (padMissingRows) { + for (const ci of options?.call_imports ?? []) { + const key = idKey(ci.id) + if (shownImportIds.has(key)) continue + shownImportIds.add(key) + rows.push({ + call_import_id: ci.id, + call_import_label: ci.label, + ...EMPTY_METRICS, + rowKind: 'call_import', + hint: CALL_IMPORT_HINT, + }) + } + } + + for (const row of resourceRaw) { + const section = row.product_section + if (!section || CALL_IMPORT_PRODUCT_SECTIONS.has(section)) continue + if (NON_COMPOSITE_RESOURCE_TYPES.has(row.resource_type || '')) continue + if ( + row.total_tokens === 0 && + row.call_count === 0 && + !row.audio_seconds && + !row.tts_characters + ) { + continue + } + if (!rowHasAnyUsage(row)) { + continue + } + rows.push({ + ...row, + rowKind: 'workspace_resource', + product_section: section, + product_section_label: row.product_section_label || section, + resource_label: + resourceLabelById.get(idKey(row.resource_id)) || + usableResourceLabel(row.resource_label), + hint: PRODUCT_SECTION_HINTS[section] || 'Product usage', + }) + } + + return sortRows(rows) as WorkspaceSourceRow[] +} + +function drillColumnLabel(groupBy: DrillGroupBy, composite = false): string { + if (composite) return 'Source' + if (groupBy === 'workspace') return 'Workspace' + if (groupBy === 'call_import') return 'Call import' + if (groupBy === 'resource') return 'Evaluation run' + if (groupBy === 'product_section') return 'Product area' + if (groupBy === 'model') return 'Model' + return 'Kind' +} + +function rowLabel(groupBy: DrillGroupBy, row: BreakdownRow, options?: FilterOptions): string { + if (groupBy === 'workspace') return row.workspace_name || 'Unknown' + if (groupBy === 'call_import') return row.call_import_label || 'Call import' + if (groupBy === 'resource') { + const fromFilters = row.resource_id + ? options?.resources?.find((r) => idKey(r.id) === idKey(row.resource_id))?.label + : undefined + return ( + usableResourceLabel(row.resource_label) || fromFilters || 'Unscoped' + ) + } + if (groupBy === 'model') return row.model || '—' + if (groupBy === 'product_section') + return row.product_section_label || row.product_section || '—' + if (row.usage_kind === 'stt') return 'STT' + if (row.usage_kind === 'llm') return 'LLM' + if (row.usage_kind === 'tts') return 'TTS' + return row.usage_kind || '—' +} + +function tableRowLabel( + groupBy: DrillGroupBy, + row: TableRow, + options?: FilterOptions, +): string { + if ('rowKind' in row) { + if (row.rowKind === 'call_import') return row.call_import_label || 'Call import batch' + return compositeRowTitle(row, options) + } + return rowLabel(groupBy, row, options) +} + +function idKey(id: string | null | undefined): string { + return id ? String(id).toLowerCase() : '' +} + +function mergeDrillRows( + groupBy: DrillGroupBy, + rawRows: BreakdownRow[], + options?: FilterOptions, + padMissingRows = true, +): BreakdownRow[] { + if (groupBy === 'workspace' && options?.workspaces?.length && padMissingRows) { + const byId = new Map( + rawRows + .filter((r) => r.workspace_id) + .map((r) => [idKey(r.workspace_id), r]), + ) + const merged: BreakdownRow[] = options.workspaces.map((ws) => ({ + workspace_id: ws.id, + workspace_name: ws.name, + ...(byId.get(idKey(ws.id)) ?? EMPTY_METRICS), + })) + for (const row of rawRows) { + if (!row.workspace_id) { + merged.push({ + ...row, + workspace_name: row.workspace_name || 'No workspace', + }) + } + } + return sortRows(merged) + } + + if (groupBy === 'call_import') { + if (rawRows.length === 0) return [] + + const labelById = new Map( + (options?.call_imports ?? []).map((c) => [idKey(c.id), c.label]), + ) + const shown = new Set() + const merged: BreakdownRow[] = [] + + for (const row of rawRows) { + if (row.call_import_id) { + const key = idKey(row.call_import_id) + shown.add(key) + merged.push({ + ...row, + call_import_label: + labelById.get(key) || row.call_import_label || 'Call import', + }) + } + } + + if (padMissingRows) { + for (const ci of options?.call_imports ?? []) { + const key = idKey(ci.id) + if (!shown.has(key)) { + merged.push({ + call_import_id: ci.id, + call_import_label: ci.label, + ...EMPTY_METRICS, + }) + } + } + } + + return sortRows(merged) + } + + if (groupBy === 'resource') { + if (rawRows.length === 0) return [] + + const evalLabelById = new Map( + (options?.evaluations ?? []).map((e) => [idKey(e.id), e.label]), + ) + const resourceLabelById = new Map( + (options?.resources ?? []).map((r) => [idKey(r.id), r.label]), + ) + const shown = new Set() + const merged: BreakdownRow[] = [] + + for (const row of rawRows) { + if (row.resource_id) { + const key = idKey(row.resource_id) + shown.add(key) + merged.push({ + ...row, + resource_label: + resourceLabelById.get(key) || + evalLabelById.get(key) || + usableResourceLabel(row.resource_label) || + 'Evaluation', + }) + } + } + + if (padMissingRows) { + for (const ev of options?.evaluations ?? []) { + const key = idKey(ev.id) + if (!shown.has(key)) { + merged.push({ + resource_id: ev.id, + resource_label: ev.label, + ...EMPTY_METRICS, + }) + } + } + } + + return sortRows(merged) + } + + if (groupBy === 'model' && options?.models?.length && padMissingRows) { + if (rawRows.length === 0) return [] + const byName = new Map( + rawRows.filter((r) => r.model).map((r) => [r.model!, r]), + ) + const merged = options.models.map((name) => ({ + model: name, + ...(byName.get(name) ?? EMPTY_METRICS), + })) + return sortRows(merged) + } + + if (groupBy === 'usage_kind') { + return sortRows( + rawRows.filter((row) => rowHasUsageForKind(row, row.usage_kind as Kind)), + ) + } + + return sortRows(rawRows) +} export default function Usage() { const [searchParams, setSearchParams] = useSearchParams() - const today = useMemo(() => new Date(), []) - const defaultStart = useMemo(() => { - const d = new Date() - d.setDate(d.getDate() - 29) - return toDateInput(d) - }, []) - - const start = searchParams.get('start') || defaultStart - const end = searchParams.get('end') || toDateInput(today) - const groupBy = (searchParams.get('group_by') as GroupBy) || 'workspace' + const defaultRange = useMemo(() => defaultUsageDateRange(), []) + const usageTimezone = useMemo(() => getUsageTimezone(), []) + + const start = searchParams.get('start') || defaultRange.start + const end = searchParams.get('end') || defaultRange.end const workspaceId = searchParams.get('workspace_id') || '' - const productSection = searchParams.get('product_section') || '' + const callImportId = searchParams.get('call_import_id') || '' + const dataset = searchParams.get('dataset') || '' + const tagId = searchParams.get('tag_id') || '' + const evaluationId = searchParams.get('resource_id') || '' const model = searchParams.get('model') || '' - const resourceId = searchParams.get('resource_id') || '' + const usageKind = (searchParams.get('usage_kind') as Kind) || '' + const productSection = searchParams.get('product_section') || '' + + const showWorkspaceComposite = + Boolean(workspaceId) && + !callImportId && + !evaluationId && + !productSection + + const groupBy = drillGroupBy( + workspaceId, + callImportId, + evaluationId, + model, + productSection, + ) - const setParam = (key: string, value: string) => { + const setParams = (updates: Record) => { const next = new URLSearchParams(searchParams) - if (!value) next.delete(key) - else next.set(key, value) + for (const [key, value] of Object.entries(updates)) { + if (!value) next.delete(key) + else next.set(key, value) + } setSearchParams(next) } - const filterParams = { + const scopeParams = { start, end, + tz: usageTimezone, workspace_id: workspaceId || undefined, + call_import_id: callImportId || undefined, + dataset: dataset || undefined, + tag_id: tagId || undefined, product_section: productSection || undefined, + usage_kind: usageKind || undefined, model: model || undefined, - resource_id: resourceId || undefined, + resource_id: evaluationId || undefined, + evaluation_id: evaluationId || undefined, } - const { data: summary, isLoading: summaryLoading } = useQuery({ - queryKey: ['org-usage', 'summary', filterParams], - queryFn: () => apiClient.getOrgUsageSummary(filterParams), + const dataParams = { + ...scopeParams, + } + + const usageQueryDefaults = { + staleTime: 60 * 1000, + } + + const { data: summary, isLoading: summaryLoading, isFetching: summaryFetching } = useQuery({ + queryKey: ['org-usage', 'summary', dataParams], + queryFn: () => apiClient.getOrgUsageSummary(dataParams), + ...usageQueryDefaults, + placeholderData: keepPreviousData, }) - const { data: breakdown, isLoading: breakdownLoading } = useQuery({ - queryKey: ['org-usage', 'breakdown', groupBy, filterParams], + const { + data: breakdown, + isLoading: breakdownLoading, + isFetching: breakdownFetching, + } = useQuery({ + queryKey: ['org-usage', 'breakdown', groupBy, dataParams], queryFn: () => apiClient.getOrgUsageBreakdown({ - ...filterParams, + ...dataParams, group_by: groupBy, limit: 100, }), + enabled: !showWorkspaceComposite, + ...usageQueryDefaults, + placeholderData: (previousData, previousQuery) => { + if (!previousQuery || previousQuery.queryKey[2] !== groupBy) return undefined + return previousData + }, }) - const { data: filters } = useQuery({ - queryKey: ['org-usage', 'filters', start, end], - queryFn: () => apiClient.getOrgUsageFilters({ start, end }), + const { + data: importBreakdown, + isLoading: importBreakdownLoading, + isFetching: importBreakdownFetching, + } = useQuery({ + queryKey: ['org-usage', 'breakdown', 'call_import', dataParams], + queryFn: () => + apiClient.getOrgUsageBreakdown({ + ...dataParams, + group_by: 'call_import', + limit: 100, + }), + enabled: showWorkspaceComposite, + ...usageQueryDefaults, }) + const { + data: resourceBreakdown, + isLoading: resourceBreakdownLoading, + isFetching: resourceBreakdownFetching, + } = useQuery({ + queryKey: ['org-usage', 'breakdown', 'resource', dataParams], + queryFn: () => + apiClient.getOrgUsageBreakdown({ + ...dataParams, + group_by: 'resource', + limit: 100, + }), + enabled: showWorkspaceComposite, + ...usageQueryDefaults, + }) + + const { data: filterOptions, isFetching: filtersLoading } = useQuery({ + queryKey: ['org-usage', 'filters', scopeParams], + queryFn: () => apiClient.getOrgUsageFilters(scopeParams), + staleTime: 60 * 1000, + placeholderData: (previousData, previousQuery) => { + if (!previousQuery) return undefined + const prevScope = previousQuery.queryKey[2] as typeof scopeParams + if (JSON.stringify(prevScope) !== JSON.stringify(scopeParams)) return undefined + return previousData + }, + }) + + useEffect(() => { + if (!filterOptions) return + const updates: Record = {} + if ( + workspaceId && + !filterOptions.workspaces?.some((w) => w.id === workspaceId) + ) { + updates.workspace_id = null + updates.call_import_id = null + updates.resource_id = null + } + if ( + callImportId && + !filterOptions.call_imports?.some((c) => c.id === callImportId) + ) { + updates.call_import_id = null + updates.resource_id = null + } + if ( + evaluationId && + !filterOptions.evaluations?.some((e) => e.id === evaluationId) + ) { + updates.resource_id = null + } + if (model && !filterOptions.models?.includes(model)) { + updates.model = null + } + if ( + usageKind && + !filterOptions.usage_kinds?.some((k) => k.id === usageKind) + ) { + updates.usage_kind = null + } + if ( + productSection && + !filterOptions.product_sections?.some((s) => s.id === productSection) + ) { + updates.product_section = null + } + if (dataset && !filterOptions.datasets?.includes(dataset)) { + updates.dataset = null + } + if (tagId && !filterOptions.tags?.some((t) => t.id === tagId)) { + updates.tag_id = null + } + if (Object.keys(updates).length > 0) setParams(updates) + }, [ + filterOptions, + workspaceId, + callImportId, + dataset, + tagId, + evaluationId, + model, + usageKind, + productSection, + ]) + + const breakdownMatchesLevel = breakdown?.group_by === groupBy + const rawRows = useMemo((): BreakdownRow[] => { + if (!breakdown || !breakdownMatchesLevel) return [] + return breakdown.rows as BreakdownRow[] + }, [breakdown, breakdownMatchesLevel]) + + const mergeOptions = filterOptions + const scopeActive = isUsageScopeActive( + workspaceId, + callImportId, + evaluationId, + dataset, + tagId, + usageKind, + model, + productSection, + ) + const padMissingRows = !scopeActive + const filteredRawRows = useMemo( + () => filterTableRows(rawRows, usageKind, scopeActive), + [rawRows, usageKind, scopeActive], + ) + const rows: TableRow[] = useMemo(() => { + if (showWorkspaceComposite) { + const importReady = importBreakdown?.group_by === 'call_import' + const resourceReady = resourceBreakdown?.group_by === 'resource' + if (!importReady && !resourceReady) return [] + return buildWorkspaceCompositeRows( + importReady + ? filterTableRows(importBreakdown.rows as BreakdownRow[], usageKind, scopeActive) + : [], + resourceReady + ? filterTableRows(resourceBreakdown.rows as BreakdownRow[], usageKind, scopeActive) + : [], + mergeOptions, + padMissingRows, + ) + } + return mergeDrillRows(groupBy, filteredRawRows, mergeOptions, padMissingRows) + }, [ + showWorkspaceComposite, + importBreakdown, + resourceBreakdown, + groupBy, + filteredRawRows, + mergeOptions, + padMissingRows, + usageKind, + scopeActive, + ]) + + const breakdownStale = + !showWorkspaceComposite && breakdownFetching && !breakdownMatchesLevel + const tableLoading = showWorkspaceComposite + ? (importBreakdownLoading || resourceBreakdownLoading) && + !importBreakdown && + !resourceBreakdown + : breakdownLoading && !breakdown + const tableFetching = showWorkspaceComposite + ? importBreakdownFetching || resourceBreakdownFetching + : breakdownFetching const totals = summary?.totals - const rows = breakdown?.rows || [] - - const dimensionLabel = (row: (typeof rows)[number]): string => { - if (groupBy === 'workspace') return row.workspace_name || 'Unknown' - if (groupBy === 'product_section') - return row.product_section_label || row.product_section || '—' - if (groupBy === 'model') return row.model || '—' - return row.resource_label || row.resource_id || 'Unscoped' + const showAudio = Boolean(totals?.audio_seconds) + const showTts = + Boolean(totals?.tts_characters) || + rows.some((r) => Boolean(r.tts_characters)) + + const showTruncation = + (!showWorkspaceComposite && + breakdownMatchesLevel && + Boolean(breakdown?.truncated_at_limit)) || + (showWorkspaceComposite && + Boolean( + importBreakdown?.truncated_at_limit || resourceBreakdown?.truncated_at_limit, + )) + + const workspaceLabel = + filterOptions?.workspaces?.find((w) => w.id === workspaceId)?.name + const callImportLabel = + filterOptions?.call_imports?.find((c) => c.id === callImportId)?.label + const evaluationLabel = + filterOptions?.resources?.find((r) => r.id === evaluationId)?.label || + filterOptions?.evaluations?.find((e) => e.id === evaluationId)?.label + + const productSectionLabel = + filterOptions?.product_sections?.find((s) => s.id === productSection)?.label + + const scopeSubtitle = model + ? model + : evaluationId + ? evaluationLabel || 'Evaluation' + : callImportId + ? callImportLabel || 'Call import' + : productSection + ? productSectionLabel || 'Product area' + : workspaceId + ? workspaceLabel || 'Workspace' + : 'Organization' + + const levelHint = (() => { + if (showWorkspaceComposite) { + return 'Call import batches and other product usage — click a row to drill down' + } + if (groupBy === 'workspace') return 'Click a workspace to drill down' + if (groupBy === 'call_import') return 'Click a call import to see evaluation runs' + if (groupBy === 'product_section') return 'Click a product area to see models used' + if (groupBy === 'resource') return 'Click an evaluation to see models used' + if (groupBy === 'model') return 'Click a model to see usage by kind' + return 'Token totals by LLM / STT / TTS' + })() + + const drillCrumbs = [ + { + label: 'Organization', + onClick: + workspaceId || + callImportId || + evaluationId || + model || + productSection + ? () => + setParams({ + workspace_id: null, + call_import_id: null, + resource_id: null, + model: null, + usage_kind: null, + product_section: null, + }) + : undefined, + }, + ...(workspaceId + ? [ + { + label: workspaceLabel || 'Workspace', + onClick: + callImportId || evaluationId || model || productSection + ? () => + setParams({ + call_import_id: null, + resource_id: null, + model: null, + usage_kind: null, + product_section: null, + }) + : undefined, + }, + ] + : []), + ...(productSection + ? [ + { + label: productSectionLabel || 'Product area', + onClick: + model + ? () => setParams({ model: null, usage_kind: null }) + : undefined, + }, + ] + : []), + ...(callImportId + ? [ + { + label: callImportLabel || 'Call import', + onClick: + evaluationId || model + ? () => + setParams({ + resource_id: null, + model: null, + usage_kind: null, + product_section: null, + }) + : undefined, + }, + ] + : []), + ...(evaluationId + ? [ + { + label: evaluationLabel || 'Evaluation', + onClick: model + ? () => setParams({ model: null, usage_kind: null }) + : undefined, + }, + ] + : []), + ...(model ? [{ label: model }] : []), + ] + + const handleWorkspaceChange = (id: string) => { + setParams({ + workspace_id: id || null, + call_import_id: null, + resource_id: null, + model: null, + usage_kind: null, + product_section: null, + }) + } + + const handleCallImportChange = (id: string) => { + setParams({ + call_import_id: id || null, + resource_id: null, + model: null, + usage_kind: null, + product_section: null, + }) + } + + const handleClearAll = () => { + setParams({ + workspace_id: null, + call_import_id: null, + dataset: null, + tag_id: null, + resource_id: null, + usage_kind: null, + model: null, + product_section: null, + }) + } + + const handleRowDrill = (row: TableRow) => { + if ('rowKind' in row) { + if (row.rowKind === 'call_import' && row.call_import_id) { + setParams({ + call_import_id: row.call_import_id, + resource_id: null, + model: null, + usage_kind: null, + product_section: null, + }) + return + } + if (row.rowKind === 'workspace_resource' && row.product_section) { + setParams({ + product_section: row.product_section, + call_import_id: null, + resource_id: row.resource_id || null, + model: null, + usage_kind: null, + }) + return + } + } + if (groupBy === 'workspace' && row.workspace_id) { + setParams({ + workspace_id: row.workspace_id, + call_import_id: null, + resource_id: null, + model: null, + usage_kind: null, + product_section: null, + }) + return + } + if (groupBy === 'call_import' && row.call_import_id) { + setParams({ + call_import_id: row.call_import_id, + resource_id: null, + model: null, + usage_kind: null, + product_section: null, + }) + return + } + if (groupBy === 'product_section' && row.product_section) { + setParams({ + product_section: row.product_section, + call_import_id: null, + resource_id: null, + model: null, + usage_kind: null, + }) + return + } + if (groupBy === 'resource' && row.resource_id) { + setParams({ + resource_id: row.resource_id, + model: null, + usage_kind: null, + product_section: null, + }) + return + } + if (groupBy === 'model' && row.model) { + setParams({ model: row.model, usage_kind: null }) + } + } + + const isRowDrillable = (row: TableRow): boolean => { + if ('rowKind' in row) { + if (row.rowKind === 'call_import') return Boolean(row.call_import_id) + return Boolean(row.product_section) + } + if (groupBy === 'workspace') return Boolean(row.workspace_id) + if (groupBy === 'call_import') return Boolean(row.call_import_id) + if (groupBy === 'product_section') return Boolean(row.product_section) + if (groupBy === 'resource') return Boolean(row.resource_id) + if (groupBy === 'model') return Boolean(row.model) + return false } return ( @@ -102,167 +976,234 @@ export default function Usage() {

- + Usage

- Organization-wide LLM tokens and calls. Filter by workspace, product - section, model, or evaluation. + Cards show usage for {scopeSubtitle}. + Drill down: workspaces → call imports or product areas → evaluations / models.

- {summary?.last_updated_at && ( + {summary?.last_updated_at ? (

Updated {new Date(summary.last_updated_at).toLocaleString()}

- )} + ) : null}
-
+
- {(totals?.cache_read_tokens || totals?.cache_creation_tokens || totals?.reasoning_tokens) ? ( -
- - - + {(showAudio || + showTts || + totals?.cache_read_tokens || + totals?.cache_creation_tokens || + totals?.reasoning_tokens) ? ( +
+ {showAudio ? ( + + ) : null} + {showTts ? ( + + ) : null} + {(totals?.cache_read_tokens || 0) > 0 ? ( + + ) : null} + {(totals?.cache_creation_tokens || 0) > 0 ? ( + + ) : null} + {(totals?.reasoning_tokens || 0) > 0 ? ( + + ) : null}
) : null} - - -
- - setParam('start', e.target.value)} - /> - - - setParam('end', e.target.value)} - /> - - - - - - - - - - - - - -
- - + setParams({ start: s, end: e })} + onWorkspaceChange={handleWorkspaceChange} + onCallImportChange={handleCallImportChange} + onDatasetChange={(v) => + setParams({ + dataset: v || null, + call_import_id: null, + resource_id: null, + }) + } + onTagChange={(v) => + setParams({ + tag_id: v || null, + call_import_id: null, + resource_id: null, + }) + } + onEvaluationChange={(id) => { + const resource = filterOptions?.resources?.find((r) => r.id === id) + setParams({ + resource_id: id || null, + product_section: id ? resource?.product_section || null : null, + model: null, + usage_kind: null, + }) + }} + onUsageKindChange={(k) => setParams({ usage_kind: k || null })} + onModelChange={(v) => setParams({ model: v || null })} + onClearAll={handleClearAll} + /> - - - {breakdownLoading ? ( + +
+ + {showTruncation ? ( +

+ Showing the first 100 rows for this level. Narrow the date range or drill + further for complete detail. +

+ ) : null} +
+ + {tableLoading || breakdownStale ? (
) : rows.length === 0 ? (
- No usage in this period. Run evaluations or playground calls, then - refresh in a minute. + No usage in this period for the current scope. Try 7d or 30d, or go back up a + level.
) : ( +
+ {tableFetching ? ( +
+ +
+ ) : null} - + - - - - + + + + + + - {rows.map((row, idx) => ( - - - - - - - - - ))} + {rows.map((row, idx) => { + const drillable = isRowDrillable(row) + const compositeHeadline = + 'rowKind' in row ? compositeRowHeadline(row) : null + const rowTitle = tableRowLabel(groupBy, row, filterOptions) + const showRowTitle = + 'rowKind' in row + ? row.rowKind === 'workspace_resource' || + (compositeHeadline && + rowTitle.toLowerCase() !== compositeHeadline.toLowerCase()) + : true + return ( + drillable && handleRowDrill(row)} + > + + + + + + + + + + ) + })}
- {groupBy === 'workspace' - ? 'Workspace' - : groupBy === 'product_section' - ? 'Section' - : groupBy === 'model' - ? 'Model' - : 'Resource'} + {drillColumnLabel(groupBy, showWorkspaceComposite)} CallsInputOutputTotalLLM callsInput tokensOutput tokensTotal tokensSTT audioTTS chars Cache read
{dimensionLabel(row)} - {formatNumber(row.call_count)} - - {formatNumber(row.prompt_tokens)} - - {formatNumber(row.completion_tokens)} - - {formatNumber(row.total_tokens)} - - {formatNumber(row.cache_read_tokens)} -
+ + + {'rowKind' in row ? ( + + {compositeHeadline} + + ) : null} + {showRowTitle ? ( + + {rowTitle} + + ) : !('rowKind' in row) ? ( + {rowTitle} + ) : null} + {'hint' in row && row.hint ? ( + + {row.hint} + + ) : null} + + {drillable ? ( + + ) : null} + + + {formatNumber(row.call_count)} + + {formatNumber(row.prompt_tokens)} + + {formatNumber(row.completion_tokens)} + + {formatNumber(row.total_tokens)} + + {row.audio_seconds ? formatAudio(row.audio_seconds) : '—'} + + {row.tts_characters ? formatNumber(row.tts_characters) : '—'} + + {formatNumber(row.cache_read_tokens)} +
+
)} @@ -273,20 +1214,22 @@ export default function Usage() { function StatCard({ label, value, + valueLabel, loading, }: { label: string value?: number + valueLabel?: string loading?: boolean }) { return ( - - + +

{label}

-

- {loading ? '—' : formatNumber(value || 0)} +

+ {loading ? '—' : valueLabel ?? formatNumber(value || 0)}

diff --git a/frontend/src/pages/usage/UsageDateRangePicker.tsx b/frontend/src/pages/usage/UsageDateRangePicker.tsx new file mode 100644 index 00000000..3ffeaeec --- /dev/null +++ b/frontend/src/pages/usage/UsageDateRangePicker.tsx @@ -0,0 +1,252 @@ +import { useEffect, useMemo, useRef, useState } from 'react' +import { Calendar } from 'lucide-react' +import { usageTheme } from './usageTheme' +import { getUsageTimezone, formatUsageTimezoneLabel } from './usageTimezone' + +type Mode = 'relative' | 'absolute' + +type UsageDateRangePickerProps = { + start: string + end: string + onApply: (start: string, end: string) => void +} + +function toDateInput(d: Date): string { + const y = d.getFullYear() + const m = String(d.getMonth() + 1).padStart(2, '0') + const day = String(d.getDate()).padStart(2, '0') + return `${y}-${m}-${day}` +} + +function formatDisplay(start: string, end: string): string { + if (start === end) return start + return `${start} → ${end}` +} + +function rangeForDays(days: number): { start: string; end: string } { + const end = new Date() + const start = new Date() + start.setDate(start.getDate() - (days - 1)) + return { start: toDateInput(start), end: toDateInput(end) } +} + +const QUICK_RANGES = [ + { label: '1d', days: 1 }, + { label: '7d', days: 7 }, + { label: '30d', days: 30 }, + { label: '90d', days: 90 }, +] as const + +const RELATIVE_DAYS = [1, 2, 3, 4, 5, 6] +const RELATIVE_WEEKS = [1, 2, 3, 4] + +export default function UsageDateRangePicker({ + start, + end, + onApply, +}: UsageDateRangePickerProps) { + const [open, setOpen] = useState(false) + const [mode, setMode] = useState('relative') + const [draftStart, setDraftStart] = useState(start) + const [draftEnd, setDraftEnd] = useState(end) + const [relDays, setRelDays] = useState(1) + const rootRef = useRef(null) + + const activeQuick = useMemo(() => { + for (const q of QUICK_RANGES) { + const r = rangeForDays(q.days) + if (r.start === start && r.end === end) return q.label + } + return null + }, [start, end]) + + useEffect(() => { + if (open) { + setDraftStart(start) + setDraftEnd(end) + } + }, [open, start, end]) + + useEffect(() => { + const onDoc = (e: MouseEvent) => { + if (!rootRef.current?.contains(e.target as Node)) setOpen(false) + } + document.addEventListener('mousedown', onDoc) + return () => document.removeEventListener('mousedown', onDoc) + }, []) + + const applyRelativeDays = (days: number) => { + const r = rangeForDays(days) + onApply(r.start, r.end) + setOpen(false) + } + + const applyAbsolute = () => { + if (!draftStart || !draftEnd) return + if (draftEnd < draftStart) return + onApply(draftStart, draftEnd) + setOpen(false) + } + + const pillClass = (active: boolean) => + `rounded-md px-2.5 py-1 text-xs font-medium transition-colors border ${ + active ? usageTheme.pillActive : usageTheme.pillInactive + }` + + return ( +
+
+ {QUICK_RANGES.map((q) => ( + + ))} + +
+ + + {formatDisplay(start, end)} + + {' '} + · {formatUsageTimezoneLabel(getUsageTimezone())} + + + + {open ? ( +
+
+ + +
+ + {mode === 'relative' ? ( +
+

+ Dates use your local timezone ({formatUsageTimezoneLabel(getUsageTimezone())}). + Usage is stored by UTC day; filters include activity that happened on each + selected local day. +

+
+

Days

+
+ {RELATIVE_DAYS.map((d) => ( + + ))} +
+
+
+

Weeks

+
+ {RELATIVE_WEEKS.map((w) => ( + + ))} +
+
+
+ + +
+
+ ) : ( +
+
+ + +
+
+ + +
+
+ )} +
+ ) : null} +
+ ) +} + +export function defaultUsageDateRange(): { start: string; end: string } { + const today = toDateInput(new Date()) + return { start: today, end: today } +} diff --git a/frontend/src/pages/usage/UsageDrillPath.tsx b/frontend/src/pages/usage/UsageDrillPath.tsx new file mode 100644 index 00000000..0e25d673 --- /dev/null +++ b/frontend/src/pages/usage/UsageDrillPath.tsx @@ -0,0 +1,40 @@ +import { ChevronRight } from 'lucide-react' +import { usageTheme } from './usageTheme' + +type DrillCrumb = { + label: string + onClick?: () => void +} + +type UsageDrillPathProps = { + crumbs: DrillCrumb[] + levelLabel: string +} + +export default function UsageDrillPath({ crumbs, levelLabel }: UsageDrillPathProps) { + return ( +
+ +

{levelLabel}

+
+ ) +} diff --git a/frontend/src/pages/usage/UsageFiltersBar.tsx b/frontend/src/pages/usage/UsageFiltersBar.tsx new file mode 100644 index 00000000..747fc2ed --- /dev/null +++ b/frontend/src/pages/usage/UsageFiltersBar.tsx @@ -0,0 +1,353 @@ +import { useMemo, useState } from 'react' +import { Chip } from '@heroui/react' +import { ChevronDown, Filter, SlidersHorizontal } from 'lucide-react' +import SearchableSelect from './SearchableSelect' +import UsageDateRangePicker from './UsageDateRangePicker' +import { usageTheme } from './usageTheme' + +type Kind = '' | 'llm' | 'stt' | 'tts' + +type FilterOptions = { + workspaces: Array<{ id: string; name: string }> + call_imports: Array<{ id: string; label: string }> + evaluations: Array<{ id: string; label: string }> + resources?: Array<{ id: string; label: string; type?: string; product_section?: string }> + models: string[] + usage_kinds: Array<{ id: string; label: string }> + datasets?: string[] + tags?: Array<{ id: string; label: string }> +} + +type ActiveChip = { key: string; label: string; onClear: () => void } + +type UsageFiltersBarProps = { + start: string + end: string + workspaceId: string + callImportId: string + evaluationId: string + dataset: string + tagId: string + usageKind: Kind + model: string + options?: FilterOptions + filtersLoading?: boolean + onDateApply: (start: string, end: string) => void + onWorkspaceChange: (id: string) => void + onCallImportChange: (id: string) => void + onDatasetChange: (value: string) => void + onTagChange: (value: string) => void + onEvaluationChange: (id: string) => void + onUsageKindChange: (v: Kind) => void + onModelChange: (v: string) => void + onClearAll: () => void +} + +const KIND_OPTIONS: Array<{ id: Kind; label: string }> = [ + { id: '', label: 'All' }, + { id: 'llm', label: 'LLM' }, + { id: 'stt', label: 'STT' }, + { id: 'tts', label: 'TTS' }, +] + +export default function UsageFiltersBar({ + start, + end, + workspaceId, + callImportId, + evaluationId, + dataset, + tagId, + usageKind, + model, + options, + filtersLoading, + onDateApply, + onWorkspaceChange, + onCallImportChange, + onDatasetChange, + onTagChange, + onEvaluationChange, + onUsageKindChange, + onModelChange, + onClearAll, +}: UsageFiltersBarProps) { + const [expanded, setExpanded] = useState(false) + + const workspaces = options?.workspaces ?? [] + const callImports = options?.call_imports ?? [] + const evaluations = options?.evaluations ?? [] + const resources = options?.resources ?? [] + const datasets = options?.datasets ?? [] + const tags = options?.tags ?? [] + const models = options?.models ?? [] + const availableKinds = useMemo( + () => new Set(options?.usage_kinds?.map((k) => k.id) ?? []), + [options?.usage_kinds], + ) + + const sourceOptions = callImportId ? evaluations : resources + + const activeChips = useMemo((): ActiveChip[] => { + const chips: ActiveChip[] = [] + const wsLabel = workspaces.find((w) => w.id === workspaceId)?.name + if (workspaceId && wsLabel) { + chips.push({ + key: 'workspace', + label: wsLabel, + onClear: () => onWorkspaceChange(''), + }) + } + const importLabel = callImports.find((c) => c.id === callImportId)?.label + if (callImportId && importLabel) { + chips.push({ + key: 'call_import', + label: importLabel, + onClear: () => onCallImportChange(''), + }) + } + if (dataset) { + chips.push({ + key: 'dataset', + label: dataset, + onClear: () => onDatasetChange(''), + }) + } + if (tagId) { + const tagLabel = tags.find((t) => t.id === tagId)?.label + if (tagLabel) { + chips.push({ + key: 'tag', + label: tagLabel, + onClear: () => onTagChange(''), + }) + } + } + const sourceLabel = + sourceOptions.find((e) => e.id === evaluationId)?.label || + evaluations.find((e) => e.id === evaluationId)?.label || + resources.find((r) => r.id === evaluationId)?.label + if (evaluationId && sourceLabel) { + chips.push({ + key: 'evaluation', + label: sourceLabel, + onClear: () => onEvaluationChange(''), + }) + } + if (usageKind) { + chips.push({ + key: 'kind', + label: usageKind.toUpperCase(), + onClear: () => onUsageKindChange(''), + }) + } + if (model) { + chips.push({ + key: 'model', + label: model, + onClear: () => onModelChange(''), + }) + } + return chips + }, [ + workspaceId, + workspaces, + callImportId, + callImports, + dataset, + tagId, + tags, + evaluationId, + sourceOptions, + evaluations, + resources, + usageKind, + model, + onWorkspaceChange, + onCallImportChange, + onDatasetChange, + onTagChange, + onEvaluationChange, + onUsageKindChange, + onModelChange, + ]) + + const hasScopeFilters = activeChips.length > 0 + + const kindHint = (kind: Kind): string | undefined => { + if (!kind) return undefined + if (availableKinds.has(kind)) return undefined + return `No ${kind.toUpperCase()} usage in this date range` + } + + return ( +
+
+ + +
+ + + + {filtersLoading ? ( + Updating options… + ) : null} + + {hasScopeFilters ? ( + + ) : null} +
+ + {!expanded && hasScopeFilters ? ( +
+ {activeChips.map((chip) => ( + + {chip.label} + + ))} +
+ ) : null} + + {expanded ? ( +
+

+ + Jump to a level or click rows in the table to drill down. +

+ +
+ ({ id: w.id, label: w.name }))} + onChange={onWorkspaceChange} + emptyMessage="No workspace usage in this date range" + /> + ({ id: d, label: d }))} + onChange={onDatasetChange} + emptyMessage="No datasets in this workspace" + disabled={datasets.length === 0} + /> + +
+ +
+ + +
+ +
+
+ Kind + {KIND_OPTIONS.map((k) => { + const disabled = k.id !== '' && !availableKinds.has(k.id) + const hint = kindHint(k.id) + return ( + + ) + })} + {!availableKinds.has('tts') ? ( + No TTS + ) : null} + {!availableKinds.has('stt') ? ( + No STT + ) : null} +
+ +
+ + {models.length === 0 ? ( + No models in this range + ) : null} +
+
+
+ ) : null} +
+ ) +} diff --git a/frontend/src/pages/usage/usageProductHints.ts b/frontend/src/pages/usage/usageProductHints.ts new file mode 100644 index 00000000..4a057b1c --- /dev/null +++ b/frontend/src/pages/usage/usageProductHints.ts @@ -0,0 +1,47 @@ +/** Product areas shown separately from call-import drill-down. */ +export const CALL_IMPORT_PRODUCT_SECTIONS = new Set([ + 'call_imports', + 'call_import_evaluations', +]) + +/** Short table headline per product section (workspace composite rows). */ +export const PRODUCT_SECTION_HEADLINES: Record = { + voice_playground: 'Voice playground', + playground: 'Playground', + chat: 'Chat', + telephony: 'Telephony', + evaluators: 'Evaluators', + metrics: 'Metrics', + judge_alignment: 'Judge alignment', + prompt_optimization: 'Prompt optimization', + personas: 'Personas', + agents: 'Agents', + prompt_partials: 'Prompt partials', + conversation_evaluations: 'Conversation evaluations', + test_agent: 'Test agent', + call_import_evaluations: 'Call import evaluations', + call_imports: 'Call imports', + other: 'Other', +} + +export const CALL_IMPORT_BATCH_HEADLINE = 'Call import batch' + +export const PRODUCT_SECTION_HINTS: Record = { + voice_playground: 'Voice agent playground — LLM, STT, and TTS', + playground: 'Text playground and experiments', + chat: 'Chat conversations', + telephony: 'Telephony and live calls', + evaluators: 'Evaluator definitions and runs', + metrics: 'Metrics and scoring', + judge_alignment: 'Judge alignment workflows', + prompt_optimization: 'Prompt optimization jobs', + personas: 'Persona generation', + agents: 'Agent configuration', + prompt_partials: 'Prompt partials', + conversation_evaluations: 'Conversation evaluations', + test_agent: 'Test agent sessions', + other: 'Other product usage', +} + +export const CALL_IMPORT_HINT = + 'Call import batch — CSV upload or manual audio recordings' diff --git a/frontend/src/pages/usage/usageTheme.ts b/frontend/src/pages/usage/usageTheme.ts new file mode 100644 index 00000000..fc4b06be --- /dev/null +++ b/frontend/src/pages/usage/usageTheme.ts @@ -0,0 +1,18 @@ +/** Matches platform primary buttons (soft gold, not solid yellow). */ +export const usageTheme = { + pillActive: + 'bg-[#fef9c3] border border-[#facc15] text-[#a16207] font-semibold shadow-sm', + pillInactive: + 'text-gray-600 hover:bg-[#fefce8] hover:text-[#854d0e] border border-transparent', + pillMuted: 'bg-gray-50 border border-gray-200 text-gray-600', + link: 'text-primary-600 hover:text-primary-700', + linkStrong: 'text-[#a16207] hover:text-[#854d0e] font-medium', + chipBase: 'bg-[#fefce8] border border-[#fde047]', + chipContent: 'text-[#854d0e]', + panel: 'rounded-xl border border-gray-200 bg-white shadow-sm ring-1 ring-[#fde047]/30', + panelHeader: 'bg-[#fefce8]/50 border-b border-[#fde047]/40', + focusRing: 'focus:border-[#facc15] focus:ring-2 focus:ring-[#fef9c3]', + selectHighlight: 'bg-[#fefce8] text-[#854d0e] font-medium', + applyBtn: + 'rounded-lg bg-[#fef9c3] border border-[#facc15] px-4 py-1.5 text-sm font-semibold text-[#a16207] hover:bg-[#fef08a]', +} diff --git a/frontend/src/pages/usage/usageTimezone.ts b/frontend/src/pages/usage/usageTimezone.ts new file mode 100644 index 00000000..f222a038 --- /dev/null +++ b/frontend/src/pages/usage/usageTimezone.ts @@ -0,0 +1,11 @@ +export function getUsageTimezone(): string { + try { + return Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC' + } catch { + return 'UTC' + } +} + +export function formatUsageTimezoneLabel(tz: string): string { + return tz.replace(/_/g, ' ') +} diff --git a/tests/test_services/test_evaluators/test_call_data_transcript.py b/tests/test_services/test_evaluators/test_call_data_transcript.py new file mode 100644 index 00000000..ef2e2707 --- /dev/null +++ b/tests/test_services/test_evaluators/test_call_data_transcript.py @@ -0,0 +1,34 @@ +"""Tests for call_data transcript extraction.""" + +from app.services.evaluators.call_data_transcript import extract_transcript_from_call_data + + +def test_vapi_extracts_transcript_from_messages_when_transcript_field_empty(): + call_data = { + "transcript": "", + "endedReason": "call.in-progress.error-assistant-did-not-receive-customer-audio", + "messages": [ + {"role": "assistant", "message": "Hello, how can I help?"}, + {"role": "user", "message": "Hi there"}, + ], + } + text, segments = extract_transcript_from_call_data(call_data, "vapi") + assert "Hello, how can I help?" in text + assert "Hi there" in text + assert len(segments) == 2 + + +def test_hydrate_helper_permanent_vapi_reason(): + from app.workers.tasks import process_evaluator_result as task_module + + class _Result: + audio_s3_key = None + transcription = None + provider_platform = "vapi" + call_data = { + "endedReason": "call.in-progress.error-assistant-did-not-receive-customer-audio", + } + + msg = task_module._permanent_input_failure_message(_Result(), db=None) + assert msg is not None + assert "microphone" in msg.lower() diff --git a/tests/test_services/test_usage/test_bucket_context.py b/tests/test_services/test_usage/test_bucket_context.py new file mode 100644 index 00000000..5cedb81c --- /dev/null +++ b/tests/test_services/test_usage/test_bucket_context.py @@ -0,0 +1,40 @@ +"""Tests for usage bucket context helpers.""" + +from uuid import uuid4 + +from app.services.usage.bucket_context import ( + build_bucket_context, + context_bucket_token, + legacy_resource_context, + parse_context_bucket_token, + resource_id_from_context, +) + + +def test_build_and_parse_context_roundtrip(): + rid = uuid4() + ctx = build_bucket_context( + resource_id=rid, + resource_type="call_import_evaluation", + extra={"agent_id": str(uuid4())}, + ) + token = context_bucket_token(ctx) + parsed = parse_context_bucket_token(token) + assert parsed["resource_id"] == str(rid) + assert parsed["resource_type"] == "call_import_evaluation" + assert "agent_id" in parsed + + +def test_extra_rejects_metric_keys(): + ctx = build_bucket_context( + resource_id=uuid4(), + extra={"prompt_tokens": "999", "agent_id": str(uuid4())}, + ) + assert "prompt_tokens" not in ctx + assert "agent_id" in ctx + + +def test_legacy_resource_context(): + rid = uuid4() + ctx = legacy_resource_context(rid, "call_import") + assert resource_id_from_context(ctx) == rid diff --git a/tests/test_services/test_usage/test_call_import_context.py b/tests/test_services/test_usage/test_call_import_context.py new file mode 100644 index 00000000..6d798f65 --- /dev/null +++ b/tests/test_services/test_usage/test_call_import_context.py @@ -0,0 +1,120 @@ +"""Tests for call-import usage attribution context.""" + +from uuid import uuid4 + +from app.services.usage.bucket_context import build_bucket_context +from app.services.usage.call_import_context import ( + call_import_evaluation_usage_context, + call_import_ids_from_usage_context, + call_import_row_usage_context, + enrich_usage_context_workspace, +) + + +def test_evaluation_row_gets_distinct_bucket_context(): + org_id = uuid4() + ws_id = uuid4() + eval_id = uuid4() + import_id = uuid4() + row_a = uuid4() + row_b = uuid4() + source_a = uuid4() + source_b = uuid4() + + ctx_a = call_import_evaluation_usage_context( + organization_id=org_id, + workspace_id=ws_id, + evaluation_id=eval_id, + call_import_id=import_id, + evaluation_row_id=row_a, + call_import_row_id=source_a, + ) + ctx_b = call_import_evaluation_usage_context( + organization_id=org_id, + workspace_id=ws_id, + evaluation_id=eval_id, + call_import_id=import_id, + evaluation_row_id=row_b, + call_import_row_id=source_b, + ) + + bucket_a = build_bucket_context( + resource_id=ctx_a.resource_id, + resource_type=ctx_a.resource_type, + extra=ctx_a.extra, + ) + bucket_b = build_bucket_context( + resource_id=ctx_b.resource_id, + resource_type=ctx_b.resource_type, + extra=ctx_b.extra, + ) + assert bucket_a != bucket_b + assert bucket_a["evaluation_row_id"] == str(row_a) + assert bucket_a["call_import_id"] == str(import_id) + + +def test_row_only_transcribe_context(): + org_id = uuid4() + import_id = uuid4() + row_id = uuid4() + ctx = call_import_row_usage_context( + organization_id=org_id, + workspace_id=None, + call_import_id=import_id, + call_import_row_id=row_id, + ) + assert ctx.extra is not None + assert ctx.extra["call_import_row_id"] == str(row_id) + assert ctx.resource_type == "call_import" + + +def test_call_import_ids_from_evaluation_context(): + org_id = uuid4() + eval_id = uuid4() + import_id = uuid4() + row_id = uuid4() + source_id = uuid4() + ctx = call_import_evaluation_usage_context( + organization_id=org_id, + workspace_id=uuid4(), + evaluation_id=eval_id, + call_import_id=import_id, + evaluation_row_id=row_id, + call_import_row_id=source_id, + ) + ids = call_import_ids_from_usage_context(ctx) + assert ids["call_import_id"] == import_id + assert ids["evaluation_id"] == eval_id + assert ids["evaluation_row_id"] == row_id + assert ids["call_import_row_id"] == source_id + + +def test_enrich_usage_context_workspace_noop_when_set(): + ws_id = uuid4() + ctx = call_import_evaluation_usage_context( + organization_id=uuid4(), + workspace_id=ws_id, + evaluation_id=uuid4(), + call_import_id=uuid4(), + ) + assert enrich_usage_context_workspace(ctx).workspace_id == ws_id + + +def test_enrich_usage_context_workspace_from_resolver(): + org_id = uuid4() + import_id = uuid4() + ws_id = uuid4() + ctx = call_import_evaluation_usage_context( + organization_id=org_id, + workspace_id=None, + evaluation_id=uuid4(), + call_import_id=import_id, + ) + from unittest.mock import patch + + with patch( + "app.services.usage.call_import_context.resolve_workspace_id_for_usage_context", + return_value=ws_id, + ): + enriched = enrich_usage_context_workspace(ctx) + assert enriched.workspace_id == ws_id diff --git a/tests/test_services/test_usage/test_llm_usage.py b/tests/test_services/test_usage/test_llm_usage.py index 0fdd77bf..824de3cd 100644 --- a/tests/test_services/test_usage/test_llm_usage.py +++ b/tests/test_services/test_usage/test_llm_usage.py @@ -3,6 +3,7 @@ from __future__ import annotations from datetime import date +from types import SimpleNamespace from typing import Any, Dict, List, Optional from unittest.mock import MagicMock from uuid import uuid4 @@ -105,6 +106,9 @@ def srem(self, key: str, *members: str) -> int: def smembers(self, key: str) -> set: return set(self.sets.get(key, set())) + def sismember(self, key: str, member: str) -> bool: + return member in self.sets.get(key, set()) + def exists(self, key: str) -> int: return int(key in self.hashes or key in self.sets or key in self.kv) @@ -210,9 +214,10 @@ def test_record_increments_pending_and_counts_zero_token_calls(fake_redis, org_c assert prompt == 10 assert completion == 5 assert calls == 2 + assert any(k.rsplit("|", 1)[0].endswith("|llm") for k in fields) -def test_record_skipped_without_context(fake_redis): +def test_record_skipped_without_organization(fake_redis): usage_mod.record_llm_usage( "gpt-test", UsageSnapshot(prompt_tokens=10, completion_tokens=5), @@ -221,7 +226,137 @@ def test_record_skipped_without_context(fake_redis): assert fake_redis.sets == {} -def test_flush_commits_and_acks_claim(fake_redis, org_ctx): +def test_record_with_organization_id_without_context(fake_redis): + org_id = uuid4() + usage_mod.record_llm_usage( + "gpt-test", + UsageSnapshot(prompt_tokens=3, completion_tokens=1), + organization_id=org_id, + usage_date=date(2026, 8, 11), + ) + pending_key = usage_mod._pending_hash_key(org_id) + fields = fake_redis.hgetall(pending_key) + assert fields + assert sum(int(v) for k, v in fields.items() if k.endswith("|call_count")) == 1 + assert any("|other|" in k for k in fields) + + +def test_record_call_usage(fake_redis, org_ctx): + org_id, _workspace_id, ctx = org_ctx + with llm_usage_context(ctx): + usage_mod.record_call_usage( + "voice-agent-call", + audio_seconds=42, + usage_date=date(2026, 8, 11), + ) + + pending_key = usage_mod._pending_hash_key(org_id) + fields = fake_redis.hgetall(pending_key) + audio = sum(int(v) for k, v in fields.items() if k.endswith("|audio_seconds")) + calls = sum(int(v) for k, v in fields.items() if k.endswith("|call_count")) + assert audio == 42 + assert calls == 1 + assert any(k.rsplit("|", 1)[0].endswith("|llm") for k in fields) + + +def test_agent_usage_context_reuses_single_bucket(fake_redis): + """Stable agent context avoids per-call Redis/DB bucket explosion.""" + from app.services.usage.context import usage_context_for_evaluator_result + + org_id = uuid4() + workspace_id = uuid4() + agent_id = uuid4() + prefixes = set() + for idx in range(3): + result = SimpleNamespace( + id=uuid4(), + result_id=f"res-{idx}", + organization_id=org_id, + workspace_id=workspace_id, + evaluator_id=uuid4(), + agent_id=agent_id, + ) + ctx = usage_context_for_evaluator_result(result) + with llm_usage_context(ctx): + usage_mod.record_call_usage( + "voice-agent-call", + audio_seconds=10, + usage_date=date(2026, 8, 13), + ) + fields = fake_redis.hgetall(usage_mod._pending_hash_key(org_id)) + prefixes.update(k.rsplit("|", 1)[0] for k in fields if k.endswith("|call_count")) + + assert len(prefixes) == 1 + calls = sum(int(v) for k, v in fake_redis.hgetall(usage_mod._pending_hash_key(org_id)).items() if k.endswith("|call_count")) + assert calls == 3 + + +def test_record_stt_usage(fake_redis, org_ctx): + org_id, _workspace_id, ctx = org_ctx + with llm_usage_context(ctx): + usage_mod.record_stt_usage( + "nova-2", + audio_seconds=12.2, + usage_date=date(2026, 8, 11), + ) + + pending_key = usage_mod._pending_hash_key(org_id) + fields = fake_redis.hgetall(pending_key) + audio = sum(int(v) for k, v in fields.items() if k.endswith("|audio_seconds")) + calls = sum(int(v) for k, v in fields.items() if k.endswith("|call_count")) + assert audio == 13 # ceil + assert calls == 1 + assert any(k.rsplit("|", 1)[0].endswith("|stt") for k in fields) + + +def test_record_tts_usage(fake_redis, org_ctx): + org_id, _workspace_id, ctx = org_ctx + with llm_usage_context(ctx): + usage_mod.record_tts_usage( + "eleven_flash_v2_5", + characters=142, + usage_date=date(2026, 8, 11), + ) + + pending_key = usage_mod._pending_hash_key(org_id) + fields = fake_redis.hgetall(pending_key) + chars = sum(int(v) for k, v in fields.items() if k.endswith("|tts_characters")) + calls = sum(int(v) for k, v in fields.items() if k.endswith("|call_count")) + assert chars == 142 + assert calls == 1 + assert any(k.rsplit("|", 1)[0].endswith("|tts") for k in fields) + + +def test_redis_failure_buffers_to_postgres(fake_redis, org_ctx, monkeypatch): + org_id, _workspace_id, ctx = org_ctx + + def _boom_pipeline(): + raise usage_mod.redis.RedisError("redis down") + + monkeypatch.setattr(fake_redis, "pipeline", _boom_pipeline) + + captured = {} + + def _fake_buffer(organization_id, bucket, deltas): + captured["organization_id"] = organization_id + captured["bucket"] = bucket + captured["deltas"] = deltas + + monkeypatch.setattr(usage_mod, "_buffer_to_postgres", _fake_buffer) + + with llm_usage_context(ctx): + usage_mod.record_llm_usage( + "gpt-test", + UsageSnapshot(prompt_tokens=4, completion_tokens=2), + usage_date=date(2026, 8, 11), + ) + + assert captured["organization_id"] == org_id + assert captured["deltas"]["prompt_tokens"] == 4 + assert captured["bucket"]["usage_kind"] == "llm" + + +def test_flush_commits_and_acks_claim(fake_redis, org_ctx, monkeypatch): org_id, _workspace_id, ctx = org_ctx with llm_usage_context(ctx): usage_mod.record_llm_usage( @@ -232,6 +367,7 @@ def test_flush_commits_and_acks_claim(fake_redis, org_ctx): db = MagicMock() db.execute.return_value = MagicMock(rowcount=1) + monkeypatch.setattr(usage_mod, "_flush_pending_buffer", lambda _db, _org: 0) flushed = usage_mod.flush_usage_to_catalog(db, org_id) assert flushed == 1 @@ -242,7 +378,7 @@ def test_flush_commits_and_acks_claim(fake_redis, org_ctx): assert fake_redis.get(usage_mod._flush_lock_key(org_id)) is None -def test_flush_restores_redis_when_db_fails(fake_redis, org_ctx): +def test_flush_restores_redis_when_db_fails(fake_redis, org_ctx, monkeypatch): org_id, _workspace_id, ctx = org_ctx with llm_usage_context(ctx): usage_mod.record_llm_usage( @@ -253,6 +389,7 @@ def test_flush_restores_redis_when_db_fails(fake_redis, org_ctx): db = MagicMock() db.execute.side_effect = RuntimeError("db down") + monkeypatch.setattr(usage_mod, "_flush_pending_buffer", lambda _db, _org: 0) flushed = usage_mod.flush_usage_to_catalog(db, org_id) assert flushed == 0 @@ -270,7 +407,7 @@ def test_flush_restores_redis_when_db_fails(fake_redis, org_ctx): assert not any(k.startswith("usage:flushing:") for k in fake_redis.hashes) -def test_flush_drops_pending_when_organization_missing(fake_redis, org_ctx): +def test_flush_drops_pending_when_organization_missing(fake_redis, org_ctx, monkeypatch): """Unknown org FK must not restore Redis (avoids infinite beat retries).""" from sqlalchemy.exc import IntegrityError @@ -291,6 +428,7 @@ def test_flush_drops_pending_when_organization_missing(fake_redis, org_ctx): 'constraint "llm_usage_daily_organization_id_fkey"' ), ) + monkeypatch.setattr(usage_mod, "_flush_pending_buffer", lambda _db, _org: 0) flushed = usage_mod.flush_usage_to_catalog(db, org_id) assert flushed == 0 @@ -300,7 +438,7 @@ def test_flush_drops_pending_when_organization_missing(fake_redis, org_ctx): assert not any(k.startswith("usage:flushing:") for k in fake_redis.hashes) -def test_concurrent_flush_does_not_double_count(fake_redis, org_ctx): +def test_concurrent_flush_does_not_double_count(fake_redis, org_ctx, monkeypatch): org_id, _workspace_id, ctx = org_ctx with llm_usage_context(ctx): usage_mod.record_llm_usage( @@ -313,6 +451,7 @@ def test_concurrent_flush_does_not_double_count(fake_redis, org_ctx): db_a.execute.return_value = MagicMock(rowcount=0) # force INSERT path db_b = MagicMock() db_b.execute.return_value = MagicMock(rowcount=1) + monkeypatch.setattr(usage_mod, "_flush_pending_buffer", lambda _db, _org: 0) first = usage_mod.flush_usage_to_catalog(db_a, org_id) second = usage_mod.flush_usage_to_catalog(db_b, org_id) @@ -385,3 +524,64 @@ def test_ensure_uses_workspace_and_section_hints(): finally: reset_usage_hints(hint_tokens) reset_usage_context(ctx_token) + + +def test_parse_legacy_resource_bucket_prefix(): + rid = uuid4() + parsed = usage_mod._parse_bucket_prefix( + f"{uuid4()}|call_imports|nova-2|{rid}|call_import|2026-08-11|stt" + ) + assert parsed is not None + assert parsed["usage_kind"] == "stt" + assert parsed["context"]["resource_id"] == str(rid) + assert parsed["context"]["resource_type"] == "call_import" + + +def test_parse_context_bucket_prefix(): + ctx = '{"resource_id":"00000000-0000-0000-0000-000000000001","resource_type":"call_import_evaluation"}' + parsed = usage_mod._parse_bucket_prefix( + f"{uuid4()}|call_imports|gpt-4|{ctx}|2026-08-11|llm" + ) + assert parsed is not None + assert parsed["usage_kind"] == "llm" + assert parsed["context"]["resource_id"] == "00000000-0000-0000-0000-000000000001" + + +def test_upsert_bucket_sql_uses_valid_empty_jsonb_literal(): + """Regression: '{{}}'::jsonb is invalid JSON and breaks catalog flush.""" + import inspect + + source = inspect.getsource(usage_mod._upsert_bucket) + assert "'{{}}'::jsonb" not in source + assert "'{}'::jsonb" in source + + +def test_upsert_bucket_matches_legacy_resource_context_key(monkeypatch): + """Per-row context must merge into an existing evaluation-level bucket.""" + org_id = uuid4() + evaluation_id = uuid4() + bucket = { + "workspace_id": uuid4(), + "product_section": "call_import_evaluations", + "model": "gpt-test", + "context": { + "resource_id": str(evaluation_id), + "resource_type": "call_import_evaluation", + "evaluation_row_id": str(uuid4()), + }, + "usage_date": date(2026, 8, 12), + "usage_kind": "llm", + } + deltas = {"prompt_tokens": 10, "completion_tokens": 5, "call_count": 1} + + exact_update = MagicMock(rowcount=0) + legacy_update = MagicMock(rowcount=1) + db = MagicMock() + db.execute.side_effect = [exact_update, legacy_update] + + usage_mod._upsert_bucket(db, org_id, bucket, deltas) + + assert db.execute.call_count == 2 + legacy_sql = str(db.execute.call_args_list[1][0][0]) + assert "context->>'resource_id'" in legacy_sql + assert "context->>'resource_type'" in legacy_sql diff --git a/tests/test_services/test_usage/test_usage_dates.py b/tests/test_services/test_usage/test_usage_dates.py new file mode 100644 index 00000000..b73290c9 --- /dev/null +++ b/tests/test_services/test_usage/test_usage_dates.py @@ -0,0 +1,32 @@ +"""Tests for usage date timezone mapping.""" + +from datetime import date + +from app.services.usage.dates import usage_date_filter_bounds, usage_local_today + + +def test_usage_date_filter_bounds_india_late_night(): + # 1:26 AM IST on Aug 13 is still Aug 12 in UTC usage_date bucket. + start = date(2026, 8, 13) + end = date(2026, 8, 13) + filter_start, filter_end = usage_date_filter_bounds(start, end, "Asia/Kolkata") + assert filter_start == date(2026, 8, 12) + assert filter_end == date(2026, 8, 13) + + +def test_usage_date_filter_bounds_without_tz_uses_exact_dates(): + start = date(2026, 8, 13) + end = date(2026, 8, 13) + assert usage_date_filter_bounds(start, end, None) == (start, end) + + +def test_usage_date_filter_bounds_utc_matches_calendar_days(): + start = date(2026, 8, 13) + end = date(2026, 8, 13) + assert usage_date_filter_bounds(start, end, "UTC") == (start, end) + + +def test_usage_local_today_respects_timezone(): + today_utc = usage_local_today("UTC") + today_india = usage_local_today("Asia/Kolkata") + assert today_utc <= today_india or today_india <= today_utc diff --git a/tests/test_services/test_usage/test_usage_labels.py b/tests/test_services/test_usage/test_usage_labels.py new file mode 100644 index 00000000..233fe900 --- /dev/null +++ b/tests/test_services/test_usage/test_usage_labels.py @@ -0,0 +1,287 @@ +"""Tests for usage label helpers.""" + +from uuid import UUID + +from app.services.usage.usage_labels import ( + build_usage_resource_label, + format_entity_label, + labels_for_resource_buckets, + usage_kind_label, + UsageNameResolver, +) + + +class _FakeEvaluation: + def __init__(self, id, name): + self.id = id + self.name = name + + +class _FakeCallImport: + def __init__(self, id, dataset=None, original_filename=None): + self.id = id + self.dataset = dataset + self.original_filename = original_filename + + +class _FakeCallImportRow: + def __init__(self, id, conversation_id): + self.id = id + self.conversation_id = conversation_id + + +class _FakeTTSComparison: + def __init__(self, id, name=None, simulation_id=None): + self.id = id + self.name = name + self.simulation_id = simulation_id + + +class _FakeAgent: + def __init__(self, id, name=None, agent_id=None): + self.id = id + self.name = name + self.agent_id = agent_id + + +class _FakeQuery: + def __init__(self, rows): + self._rows = rows + + def filter(self, *args, **kwargs): + return self + + def join(self, *args, **kwargs): + return self + + def all(self): + return self._rows + + +class _FakeDb: + def __init__(self, evaluations=(), imports=(), rows=(), tts_comparisons=(), agents=()): + self._evaluations = evaluations + self._imports = imports + self._rows = rows + self._tts_comparisons = tts_comparisons + self._agents = agents + + def query(self, *entities): + if len(entities) == 1: + model = entities[0] + if model.__name__ == "CallImportEvaluation": + return _FakeQuery(self._evaluations) + if model.__name__ == "CallImport": + return _FakeQuery(self._imports) + if model.__name__ == "CallImportRow": + return _FakeQuery(self._rows) + if model.__name__ == "TTSComparison": + return _FakeQuery(self._tts_comparisons) + if model.__name__ == "Agent": + return _FakeQuery(self._agents) + raise AssertionError(f"unexpected model {model}") + return _FakeQuery(()) + + +EVAL_ID = UUID("984039ab-cdef-4567-8901-234567890abc") +IMPORT_ID = UUID("3111d376-e5f6-7890-abcd-ef1234567890") +ROW_ID = UUID("fedcba98-7654-3210-fedc-ba9876543210") +COMP_ID = UUID("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee") +AGENT_ID = UUID("bbbbbbbb-cccc-dddd-eeee-ffffffffffff") +ORG_ID = UUID("11111111-2222-3333-4444-555555555555") + + +def test_format_entity_label_custom_name_and_default(): + assert format_entity_label("test", EVAL_ID, "Evaluation") == "test-984039ab" + assert format_entity_label(None, EVAL_ID, "Evaluation") == "Evaluation-984039ab" + + +def test_call_import_label_includes_dataset_and_tags(): + db = _FakeDb( + imports=[ + _FakeCallImport( + IMPORT_ID, + dataset="QA batch", + original_filename="4 manual recordings", + ) + ] + ) + # Extend fake db for tag query + class _TagQuery: + def join(self, *args, **kwargs): + return self + + def filter(self, *args, **kwargs): + return self + + def all(self): + return [(IMPORT_ID, "support")] + + def query_router(*entities): + if len(entities) == 1: + model = entities[0] + if model.__name__ == "CallImportTagAssignment": + return _TagQuery() + if model.__name__ == "CallImportEvaluation": + return _FakeQuery(()) + if model.__name__ == "CallImport": + return _FakeQuery(db._imports) + if model.__name__ == "CallImportRow": + return _FakeQuery(()) + return _TagQuery() + + db.query = query_router # type: ignore[method-assign] + + resolver = UsageNameResolver(db, ORG_ID) + resolver.preload([{"call_import_id": str(IMPORT_ID)}]) + label = resolver.call_import_name(str(IMPORT_ID)) + assert "4 manual recordings" in label + assert "QA batch" in label + assert "support" in label + + +def test_call_import_label_prefers_filename_over_dataset(): + db = _FakeDb( + imports=[ + _FakeCallImport( + IMPORT_ID, + dataset="calls", + original_filename="unauthenticated sheet.xlsx", + ) + ] + ) + resolver = UsageNameResolver(db, ORG_ID) + resolver.preload([{"call_import_id": str(IMPORT_ID)}]) + assert resolver.call_import_name(str(IMPORT_ID)) == ( + "unauthenticated sheet.xlsx (calls)-3111d376" + ) + + +def test_build_usage_resource_label_hierarchical(): + db = _FakeDb( + evaluations=[_FakeEvaluation(EVAL_ID, "March QA pass")], + imports=[ + _FakeCallImport( + IMPORT_ID, + dataset="calls", + original_filename="unauthenticated sheet.xlsx", + ) + ], + rows=[_FakeCallImportRow(ROW_ID, "ext-call-8842")], + ) + resolver = UsageNameResolver(db, ORG_ID) + resolver.preload( + [ + { + "evaluation_id": str(EVAL_ID), + "call_import_id": str(IMPORT_ID), + "call_import_row_id": str(ROW_ID), + "resource_type": "call_import_evaluation", + "resource_id": str(EVAL_ID), + } + ] + ) + label = build_usage_resource_label( + { + "evaluation_id": str(EVAL_ID), + "call_import_id": str(IMPORT_ID), + "call_import_row_id": str(ROW_ID), + "resource_type": "call_import_evaluation", + "resource_id": str(EVAL_ID), + }, + "call_import_evaluation", + resolver, + ) + assert label == ( + "unauthenticated sheet.xlsx (calls)-3111d376 / March QA pass-984039ab / ext-call-8842" + ) + + +def test_build_usage_resource_label_default_evaluation_name(): + db = _FakeDb(evaluations=[_FakeEvaluation(EVAL_ID, None)]) + resolver = UsageNameResolver(db, ORG_ID) + resolver.preload( + [ + { + "evaluation_id": str(EVAL_ID), + "resource_type": "call_import_evaluation", + "resource_id": str(EVAL_ID), + } + ] + ) + label = build_usage_resource_label( + {"evaluation_id": str(EVAL_ID)}, + "call_import_evaluation", + resolver, + ) + assert label == "Evaluation-984039ab" + + +def test_build_usage_resource_label_tts_comparison_with_simulation_id(): + db = _FakeDb( + tts_comparisons=[ + _FakeTTSComparison( + COMP_ID, + name="elevenlabs benchmark", + simulation_id="781879", + ) + ] + ) + resolver = UsageNameResolver(db, ORG_ID) + resolver.preload( + [ + { + "resource_type": "tts_comparison", + "resource_id": str(COMP_ID), + } + ] + ) + label = build_usage_resource_label( + { + "resource_type": "tts_comparison", + "resource_id": str(COMP_ID), + }, + "tts_comparison", + resolver, + ) + assert label == "elevenlabs benchmark #781879" + + +def test_build_usage_resource_label_agent_with_short_id(): + db = _FakeDb( + agents=[_FakeAgent(AGENT_ID, name="Support bot", agent_id="482910")] + ) + resolver = UsageNameResolver(db, ORG_ID) + resolver.preload( + [ + { + "resource_type": "agent", + "resource_id": str(AGENT_ID), + } + ] + ) + label = build_usage_resource_label( + {"resource_type": "agent", "resource_id": str(AGENT_ID)}, + "agent", + resolver, + ) + assert label == "Support bot #482910" + + +def test_labels_for_resource_buckets_empty_context_uses_bucket_id(): + db = _FakeDb( + agents=[_FakeAgent(AGENT_ID, name="Support bot", agent_id="482910")] + ) + resolver = UsageNameResolver(db, ORG_ID) + resolver.preload([{"resource_type": "agent", "resource_id": str(AGENT_ID)}]) + labels = labels_for_resource_buckets( + [(str(AGENT_ID), "agent", [{}])], + resolver, + ) + assert labels[str(AGENT_ID)] == "Support bot #482910" + + +def test_usage_kind_label(): + assert usage_kind_label("stt") == "STT" + assert usage_kind_label("llm") == "LLM" + assert usage_kind_label("tts") == "TTS" diff --git a/tests/test_workers/test_process_evaluator_result_helpers.py b/tests/test_workers/test_process_evaluator_result_helpers.py index 794248a3..0600c22f 100644 --- a/tests/test_workers/test_process_evaluator_result_helpers.py +++ b/tests/test_workers/test_process_evaluator_result_helpers.py @@ -1,28 +1,37 @@ -"""Helper tests for process_evaluator_result task module.""" +"""Unit tests for process_evaluator_result helper utilities.""" + +import importlib +from types import SimpleNamespace +from uuid import uuid4 -import importlib.util -from pathlib import Path -_TASK_PATH = Path(__file__).resolve().parents[2] / "app" / "workers" / "tasks" / "process_evaluator_result.py" -_TASK_SPEC = importlib.util.spec_from_file_location("process_evaluator_result_under_test", _TASK_PATH) -process_evaluator_result = importlib.util.module_from_spec(_TASK_SPEC) -assert _TASK_SPEC is not None and _TASK_SPEC.loader is not None -_TASK_SPEC.loader.exec_module(process_evaluator_result) +def _task_module(): + return importlib.import_module("app.workers.tasks.process_evaluator_result") def test_extract_audio_url_supports_smallest_recordings(): + task_module = _task_module() call_data = {"recording_url": "https://audio.smallest.ai/call.wav"} - audio_url = process_evaluator_result._extract_audio_url(call_data, "smallest") + assert task_module._extract_audio_url(call_data, "smallest") == "https://audio.smallest.ai/call.wav" - assert audio_url == "https://audio.smallest.ai/call.wav" -"""Unit tests for process_evaluator_result helper utilities.""" +def test_should_record_external_agent_call_usage_only_for_provider_calls(): + task_module = _task_module() + agent_id = uuid4() -def test_extract_audio_url_supports_smallest_recordings(): - import importlib + external = SimpleNamespace(agent_id=agent_id, provider_platform="vapi") + internal = SimpleNamespace(agent_id=agent_id, provider_platform=None) + simulation = SimpleNamespace(agent_id=agent_id, provider_platform="") + no_agent = SimpleNamespace(agent_id=None, provider_platform="vapi") - task_module = importlib.import_module("app.workers.tasks.process_evaluator_result") + assert task_module._should_record_external_agent_call_usage(external) is True + assert task_module._should_record_external_agent_call_usage(internal) is False + assert task_module._should_record_external_agent_call_usage(simulation) is False + assert task_module._should_record_external_agent_call_usage(no_agent) is False - call_data = {"recording_url": "https://audio.smallest.ai/call.wav"} - assert task_module._extract_audio_url(call_data, "smallest") == "https://audio.smallest.ai/call.wav" +def test_resolve_call_duration_seconds_prefers_result_field(): + task_module = _task_module() + result = SimpleNamespace(duration_seconds=12.4, call_data={}) + assert task_module._resolve_call_duration_seconds(result) == 13 + From 09b9793483c7679b938e6603c474d79d7e556267 Mon Sep 17 00:00:00 2001 From: M Sami Date: Thu, 13 Aug 2026 11:49:00 +0530 Subject: [PATCH 17/32] fix: committed claims management in db --- .../063_usage_kind_stt_and_buffer.py | 44 ++++++++++ app/services/usage/llm_usage.py | 84 +++++++++++++++++-- ...st_call_import_diarization_and_eval_llm.py | 2 + .../test_usage/test_llm_usage.py | 4 +- 4 files changed, 124 insertions(+), 10 deletions(-) diff --git a/app/migrations/063_usage_kind_stt_and_buffer.py b/app/migrations/063_usage_kind_stt_and_buffer.py index 1cbb51d9..fb1d9e4c 100644 --- a/app/migrations/063_usage_kind_stt_and_buffer.py +++ b/app/migrations/063_usage_kind_stt_and_buffer.py @@ -146,12 +146,33 @@ def _ensure_unique_bucket_index(db: Session) -> None: ) +def _ensure_llm_usage_daily_base_columns(db: Session) -> None: + """Align pre-062 tables with the 062 schema before dedupe/index steps.""" + if not _column_exists(db, "llm_usage_daily", "resource_id"): + db.execute(text("ALTER TABLE llm_usage_daily ADD COLUMN resource_id UUID")) + if not _column_exists(db, "llm_usage_daily", "resource_type"): + db.execute( + text("ALTER TABLE llm_usage_daily ADD COLUMN resource_type VARCHAR(64)") + ) + if not _column_exists(db, "llm_usage_daily", "updated_at"): + db.execute( + text( + """ + ALTER TABLE llm_usage_daily + ADD COLUMN updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + """ + ) + ) + + def upgrade(db: Session): if not _table_exists(db, "llm_usage_daily"): print("llm_usage_daily missing; run 062 first — skipping 063") db.commit() return + _ensure_llm_usage_daily_base_columns(db) + if not _column_exists(db, "llm_usage_daily", "usage_kind"): db.execute( text( @@ -223,10 +244,33 @@ def upgrade(db: Session): ) print("Created usage_pending_buffer") + if not _table_exists(db, "usage_committed_claims"): + db.execute( + text( + """ + CREATE TABLE usage_committed_claims ( + claim_key TEXT PRIMARY KEY, + organization_id UUID NOT NULL, + committed_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_usage_committed_claims_committed_at + ON usage_committed_claims (committed_at) + """ + ) + ) + print("Created usage_committed_claims") + db.commit() def downgrade(db: Session): + db.execute(text("DROP TABLE IF EXISTS usage_committed_claims")) db.execute(text("DROP TABLE IF EXISTS usage_pending_buffer")) db.execute(text("DROP INDEX IF EXISTS ix_llm_usage_daily_org_kind_date")) # Keep usage_kind/audio_seconds columns on downgrade to avoid data loss. diff --git a/app/services/usage/llm_usage.py b/app/services/usage/llm_usage.py index 33fda254..6b3cb65d 100644 --- a/app/services/usage/llm_usage.py +++ b/app/services/usage/llm_usage.py @@ -522,24 +522,88 @@ def _restore_buckets_to_pending( _CLAIM_COMMITTED_TTL_SECONDS = 24 * 60 * 60 +_CLAIM_COMMITTED_REDIS_RETRIES = 5 def _claim_committed_key(claim_key: str) -> str: return f"usage:claim_done:{claim_key}" -def _mark_claim_committed(claim_key: str) -> None: +def _record_claim_committed_pg( + db: Session, claim_key: str, organization_id: UUID +) -> None: try: - _client().set(_claim_committed_key(claim_key), "1", ex=_CLAIM_COMMITTED_TTL_SECONDS) - except redis.RedisError: - pass + db.execute( + text( + """ + INSERT INTO usage_committed_claims (claim_key, organization_id) + VALUES (:claim_key, CAST(:organization_id AS uuid)) + ON CONFLICT (claim_key) DO NOTHING + """ + ), + { + "claim_key": claim_key, + "organization_id": str(organization_id), + }, + ) + except Exception as exc: + logger.debug("usage committed claim pg write skipped: {}", exc) + + +def _is_claim_committed_pg(claim_key: str) -> bool: + try: + from app.database import SessionLocal + + db = SessionLocal() + try: + return ( + db.execute( + text( + """ + SELECT 1 FROM usage_committed_claims + WHERE claim_key = :claim_key + """ + ), + {"claim_key": claim_key}, + ).first() + is not None + ) + finally: + db.close() + except Exception: + return False + + +def _mark_claim_committed(claim_key: str) -> bool: + for attempt in range(_CLAIM_COMMITTED_REDIS_RETRIES): + try: + _client().set( + _claim_committed_key(claim_key), + "1", + ex=_CLAIM_COMMITTED_TTL_SECONDS, + ) + return True + except redis.RedisError: + if attempt + 1 < _CLAIM_COMMITTED_REDIS_RETRIES: + time.sleep(0.05 * (attempt + 1)) + return False def _is_claim_committed(claim_key: str) -> bool: try: - return bool(_client().exists(_claim_committed_key(claim_key))) + if _client().exists(_claim_committed_key(claim_key)): + return True except redis.RedisError: - return False + pass + return _is_claim_committed_pg(claim_key) + + +def _finalize_committed_claim( + db: Session, claim_key: str, organization_id: UUID +) -> None: + _record_claim_committed_pg(db, claim_key, organization_id) + _mark_claim_committed(claim_key) + _ack_claim(claim_key, organization_id) def _has_pending_usage(organization_id: UUID) -> bool: @@ -897,6 +961,8 @@ def flush_usage_to_catalog(db: Session, organization_id: UUID, *, force: bool = deltas, ) flushed += 1 + if claim_key: + _record_claim_committed_pg(db, claim_key, organization_id) db.commit() except Exception as exc: db.rollback() @@ -907,8 +973,10 @@ def flush_usage_to_catalog(db: Session, organization_id: UUID, *, force: bool = exc, ) if claim_key: - _mark_claim_committed(claim_key) - _ack_claim(claim_key, organization_id) + _finalize_committed_claim( + db, claim_key, organization_id + ) + db.commit() claim_key = None buckets = {} else: diff --git a/tests/test_api/test_call_import_diarization_and_eval_llm.py b/tests/test_api/test_call_import_diarization_and_eval_llm.py index 936460d7..cb890fdc 100644 --- a/tests/test_api/test_call_import_diarization_and_eval_llm.py +++ b/tests/test_api/test_call_import_diarization_and_eval_llm.py @@ -32,6 +32,8 @@ def _make_fake_row(): return SimpleNamespace( id=uuid4(), organization_id=uuid4(), + call_import_id=uuid4(), + workspace_id=uuid4(), recording_s3_key="s3://bucket/key.wav", transcript="production value from CSV", transcript_source="csv", diff --git a/tests/test_services/test_usage/test_llm_usage.py b/tests/test_services/test_usage/test_llm_usage.py index 824de3cd..d39bb501 100644 --- a/tests/test_services/test_usage/test_llm_usage.py +++ b/tests/test_services/test_usage/test_llm_usage.py @@ -460,8 +460,8 @@ def test_concurrent_flush_does_not_double_count(fake_redis, org_ctx, monkeypatch assert second == 0 assert db_a.commit.call_count == 1 assert db_b.commit.call_count == 0 - # INSERT once for the claimed bucket; second flusher never upserts. - assert db_a.execute.call_count == 2 # UPDATE miss + INSERT + # One bucket upsert (2 UPDATE misses + SAVEPOINT/INSERT/RELEASE) plus committed-claim row. + assert db_a.execute.call_count == 6 assert db_b.execute.call_count == 0 From 5c51608ab8f3b4613b81c7834a5c155f7c8bdc8f Mon Sep 17 00:00:00 2001 From: M Sami Date: Thu, 13 Aug 2026 12:01:06 +0530 Subject: [PATCH 18/32] refactor: enhanced Redis handling and database transaction integrity --- app/services/usage/llm_usage.py | 163 +++++++++++++----- .../test_usage/test_llm_usage.py | 63 +++++++ 2 files changed, 179 insertions(+), 47 deletions(-) diff --git a/app/services/usage/llm_usage.py b/app/services/usage/llm_usage.py index 6b3cb65d..b992b9e5 100644 --- a/app/services/usage/llm_usage.py +++ b/app/services/usage/llm_usage.py @@ -522,60 +522,104 @@ def _restore_buckets_to_pending( _CLAIM_COMMITTED_TTL_SECONDS = 24 * 60 * 60 -_CLAIM_COMMITTED_REDIS_RETRIES = 5 +_CLAIM_COMMITTED_REDIS_RETRIES = 3 +_ORPHAN_RECOVERY_INTERVAL_SEC = 30 +_COMMITTED_CLAIMS_PRUNE_INTERVAL_SEC = 3600 +_COMMITTED_CLAIMS_RETENTION_DAYS = 2 def _claim_committed_key(claim_key: str) -> str: return f"usage:claim_done:{claim_key}" +def _redis_interval_gate(key: str, interval_sec: int) -> bool: + """Return True when the gated work should run (interval lock acquired).""" + try: + return bool(_client().set(key, "1", nx=True, ex=interval_sec)) + except redis.RedisError: + return False + + def _record_claim_committed_pg( db: Session, claim_key: str, organization_id: UUID ) -> None: + db.execute( + text( + """ + INSERT INTO usage_committed_claims (claim_key, organization_id) + VALUES (:claim_key, CAST(:organization_id AS uuid)) + ON CONFLICT (claim_key) DO NOTHING + """ + ), + { + "claim_key": claim_key, + "organization_id": str(organization_id), + }, + ) + + +def _committed_claim_keys_in_pg(claim_keys: List[str]) -> set[str]: + if not claim_keys: + return set() try: - db.execute( - text( - """ - INSERT INTO usage_committed_claims (claim_key, organization_id) - VALUES (:claim_key, CAST(:organization_id AS uuid)) - ON CONFLICT (claim_key) DO NOTHING - """ - ), - { - "claim_key": claim_key, - "organization_id": str(organization_id), - }, - ) - except Exception as exc: - logger.debug("usage committed claim pg write skipped: {}", exc) + from app.database import SessionLocal + db = SessionLocal() + try: + rows = db.execute( + text( + """ + SELECT claim_key + FROM usage_committed_claims + WHERE claim_key = ANY(CAST(:claim_keys AS text[])) + """ + ), + {"claim_keys": claim_keys}, + ).all() + return {row[0] for row in rows} + finally: + db.close() + except Exception: + return set() -def _is_claim_committed_pg(claim_key: str) -> bool: + +def _prune_old_committed_claims() -> None: try: from app.database import SessionLocal db = SessionLocal() try: - return ( - db.execute( - text( - """ - SELECT 1 FROM usage_committed_claims - WHERE claim_key = :claim_key - """ - ), - {"claim_key": claim_key}, - ).first() - is not None + db.execute( + text( + f""" + DELETE FROM usage_committed_claims + WHERE committed_at < now() - interval '{_COMMITTED_CLAIMS_RETENTION_DAYS} days' + """ + ) ) + db.commit() + except Exception as exc: + db.rollback() + logger.debug("usage committed claim prune skipped: {}", exc) finally: db.close() except Exception: - return False + pass -def _mark_claim_committed(claim_key: str) -> bool: - for attempt in range(_CLAIM_COMMITTED_REDIS_RETRIES): +def _discard_committed_claim(claim_key: str, organization_id: UUID) -> None: + """Best-effort Redis cleanup when usage was not (or must not be) persisted.""" + _mark_claim_committed(claim_key, fast=True) + _ack_claim(claim_key, organization_id) + + +def _is_claim_committed_pg(claim_key: str) -> bool: + return claim_key in _committed_claim_keys_in_pg([claim_key]) + + +def _mark_claim_committed(claim_key: str, *, fast: bool = False) -> bool: + max_attempts = 2 if fast else _CLAIM_COMMITTED_REDIS_RETRIES + for attempt in range(max_attempts): try: _client().set( _claim_committed_key(claim_key), @@ -584,7 +628,7 @@ def _mark_claim_committed(claim_key: str) -> bool: ) return True except redis.RedisError: - if attempt + 1 < _CLAIM_COMMITTED_REDIS_RETRIES: + if attempt + 1 < max_attempts and not fast: time.sleep(0.05 * (attempt + 1)) return False @@ -598,14 +642,6 @@ def _is_claim_committed(claim_key: str) -> bool: return _is_claim_committed_pg(claim_key) -def _finalize_committed_claim( - db: Session, claim_key: str, organization_id: UUID -) -> None: - _record_claim_committed_pg(db, claim_key, organization_id) - _mark_claim_committed(claim_key) - _ack_claim(claim_key, organization_id) - - def _has_pending_usage(organization_id: UUID) -> bool: try: client = _client() @@ -973,10 +1009,14 @@ def flush_usage_to_catalog(db: Session, organization_id: UUID, *, force: bool = exc, ) if claim_key: - _finalize_committed_claim( - db, claim_key, organization_id - ) - db.commit() + try: + _record_claim_committed_pg( + db, claim_key, organization_id + ) + db.commit() + except Exception: + db.rollback() + _discard_committed_claim(claim_key, organization_id) claim_key = None buckets = {} else: @@ -993,11 +1033,11 @@ def flush_usage_to_catalog(db: Session, organization_id: UUID, *, force: bool = except redis.RedisError: pass claim_key = None - return flushed + _flush_pending_buffer(db, organization_id) + return _flush_pending_buffer(db, organization_id) if skipped: _restore_buckets_to_pending(organization_id, skipped) if claim_key: - _mark_claim_committed(claim_key) + _mark_claim_committed(claim_key, fast=True) _ack_claim(claim_key, organization_id) claim_key = None finally: @@ -1009,8 +1049,13 @@ def flush_usage_to_catalog(db: Session, organization_id: UUID, *, force: bool = def _recover_orphaned_claims() -> None: + if not _redis_interval_gate( + "usage:orphan_recovery:due", _ORPHAN_RECOVERY_INTERVAL_SEC + ): + return try: client = _client() + candidates: List[Tuple[str, UUID]] = [] for claim_key in client.scan_iter(match="usage:flushing:*", count=100): parts = claim_key.split(":") if len(parts) < 4: @@ -1021,13 +1066,37 @@ def _recover_orphaned_claims() -> None: continue if client.exists(_flush_lock_key(org_id)): continue - if _is_claim_committed(claim_key): + candidates.append((claim_key, org_id)) + + if not candidates: + return + + pipe = client.pipeline() + for claim_key, _org_id in candidates: + pipe.exists(_claim_committed_key(claim_key)) + redis_committed_flags = pipe.execute() + + needs_pg: List[Tuple[str, UUID]] = [] + for index, (claim_key, org_id) in enumerate(candidates): + if redis_committed_flags[index]: + client.delete(claim_key) + continue + needs_pg.append((claim_key, org_id)) + + pg_committed = _committed_claim_keys_in_pg([key for key, _ in needs_pg]) + for claim_key, org_id in needs_pg: + if claim_key in pg_committed: client.delete(claim_key) continue buckets = _read_hash_buckets(claim_key) if buckets: _restore_buckets_to_pending(org_id, buckets) client.delete(claim_key) + + if _redis_interval_gate( + "usage:committed_claims:prune", _COMMITTED_CLAIMS_PRUNE_INTERVAL_SEC + ): + _prune_old_committed_claims() except redis.RedisError as exc: logger.warning("llm usage orphan claim recovery failed: {}", exc) diff --git a/tests/test_services/test_usage/test_llm_usage.py b/tests/test_services/test_usage/test_llm_usage.py index d39bb501..4abdf609 100644 --- a/tests/test_services/test_usage/test_llm_usage.py +++ b/tests/test_services/test_usage/test_llm_usage.py @@ -41,6 +41,10 @@ def expire(self, key: str, ttl: int): self._ops.append(("expire", key, ttl)) return self + def exists(self, key: str): + self._ops.append(("exists", key)) + return self + def execute(self): results = [] for op in self._ops: @@ -51,6 +55,8 @@ def execute(self): results.append(self._client.sadd(op[1], *op[2])) elif kind == "expire": results.append(self._client.expire(op[1], op[2])) + elif kind == "exists": + results.append(self._client.exists(op[1])) self._ops.clear() return results @@ -129,6 +135,9 @@ def delete(self, *keys: str) -> int: def expire(self, key: str, ttl: int) -> bool: return self.exists(key) == 1 + def pipeline(self): + return _FakePipeline(self) + def set(self, key: str, value: str, nx: bool = False, ex: Optional[int] = None) -> Optional[bool]: if nx and key in self.kv: return None @@ -407,6 +416,39 @@ def test_flush_restores_redis_when_db_fails(fake_redis, org_ctx, monkeypatch): assert not any(k.startswith("usage:flushing:") for k in fake_redis.hashes) +def test_flush_restores_redis_when_committed_claim_insert_fails( + fake_redis, org_ctx, monkeypatch +): + """Committed-claim insert must share the usage transaction; failure rolls back.""" + org_id, _workspace_id, ctx = org_ctx + with llm_usage_context(ctx): + usage_mod.record_llm_usage( + "gpt-test", + UsageSnapshot(prompt_tokens=5, completion_tokens=2), + usage_date=date(2026, 8, 11), + ) + + db = MagicMock() + + def _execute_side_effect(statement, *_args, **_kwargs): + if "usage_committed_claims" in str(statement): + raise RuntimeError("usage_committed_claims unavailable") + return MagicMock(rowcount=0) + + db.execute.side_effect = _execute_side_effect + monkeypatch.setattr(usage_mod, "_flush_pending_buffer", lambda _db, _org: 0) + + flushed = usage_mod.flush_usage_to_catalog(db, org_id) + assert flushed == 0 + db.rollback.assert_called() + db.commit.assert_not_called() + + pending = fake_redis.hgetall(usage_mod._pending_hash_key(org_id)) + prompt = sum(int(v) for k, v in pending.items() if k.endswith("|prompt_tokens")) + assert prompt == 5 + assert str(org_id) in fake_redis.smembers("usage:pending:orgs") + + def test_flush_drops_pending_when_organization_missing(fake_redis, org_ctx, monkeypatch): """Unknown org FK must not restore Redis (avoids infinite beat retries).""" from sqlalchemy.exc import IntegrityError @@ -585,3 +627,24 @@ def test_upsert_bucket_matches_legacy_resource_context_key(monkeypatch): legacy_sql = str(db.execute.call_args_list[1][0][0]) assert "context->>'resource_id'" in legacy_sql assert "context->>'resource_type'" in legacy_sql + + +def test_orphan_recovery_runs_at_most_once_per_interval(fake_redis, monkeypatch): + org_id = uuid4() + claim_key = f"usage:flushing:{org_id}:{uuid4()}" + fake_redis.hashes[claim_key] = {"bucket|prompt_tokens": 3} + + pg_lookups = 0 + + def _count_pg_lookup(keys): + nonlocal pg_lookups + pg_lookups += 1 + return set() + + monkeypatch.setattr(usage_mod, "_committed_claim_keys_in_pg", _count_pg_lookup) + monkeypatch.setattr(usage_mod, "_prune_old_committed_claims", lambda: None) + + usage_mod._recover_orphaned_claims() + usage_mod._recover_orphaned_claims() + + assert pg_lookups == 1 From 4dd9fdc519dd4f7485b59a3db9ec7b1306a8c4e6 Mon Sep 17 00:00:00 2001 From: M Sami Date: Thu, 13 Aug 2026 12:17:01 +0530 Subject: [PATCH 19/32] refactor: usage commited claims --- app/migrations/068_usage_committed_claims.py | 57 ++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 app/migrations/068_usage_committed_claims.py diff --git a/app/migrations/068_usage_committed_claims.py b/app/migrations/068_usage_committed_claims.py new file mode 100644 index 00000000..25c59fb6 --- /dev/null +++ b/app/migrations/068_usage_committed_claims.py @@ -0,0 +1,57 @@ +"""Migration: usage_committed_claims for durable flush idempotency.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Create usage_committed_claims when missing (063 may have run before this table was added)" +) + + +def _table_exists(db: Session, table: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.tables + WHERE table_name = :table_name + """ + ), + {"table_name": table}, + ).first() + is not None + ) + + +def upgrade(db: Session): + if _table_exists(db, "usage_committed_claims"): + print("usage_committed_claims already exists, skipping 068") + db.commit() + return + + db.execute( + text( + """ + CREATE TABLE usage_committed_claims ( + claim_key TEXT PRIMARY KEY, + organization_id UUID NOT NULL, + committed_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX ix_usage_committed_claims_committed_at + ON usage_committed_claims (committed_at) + """ + ) + ) + print("Created usage_committed_claims") + db.commit() + + +def downgrade(db: Session): + db.execute(text("DROP TABLE IF EXISTS usage_committed_claims")) + db.commit() From cbe1c47f8c7fca5e7063bcc529eb2276ec860b73 Mon Sep 17 00:00:00 2001 From: M Sami Date: Fri, 14 Aug 2026 11:24:20 +0530 Subject: [PATCH 20/32] feat: usage pricing and integrate usage context across various routes --- README.md | 84 +- app/api/v1/api.py | 2 + app/api/v1/routes/agents.py | 147 +- app/api/v1/routes/call_import_evaluations.py | 10 + app/api/v1/routes/org_usage.py | 236 +- app/api/v1/routes/usage_pricing.py | 241 ++ app/api/v1/routes/voice_playground.py | 32 +- app/cli.py | 291 +- app/config/models.json | 3054 +++++++++++------ app/migrations/069_usage_pricing.py | 190 + app/migrations/070_usage_margin_multiplier.py | 45 + app/migrations/071_reseed_pricing_catalog.py | 37 + app/migrations/072_usage_pricing_phase1.py | 202 ++ .../073_usage_cost_recompute_jobs.py | 72 + ...pricing_rates_source_and_effective_from.py | 81 + app/models/database.py | 41 + app/services/ai/stt_clients/google.py | 25 + app/services/call_import_user_insights.py | 33 +- app/services/usage/llm_usage.py | 170 +- app/services/usage/pricing.py | 912 +++++ app/services/usage/pricing_cache.py | 87 + app/services/usage/pricing_jobs.py | 145 + app/services/usage/pricing_ops.py | 137 + app/services/usage/pricing_overrides.py | 530 +++ app/services/usage/usage_costs.py | 51 + app/workers/config.py | 10 +- app/workers/tasks/__init__.py | 2 + app/workers/tasks/recompute_usage_costs.py | 79 + docker-compose.observability.yml | 11 + docker-compose.yml | 57 + frontend/src/App.tsx | 2 + frontend/src/lib/api.ts | 169 + frontend/src/pages/usage/SearchableSelect.tsx | 7 +- frontend/src/pages/usage/Usage.tsx | 267 +- .../pages/usage/UsageCostBreakdownModal.tsx | 133 + frontend/src/pages/usage/UsageFiltersBar.tsx | 27 +- frontend/src/pages/usage/UsagePricing.tsx | 393 +++ frontend/src/pages/usage/usageProductHints.ts | 3 + scripts/merge_pricing_into_models_json.py | 107 + scripts/sync_pricing_catalog_from_litellm.py | 428 +++ .../test_usage/test_llm_usage.py | 32 + .../test_usage/test_org_usage_costs.py | 76 + .../test_services/test_usage/test_pricing.py | 211 ++ .../test_usage/test_pricing_cache.py | 91 + .../test_usage/test_pricing_ops.py | 10 + .../test_usage/test_pricing_overrides.py | 72 + .../test_usage/test_usage_costs.py | 33 + .../test_workers/test_usage_queue_routing.py | 18 + 48 files changed, 7777 insertions(+), 1316 deletions(-) create mode 100644 app/api/v1/routes/usage_pricing.py create mode 100644 app/migrations/069_usage_pricing.py create mode 100644 app/migrations/070_usage_margin_multiplier.py create mode 100644 app/migrations/071_reseed_pricing_catalog.py create mode 100644 app/migrations/072_usage_pricing_phase1.py create mode 100644 app/migrations/073_usage_cost_recompute_jobs.py create mode 100644 app/migrations/074_pricing_rates_source_and_effective_from.py create mode 100644 app/services/usage/pricing.py create mode 100644 app/services/usage/pricing_cache.py create mode 100644 app/services/usage/pricing_jobs.py create mode 100644 app/services/usage/pricing_ops.py create mode 100644 app/services/usage/pricing_overrides.py create mode 100644 app/services/usage/usage_costs.py create mode 100644 app/workers/tasks/recompute_usage_costs.py create mode 100644 frontend/src/pages/usage/UsageCostBreakdownModal.tsx create mode 100644 frontend/src/pages/usage/UsagePricing.tsx create mode 100644 scripts/merge_pricing_into_models_json.py create mode 100644 scripts/sync_pricing_catalog_from_litellm.py create mode 100644 tests/test_services/test_usage/test_org_usage_costs.py create mode 100644 tests/test_services/test_usage/test_pricing.py create mode 100644 tests/test_services/test_usage/test_pricing_cache.py create mode 100644 tests/test_services/test_usage/test_pricing_ops.py create mode 100644 tests/test_services/test_usage/test_pricing_overrides.py create mode 100644 tests/test_services/test_usage/test_usage_costs.py create mode 100644 tests/test_workers/test_usage_queue_routing.py diff --git a/README.md b/README.md index cc48559e..009ca348 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,21 @@ There are two ways to run the application: This will automatically: - Pull pre-built images from GitHub Container Registry (no build required!) - - Start all services (database, Redis, API, worker) + - Start all services: `db`, `redis`, `api`, `media`, `worker`, `worker-imports`, `worker-usage`, `beat` - Run database migrations automatically on startup + + | Service | Purpose | + |---------|---------| + | `db` | PostgreSQL | + | `redis` | Redis (Celery broker + usage counters) | + | `api` | HTTP API + frontend | + | `media` | Live voice WebSocket media server | + | `worker` | Celery: `celery`, `audio-metrics` queues | + | `worker-imports` | Celery: `imports`, `diarization`, `eval-control`, `evaluations` | + | `worker-usage` | Celery: `usage` queue (flush Redis counters + cost recompute) | + | `beat` | Celery Beat scheduler (periodic `flush_usage_counters`; requires `worker-usage`) | + + **Usage costs:** token/cost rollups stay stale without `worker-usage` and `beat` (or the equivalent processes from `eai start-all`). **Using a specific version:** ```bash @@ -137,20 +150,29 @@ docker compose up -d url: "redis://localhost:6379/0" ``` -4. **Start the application and worker** +4. **Start the application and workers** + + **Infra only (optional):** if Postgres/Redis run in Docker but the app runs locally: + ```bash + docker compose up -d db redis + ``` - **Option A: Start both together (Recommended)** + **Option A: Start everything together (Recommended)** ```bash eai start-all --config config.yml ``` - This single command will: - - Start the API server - - Start the Celery worker (for background task processing) - - Run database migrations automatically - - Build the frontend (if needed) + This single command spawns: + - API server (uvicorn) + - Telephony media server (`media` port, default 8001) + - Celery worker (`celery`, `audio-metrics`) + - Celery worker (`imports`, `diarization`, `eval-control`, `evaluations`) + - Celery worker (`usage` — flush + cost recompute) + - Celery Beat (periodic usage flush) + + It also runs database migrations and builds the frontend when needed. - Press `Ctrl+C` to stop both services together. + Press `Ctrl+C` to stop all processes. **Option B: Start separately (for advanced use)** @@ -247,7 +269,7 @@ make test-docker-db TEST_DB_HOST=localhost TEST_DB_PORT=5432 TEST_DB_NAME=effici ### Start Application and Worker Together (Recommended) ```bash -# Start both app and worker with default config.yml +# Start API + all workers + beat with default config.yml eai start-all # Start with custom config @@ -264,9 +286,17 @@ eai start-all --no-reload --no-build-frontend # Customize worker log level eai start-all --worker-loglevel debug + +# Skip dedicated workers (not recommended for production) +eai start-all --no-imports-worker +eai start-all --no-usage-worker +eai start-all --no-telephony-worker + +# Tune usage worker concurrency (default: 4, thread pool) +eai start-all --usage-worker-concurrency 8 ``` -**Note:** This is the recommended way to run EfficientAI. It starts both the API server and Celery worker in a single command. Press `Ctrl+C` to stop both services. +**Note:** This is the recommended local-dev workflow. One command spawns the API, telephony media server, three Celery workers (`celery,audio-metrics` · `imports,…` · `usage`), and Celery Beat. Press `Ctrl+C` to stop all processes. For Docker deployments, use `docker compose up -d` instead (separate containers per role; see Quick Start). ### Start Application Only ```bash @@ -340,7 +370,37 @@ eai worker --loglevel debug celery -A app.workers.celery_app worker --loglevel=info ``` -**Note:** The worker is required for processing background tasks (transcription, evaluation, etc.). If you use `eai start-all`, the worker starts automatically. Only use this command if you need to run the worker separately. +**Note:** Workers are required for background tasks (transcription, evaluation, usage cost flush, etc.). If you use `eai start-all`, they start automatically. Only use `eai worker` if you need to run a worker separately (e.g. `eai worker --queues usage` for the usage queue only). + +### Usage Pricing Ops +Manage model pricing rates and backfill stored usage costs on `llm_usage_daily` rollups. + +```bash +# Upsert model_pricing_rates from app/config/models.json +eai usage seed-rates --config config.yml + +# Compare models.json pricing vs Postgres +eai usage diff-rates --config config.yml + +# Backfill / recompute costs (sync, runs in this process) +eai usage recompute --config config.yml --sync + +# Enqueue Celery recompute on the usage queue (requires --organization-id) +eai usage recompute --config config.yml --organization-id + +# Optional: fetch LiteLLM prices into pricing_catalog.json +eai usage sync-litellm --local +eai usage sync-litellm --local --write-models +``` + +**After migrations or catalog changes:** +```bash +eai migrate +eai usage seed-rates --config config.yml +eai usage recompute --config config.yml --sync +``` + +Requires `worker-usage` (or `eai start-all`) for async recompute; Beat + `worker-usage` keep Redis usage counters flushed into Postgres on a schedule (`USAGE_FLUSH_BEAT_SECONDS`, default 120). ### Generate Config File ```bash diff --git a/app/api/v1/api.py b/app/api/v1/api.py index 51d81945..c1802238 100644 --- a/app/api/v1/api.py +++ b/app/api/v1/api.py @@ -46,6 +46,7 @@ dashboard, llm_gateway, org_usage, + usage_pricing, ) api_router = APIRouter() @@ -95,3 +96,4 @@ api_router.include_router(dashboard.router) api_router.include_router(llm_gateway.router) api_router.include_router(org_usage.router) +api_router.include_router(usage_pricing.router) diff --git a/app/api/v1/routes/agents.py b/app/api/v1/routes/agents.py index 4c4b295e..17094877 100644 --- a/app/api/v1/routes/agents.py +++ b/app/api/v1/routes/agents.py @@ -255,6 +255,7 @@ def _scenario_draft_responses(scenarios) -> list[GeneratedScenarioDraftResponse] async def generate_test_prompt( data: GenerateTestPromptRequest, organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), db: Session = Depends(get_db), ): @@ -262,6 +263,11 @@ async def generate_test_prompt( from app.services.testing.agent_test_setup_generation import ( generate_test_prompt_from_production, ) + from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + llm_usage_context, + ) if not data.production_prompt.strip(): raise HTTPException(400, "Production prompt is required") @@ -271,19 +277,26 @@ async def generate_test_prompt( ) try: - result = generate_test_prompt_from_production( - data.production_prompt, - agent_name=data.agent_name, - language=data.language, - call_type=data.call_type, - additional_context=data.additional_context, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - llm_config=data.llm_config, - credential_id=data.credential_id, - ) + with llm_usage_context( + LLMUsageContext( + organization_id=organization_id, + workspace_id=workspace_id, + product_section=LLMUsageProductSection.AGENTS, + ) + ): + result = generate_test_prompt_from_production( + data.production_prompt, + agent_name=data.agent_name, + language=data.language, + call_type=data.call_type, + additional_context=data.additional_context, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + llm_config=data.llm_config, + credential_id=data.credential_id, + ) return GenerateTestPromptResponse( sections=_test_prompt_section_responses(result.sections), test_agent_prompt=result.test_agent_prompt, @@ -301,6 +314,7 @@ async def generate_test_prompt( async def generate_scenarios_from_prompt( data: GenerateScenariosFromPromptRequest, organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), db: Session = Depends(get_db), ): @@ -308,6 +322,11 @@ async def generate_scenarios_from_prompt( from app.services.testing.agent_test_setup_generation import ( generate_scenarios_from_test_prompt, ) + from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + llm_usage_context, + ) if not data.test_agent_prompt.strip(): raise HTTPException(400, "Test agent prompt is required") @@ -317,20 +336,27 @@ async def generate_scenarios_from_prompt( ) try: - result = generate_scenarios_from_test_prompt( - data.test_agent_prompt, - agent_name=data.agent_name, - scenario_count=data.scenario_count, - language=data.language, - call_type=data.call_type, - additional_context=data.additional_context, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - llm_config=data.llm_config, - credential_id=data.credential_id, - ) + with llm_usage_context( + LLMUsageContext( + organization_id=organization_id, + workspace_id=workspace_id, + product_section=LLMUsageProductSection.AGENTS, + ) + ): + result = generate_scenarios_from_test_prompt( + data.test_agent_prompt, + agent_name=data.agent_name, + scenario_count=data.scenario_count, + language=data.language, + call_type=data.call_type, + additional_context=data.additional_context, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + llm_config=data.llm_config, + credential_id=data.credential_id, + ) return GenerateScenariosFromPromptResponse( scenarios=_scenario_draft_responses(result.scenarios), provider=result.provider, @@ -347,6 +373,7 @@ async def generate_scenarios_from_prompt( async def generate_test_setup( data: GenerateTestSetupRequest, organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), db: Session = Depends(get_db), ): @@ -355,6 +382,11 @@ async def generate_test_setup( generate_scenarios_from_test_prompt, generate_test_prompt_from_production, ) + from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + llm_usage_context, + ) if not data.production_prompt.strip(): raise HTTPException(400, "Production prompt is required") @@ -364,33 +396,40 @@ async def generate_test_setup( ) try: - prompt_result = generate_test_prompt_from_production( - data.production_prompt, - agent_name=data.agent_name, - language=data.language, - call_type=data.call_type, - additional_context=data.additional_context, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - llm_config=data.llm_config, - credential_id=data.credential_id, - ) - scenario_result = generate_scenarios_from_test_prompt( - prompt_result.test_agent_prompt, - agent_name=data.agent_name, - scenario_count=data.scenario_count, - language=data.language, - call_type=data.call_type, - additional_context=data.additional_context, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - llm_config=data.llm_config, - credential_id=data.credential_id, - ) + with llm_usage_context( + LLMUsageContext( + organization_id=organization_id, + workspace_id=workspace_id, + product_section=LLMUsageProductSection.AGENTS, + ) + ): + prompt_result = generate_test_prompt_from_production( + data.production_prompt, + agent_name=data.agent_name, + language=data.language, + call_type=data.call_type, + additional_context=data.additional_context, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + llm_config=data.llm_config, + credential_id=data.credential_id, + ) + scenario_result = generate_scenarios_from_test_prompt( + prompt_result.test_agent_prompt, + agent_name=data.agent_name, + scenario_count=data.scenario_count, + language=data.language, + call_type=data.call_type, + additional_context=data.additional_context, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + llm_config=data.llm_config, + credential_id=data.credential_id, + ) return GenerateTestSetupResponse( sections=_test_prompt_section_responses(prompt_result.sections), test_agent_prompt=prompt_result.test_agent_prompt, diff --git a/app/api/v1/routes/call_import_evaluations.py b/app/api/v1/routes/call_import_evaluations.py index 880393c7..528f60f3 100644 --- a/app/api/v1/routes/call_import_evaluations.py +++ b/app/api/v1/routes/call_import_evaluations.py @@ -3028,10 +3028,19 @@ def _explain_period_deltas( from app.services.ai.llm_resolver import get_llm_provider_and_model from app.services.call_import_user_insights import _call_llm, _parse_json_object + from app.services.usage.call_import_context import ( + call_import_evaluation_usage_context, + ) provider_enum, model_str = get_llm_provider_and_model( organization_id, db, provider_hint, model_hint ) + usage_ctx = call_import_evaluation_usage_context( + organization_id=organization_id, + workspace_id=evaluation.workspace_id, + evaluation_id=evaluation.id, + call_import_id=evaluation.call_import_id, + ) try: text = _call_llm( db, @@ -3051,6 +3060,7 @@ def _explain_period_deltas( ], temperature=0.3, max_tokens=900, + usage_ctx=usage_ctx, ) except Exception as exc: logger.warning("[PeriodDeltaExplain] LLM call failed: {}", exc) diff --git a/app/api/v1/routes/org_usage.py b/app/api/v1/routes/org_usage.py index 4c702138..2d6854f6 100644 --- a/app/api/v1/routes/org_usage.py +++ b/app/api/v1/routes/org_usage.py @@ -27,6 +27,7 @@ Workspace, ) from app.services.usage.llm_usage import flush_usage_to_catalog +from app.services.usage.usage_costs import costs_from_micro from app.services.usage.usage_labels import ( labels_for_call_import_ids, labels_for_resource_buckets, @@ -87,6 +88,19 @@ def _label_row_order(): ) +class UsageCosts(BaseModel): + input_cost_usd: float = 0 + output_cost_usd: float = 0 + cache_read_cost_usd: float = 0 + cache_write_cost_usd: float = 0 + reasoning_cost_usd: float = 0 + audio_cost_usd: float = 0 + tts_cost_usd: float = 0 + total_cost_usd: float = 0 + currency: str = "USD" + has_unpriced_usage: bool = False + + class UsageTotals(BaseModel): prompt_tokens: int = 0 completion_tokens: int = 0 @@ -97,6 +111,15 @@ class UsageTotals(BaseModel): audio_seconds: int = 0 tts_characters: int = 0 call_count: int = 0 + input_cost_micro_usd: int = 0 + output_cost_micro_usd: int = 0 + cache_read_cost_micro_usd: int = 0 + cache_creation_cost_micro_usd: int = 0 + reasoning_cost_micro_usd: int = 0 + audio_cost_micro_usd: int = 0 + tts_cost_micro_usd: int = 0 + total_cost_micro_usd: int = 0 + costs: UsageCosts = Field(default_factory=UsageCosts) class UsageSummaryResponse(BaseModel): @@ -127,6 +150,15 @@ class UsageBreakdownRow(BaseModel): audio_seconds: int = 0 tts_characters: int = 0 call_count: int = 0 + input_cost_micro_usd: int = 0 + output_cost_micro_usd: int = 0 + cache_read_cost_micro_usd: int = 0 + cache_creation_cost_micro_usd: int = 0 + reasoning_cost_micro_usd: int = 0 + audio_cost_micro_usd: int = 0 + tts_cost_micro_usd: int = 0 + total_cost_micro_usd: int = 0 + costs: UsageCosts = Field(default_factory=UsageCosts) class UsageBreakdownResponse(BaseModel): @@ -438,7 +470,7 @@ def _apply_filters( query = query.filter( LLMUsageDaily.context["call_import_id"].astext == str(call_import_id) ) - if evaluation_id is not None: + if evaluation_id is not None and evaluation_id != resource_id: if db is not None: query = query.filter( _evaluation_scope_filter(evaluation_id, organization_id, db) @@ -847,6 +879,123 @@ def _evaluation_label_map( return labels +def _unpriced_usage_condition(): + return and_( + LLMUsageDaily.pricing_rate_id.is_(None), + _usage_row_weight() > 0, + ) + + +def _has_unpriced_usage_column(): + return func.coalesce(func.bool_or(_unpriced_usage_condition()), False).label( + "has_unpriced_usage" + ) + + +def _usage_cost_sum_columns(): + return ( + func.coalesce(func.sum(LLMUsageDaily.input_cost_micro_usd), 0).label( + "input_cost_micro_usd" + ), + func.coalesce(func.sum(LLMUsageDaily.output_cost_micro_usd), 0).label( + "output_cost_micro_usd" + ), + func.coalesce(func.sum(LLMUsageDaily.cache_read_cost_micro_usd), 0).label( + "cache_read_cost_micro_usd" + ), + func.coalesce(func.sum(LLMUsageDaily.cache_creation_cost_micro_usd), 0).label( + "cache_creation_cost_micro_usd" + ), + func.coalesce(func.sum(LLMUsageDaily.reasoning_cost_micro_usd), 0).label( + "reasoning_cost_micro_usd" + ), + func.coalesce(func.sum(LLMUsageDaily.audio_cost_micro_usd), 0).label( + "audio_cost_micro_usd" + ), + func.coalesce(func.sum(LLMUsageDaily.tts_cost_micro_usd), 0).label( + "tts_cost_micro_usd" + ), + func.coalesce(func.sum(LLMUsageDaily.total_cost_micro_usd), 0).label( + "total_cost_micro_usd" + ), + ) + + +def _attach_costs(metrics: dict, *, has_unpriced_usage: bool = False) -> dict: + costs = costs_from_micro( + input_cost_micro_usd=metrics.get("input_cost_micro_usd", 0), + output_cost_micro_usd=metrics.get("output_cost_micro_usd", 0), + cache_read_cost_micro_usd=metrics.get("cache_read_cost_micro_usd", 0), + cache_creation_cost_micro_usd=metrics.get("cache_creation_cost_micro_usd", 0), + reasoning_cost_micro_usd=metrics.get("reasoning_cost_micro_usd", 0), + audio_cost_micro_usd=metrics.get("audio_cost_micro_usd", 0), + tts_cost_micro_usd=metrics.get("tts_cost_micro_usd", 0), + total_cost_micro_usd=metrics.get("total_cost_micro_usd", 0), + has_unpriced_usage=has_unpriced_usage, + ) + return {**metrics, "costs": costs} + + +def _usage_totals_from_row(row) -> dict: + prompt = int(row.prompt_tokens) + completion = int(row.completion_tokens) + metrics = { + "prompt_tokens": prompt, + "completion_tokens": completion, + "total_tokens": prompt + completion, + "cache_read_tokens": int(row.cache_read_tokens), + "cache_creation_tokens": int(row.cache_creation_tokens), + "reasoning_tokens": int(row.reasoning_tokens), + "audio_seconds": int(row.audio_seconds), + "tts_characters": int(row.tts_characters), + "call_count": int(row.call_count), + "input_cost_micro_usd": int(getattr(row, "input_cost_micro_usd", 0) or 0), + "output_cost_micro_usd": int(getattr(row, "output_cost_micro_usd", 0) or 0), + "cache_read_cost_micro_usd": int( + getattr(row, "cache_read_cost_micro_usd", 0) or 0 + ), + "cache_creation_cost_micro_usd": int( + getattr(row, "cache_creation_cost_micro_usd", 0) or 0 + ), + "reasoning_cost_micro_usd": int( + getattr(row, "reasoning_cost_micro_usd", 0) or 0 + ), + "audio_cost_micro_usd": int(getattr(row, "audio_cost_micro_usd", 0) or 0), + "tts_cost_micro_usd": int(getattr(row, "tts_cost_micro_usd", 0) or 0), + "total_cost_micro_usd": int(getattr(row, "total_cost_micro_usd", 0) or 0), + } + return _attach_costs( + metrics, + has_unpriced_usage=bool(getattr(row, "has_unpriced_usage", False)), + ) + + +def _breakdown_metrics_from_tuple(metrics: tuple) -> dict: + prompt = int(metrics[0]) + completion = int(metrics[1]) + base = { + "prompt_tokens": prompt, + "completion_tokens": completion, + "total_tokens": prompt + completion, + "cache_read_tokens": int(metrics[2]), + "cache_creation_tokens": int(metrics[3]), + "reasoning_tokens": int(metrics[4]), + "audio_seconds": int(metrics[5]), + "tts_characters": int(metrics[6]), + "call_count": int(metrics[7]), + "input_cost_micro_usd": int(metrics[8]), + "output_cost_micro_usd": int(metrics[9]), + "cache_read_cost_micro_usd": int(metrics[10]), + "cache_creation_cost_micro_usd": int(metrics[11]), + "reasoning_cost_micro_usd": int(metrics[12]), + "audio_cost_micro_usd": int(metrics[13]), + "tts_cost_micro_usd": int(metrics[14]), + "total_cost_micro_usd": int(metrics[15]), + } + has_unpriced = bool(metrics[16]) if len(metrics) > 16 else False + return _attach_costs(base, has_unpriced_usage=has_unpriced) + + def _last_updated(db: Session, organization_id: UUID) -> Optional[datetime]: return ( db.query(func.max(LLMUsageDaily.updated_at)) @@ -890,6 +1039,8 @@ def _summary_aggregate_query( func.coalesce(func.sum(LLMUsageDaily.audio_seconds), 0).label("audio_seconds"), func.coalesce(func.sum(LLMUsageDaily.tts_characters), 0).label("tts_characters"), func.coalesce(func.sum(LLMUsageDaily.call_count), 0).label("call_count"), + *_usage_cost_sum_columns(), + _has_unpriced_usage_column(), ), organization_id=organization_id, start=start, @@ -953,19 +1104,7 @@ def get_usage_summary( dataset=dataset, tag_id=tag_id, ).one() - prompt = int(row.prompt_tokens) - completion = int(row.completion_tokens) - totals = { - "prompt_tokens": prompt, - "completion_tokens": completion, - "total_tokens": prompt + completion, - "cache_read_tokens": int(row.cache_read_tokens), - "cache_creation_tokens": int(row.cache_creation_tokens), - "reasoning_tokens": int(row.reasoning_tokens), - "audio_seconds": int(row.audio_seconds), - "tts_characters": int(row.tts_characters), - "call_count": int(row.call_count), - } + totals = _usage_totals_from_row(row) return UsageSummaryResponse( start=display_start, end=display_end, @@ -1032,6 +1171,8 @@ def get_usage_breakdown( func.coalesce(func.sum(LLMUsageDaily.audio_seconds), 0).label("audio_seconds"), func.coalesce(func.sum(LLMUsageDaily.tts_characters), 0).label("tts_characters"), func.coalesce(func.sum(LLMUsageDaily.call_count), 0).label("call_count"), + *_usage_cost_sum_columns(), + _has_unpriced_usage_column(), ] select_cols = [dim] @@ -1124,15 +1265,7 @@ def get_usage_breakdown( UsageBreakdownRow( workspace_id=ws_id, workspace_name=workspace_names.get(ws_id) if ws_id else "Unknown", - prompt_tokens=int(metrics[0]), - completion_tokens=int(metrics[1]), - total_tokens=int(metrics[0]) + int(metrics[1]), - cache_read_tokens=int(metrics[2]), - cache_creation_tokens=int(metrics[3]), - reasoning_tokens=int(metrics[4]), - audio_seconds=int(metrics[5]), - tts_characters=int(metrics[6]), - call_count=int(metrics[7]), + **_breakdown_metrics_from_tuple(metrics), ) ) elif group_by == "product_section": @@ -1142,15 +1275,7 @@ def get_usage_breakdown( UsageBreakdownRow( product_section=section, product_section_label=SECTION_LABELS.get(section or "", section), - prompt_tokens=int(metrics[0]), - completion_tokens=int(metrics[1]), - total_tokens=int(metrics[0]) + int(metrics[1]), - cache_read_tokens=int(metrics[2]), - cache_creation_tokens=int(metrics[3]), - reasoning_tokens=int(metrics[4]), - audio_seconds=int(metrics[5]), - tts_characters=int(metrics[6]), - call_count=int(metrics[7]), + **_breakdown_metrics_from_tuple(metrics), ) ) elif group_by == "model": @@ -1159,15 +1284,7 @@ def get_usage_breakdown( rows.append( UsageBreakdownRow( model=model_name, - prompt_tokens=int(metrics[0]), - completion_tokens=int(metrics[1]), - total_tokens=int(metrics[0]) + int(metrics[1]), - cache_read_tokens=int(metrics[2]), - cache_creation_tokens=int(metrics[3]), - reasoning_tokens=int(metrics[4]), - audio_seconds=int(metrics[5]), - tts_characters=int(metrics[6]), - call_count=int(metrics[7]), + **_breakdown_metrics_from_tuple(metrics), ) ) elif group_by == "usage_kind": @@ -1176,15 +1293,7 @@ def get_usage_breakdown( rows.append( UsageBreakdownRow( usage_kind=kind, - prompt_tokens=int(metrics[0]), - completion_tokens=int(metrics[1]), - total_tokens=int(metrics[0]) + int(metrics[1]), - cache_read_tokens=int(metrics[2]), - cache_creation_tokens=int(metrics[3]), - reasoning_tokens=int(metrics[4]), - audio_seconds=int(metrics[5]), - tts_characters=int(metrics[6]), - call_count=int(metrics[7]), + **_breakdown_metrics_from_tuple(metrics), ) ) elif group_by == "call_import": @@ -1205,15 +1314,7 @@ def get_usage_breakdown( UsageBreakdownRow( call_import_id=cid, call_import_label=label, - prompt_tokens=int(metrics[0]), - completion_tokens=int(metrics[1]), - total_tokens=int(metrics[0]) + int(metrics[1]), - cache_read_tokens=int(metrics[2]), - cache_creation_tokens=int(metrics[3]), - reasoning_tokens=int(metrics[4]), - audio_seconds=int(metrics[5]), - tts_characters=int(metrics[6]), - call_count=int(metrics[7]), + **_breakdown_metrics_from_tuple(metrics), ) ) else: @@ -1241,15 +1342,7 @@ def get_usage_breakdown( ), product_section=section, product_section_label=SECTION_LABELS.get(section or "", section), - prompt_tokens=int(metrics[0]), - completion_tokens=int(metrics[1]), - total_tokens=int(metrics[0]) + int(metrics[1]), - cache_read_tokens=int(metrics[2]), - cache_creation_tokens=int(metrics[3]), - reasoning_tokens=int(metrics[4]), - audio_seconds=int(metrics[5]), - tts_characters=int(metrics[6]), - call_count=int(metrics[7]), + **_breakdown_metrics_from_tuple(metrics), ) ) @@ -1278,6 +1371,7 @@ def get_usage_filters( resource_id: Optional[UUID] = Query(None), usage_kind: Optional[str] = Query(None), call_import_id: Optional[UUID] = Query(None), + evaluation_id: Optional[UUID] = Query(None), dataset: Optional[str] = Query(None), tag_id: Optional[UUID] = Query(None), q: Optional[str] = Query(None, description="Optional resource label search"), @@ -1288,6 +1382,8 @@ def get_usage_filters( flush_usage_to_catalog(db, organization_id) + scoped_resource_id = resource_id or evaluation_id + workspace_base = _filtered_query( db, organization_id=organization_id, @@ -1312,7 +1408,9 @@ def get_usage_filters( end=filter_end, workspace_id=workspace_id, product_section=product_section, + resource_id=scoped_resource_id, call_import_id=call_import_id, + evaluation_id=evaluation_id, dataset=dataset, tag_id=tag_id, ) @@ -1323,8 +1421,10 @@ def get_usage_filters( end=filter_end, workspace_id=workspace_id, product_section=product_section, + resource_id=scoped_resource_id, usage_kind=usage_kind, call_import_id=call_import_id, + evaluation_id=evaluation_id, dataset=dataset, tag_id=tag_id, ) diff --git a/app/api/v1/routes/usage_pricing.py b/app/api/v1/routes/usage_pricing.py new file mode 100644 index 00000000..0840a29a --- /dev/null +++ b/app/api/v1/routes/usage_pricing.py @@ -0,0 +1,241 @@ +"""Org-scoped usage pricing overrides and recompute API.""" + +from __future__ import annotations + +from datetime import date, datetime +from typing import List, Optional +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query, status +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from app.core.auth.rbac import require_admin +from app.database import get_db +from app.dependencies import get_organization_id +from app.services.usage.pricing_jobs import ( + create_recompute_job, + enqueue_recompute_job, + get_recompute_job, + job_to_dict, +) +from app.services.usage.pricing_overrides import ( + delete_override, + get_effective_rate, + list_effective_pricing, + list_overrides, + upsert_override, +) + +router = APIRouter( + prefix="/organizations/usage/pricing", + tags=["Usage"], + dependencies=[Depends(require_admin)], +) + + +class PricingRatesUsd(BaseModel): + input_per_1m: Optional[float] = None + output_per_1m: Optional[float] = None + cache_read_per_1m: Optional[float] = None + cache_write_per_1m: Optional[float] = None + reasoning_per_1m: Optional[float] = None + audio_per_minute: Optional[float] = None + tts_per_1m_characters: Optional[float] = None + + +class PricingOverrideResponse(BaseModel): + id: str + organization_id: str + model: str + usage_kind: str + effective_from: date + effective_to: Optional[date] = None + rates: PricingRatesUsd + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None + recompute_enqueued: Optional[bool] = None + recompute_job_id: Optional[str] = None + + +class PricingOverrideUpsertRequest(BaseModel): + usage_kind: str = "llm" + effective_from: date + effective_to: Optional[date] = None + rates: PricingRatesUsd + recompute: bool = True + + +class EffectivePricingResponse(BaseModel): + model: str + usage_kind: str + as_of: date + catalog_rates: Optional[PricingRatesUsd] = None + catalog_rate_id: Optional[str] = None + override: Optional[PricingOverrideResponse] = None + effective_rates: Optional[PricingRatesUsd] = None + effective_source: Optional[str] = None + effective_rate_id: Optional[str] = None + has_override: bool = False + + +class PricingOverrideDeleteResponse(BaseModel): + deleted: bool + model: str + usage_kind: str + recompute_enqueued: bool = False + recompute_job_id: Optional[str] = None + + +class UsageRecomputeRequest(BaseModel): + start_date: Optional[date] = None + end_date: Optional[date] = None + model: Optional[str] = None + usage_kind: Optional[str] = None + + +class UsageRecomputeJobResponse(BaseModel): + id: UUID + organization_id: UUID + status: str + model: Optional[str] = None + usage_kind: Optional[str] = None + start_date: Optional[date] = None + end_date: Optional[date] = None + updated_rows: int = 0 + error_message: Optional[str] = None + celery_task_id: Optional[str] = None + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + + +@router.get("", response_model=List[EffectivePricingResponse]) +def list_effective_usage_pricing( + usage_kind: Optional[str] = Query(None), + model: Optional[str] = Query(None), + as_of: Optional[date] = Query(None), + limit: int = Query(200, ge=1, le=1000), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +): + day = as_of or date.today() + rows = list_effective_pricing( + db, + organization_id=organization_id, + usage_kind=usage_kind, + model=model, + as_of=day, + limit=limit, + ) + return rows + + +@router.get("/overrides", response_model=List[PricingOverrideResponse]) +def list_usage_pricing_overrides( + model: Optional[str] = Query(None), + usage_kind: Optional[str] = Query(None), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +): + return list_overrides( + db, + organization_id=organization_id, + model=model, + usage_kind=usage_kind, + ) + + +@router.get("/overrides/{model}", response_model=EffectivePricingResponse) +def get_usage_pricing_override_effective( + model: str, + usage_kind: str = Query("llm"), + as_of: Optional[date] = Query(None), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +): + return get_effective_rate( + db, + organization_id=organization_id, + model=model, + usage_kind=usage_kind, + as_of=as_of or date.today(), + ) + + +@router.put("/overrides/{model}", response_model=PricingOverrideResponse) +def upsert_usage_pricing_override( + model: str, + body: PricingOverrideUpsertRequest, + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +): + return upsert_override( + db, + organization_id=organization_id, + model=model, + usage_kind=body.usage_kind, + effective_from=body.effective_from, + effective_to=body.effective_to, + rates=body.rates.model_dump(exclude_unset=True), + recompute=body.recompute, + ) + + +@router.delete("/overrides/{model}", response_model=PricingOverrideDeleteResponse) +def delete_usage_pricing_override( + model: str, + usage_kind: str = Query("llm"), + effective_from: Optional[date] = Query(None), + recompute: bool = Query(True), + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +): + return delete_override( + db, + organization_id=organization_id, + model=model, + usage_kind=usage_kind, + effective_from=effective_from, + recompute=recompute, + ) + + +@router.post( + "/recompute", + response_model=UsageRecomputeJobResponse, + status_code=status.HTTP_202_ACCEPTED, +) +def trigger_usage_cost_recompute( + body: UsageRecomputeRequest, + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +): + if ( + body.start_date is not None + and body.end_date is not None + and body.end_date < body.start_date + ): + raise HTTPException(status_code=400, detail="end_date must be >= start_date") + + job = create_recompute_job( + db, + organization_id=organization_id, + model=body.model, + usage_kind=body.usage_kind, + start_date=body.start_date, + end_date=body.end_date, + ) + enqueue_recompute_job(db, job) + db.refresh(job) + return UsageRecomputeJobResponse(**job_to_dict(job)) + + +@router.get("/recompute/{job_id}", response_model=UsageRecomputeJobResponse) +def get_usage_cost_recompute_job( + job_id: UUID, + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +): + job = get_recompute_job(db, organization_id=organization_id, job_id=job_id) + return UsageRecomputeJobResponse(**job_to_dict(job)) diff --git a/app/api/v1/routes/voice_playground.py b/app/api/v1/routes/voice_playground.py index 8cc4519d..40477c25 100644 --- a/app/api/v1/routes/voice_playground.py +++ b/app/api/v1/routes/voice_playground.py @@ -398,6 +398,7 @@ def _serialize_custom_voice(voice: CustomTTSVoice) -> Dict[str, Any]: async def generate_sample_texts( data: GenerateSamplesRequest, organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), db: Session = Depends(get_db), ): @@ -460,16 +461,29 @@ async def generate_sample_texts( {"role": "user", "content": user_prompt}, ] + from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + llm_usage_context, + ) + try: - result = llm_service.generate_response( - messages=messages, - llm_provider=provider_enum, - llm_model=llm_model_str, - organization_id=organization_id, - db=db, - llm_config=request_llm_config, - task_defaults={"temperature": 0.8, "max_tokens": max_tokens}, - ) + with llm_usage_context( + LLMUsageContext( + organization_id=organization_id, + workspace_id=workspace_id, + product_section=LLMUsageProductSection.VOICE_PLAYGROUND, + ) + ): + result = llm_service.generate_response( + messages=messages, + llm_provider=provider_enum, + llm_model=llm_model_str, + organization_id=organization_id, + db=db, + llm_config=request_llm_config, + task_defaults={"temperature": 0.8, "max_tokens": max_tokens}, + ) except Exception as e: logger.error(f"[VoicePlayground] LLM generation failed: {e}") raise HTTPException(500, f"LLM generation failed: {str(e)}") diff --git a/app/cli.py b/app/cli.py index 453b2a2d..eed0274a 100644 --- a/app/cli.py +++ b/app/cli.py @@ -623,6 +623,20 @@ def _handle_signal(sig, frame): "is enabled; 32 threads can exhaust per-shard SQLAlchemy pools." ), ) +@click.option( + "--usage-worker/--no-usage-worker", + default=True, + help=( + "Also start a dedicated worker for the `usage` queue (flush + cost recompute; " + "default: True)." + ), +) +@click.option( + "--usage-worker-concurrency", + default=4, + type=int, + help="Concurrency for the usage worker (default: 4; thread pool).", +) @click.option( "--telephony-worker/--no-telephony-worker", default=True, @@ -646,6 +660,8 @@ def start_all( worker_loglevel: str, imports_worker: bool, imports_worker_concurrency: int, + usage_worker: bool, + usage_worker_concurrency: int, telephony_worker: bool, media_port: Optional[int], ): @@ -692,6 +708,7 @@ def start_all( # Store worker processes for cleanup. worker_process = None worker_imports_process = None + worker_usage_process = None beat_process = None telephony_process = None @@ -712,11 +729,12 @@ def _terminate(proc, label: str): def cleanup_processes(): """Clean up spawned processes.""" - nonlocal worker_process, worker_imports_process, beat_process, telephony_process + nonlocal worker_process, worker_imports_process, worker_usage_process, beat_process, telephony_process _terminate(telephony_process, "Telephony media server") _terminate(beat_process, "Celery beat") _terminate(worker_process, "Celery worker (default)") _terminate(worker_imports_process, "Celery worker (imports)") + _terminate(worker_usage_process, "Celery worker (usage)") # Register cleanup on exit atexit.register(cleanup_processes) @@ -870,6 +888,30 @@ def _stream_telephony(): prefix="[WORKER-IMPORTS]", ) + if usage_worker: + from app.workers.config import USAGE_WORKER_QUEUE + + worker_usage_process = _spawn_worker( + [ + "celery", + "-A", + "app.workers.celery_app", + "worker", + f"--loglevel={worker_loglevel}", + "-Q", + USAGE_WORKER_QUEUE, + "-P", + "threads", + "-c", + str(usage_worker_concurrency), + ], + label=( + f"Celery worker ({USAGE_WORKER_QUEUE} queue, " + f"pool=threads, concurrency={usage_worker_concurrency})" + ), + prefix="[WORKER-USAGE]", + ) + beat_process = _spawn_worker( [ "celery", @@ -920,6 +962,15 @@ def _stream_telephony(): ) else: click.echo(" Workers: default queue only (--no-imports-worker)") + if usage_worker: + from app.workers.config import USAGE_WORKER_QUEUE + + click.echo( + f" Usage worker: {USAGE_WORKER_QUEUE} queue " + f"(concurrency={usage_worker_concurrency})" + ) + else: + click.echo(" Usage worker: disabled (--no-usage-worker)") if telephony_worker: telephony_public = (settings.VOBIZ_WEBHOOK_BASE_URL or "").strip() click.echo(f" Telephony edge: http://localhost:{bind_media_port} (local)") @@ -1198,6 +1249,244 @@ def sharding_rebalance_slices( catalog.close() +@click.group() +def usage(): + """Usage pricing ops (seed rates, diff catalog, recompute costs).""" + pass + + +main.add_command(usage) + + +def _load_cli_config(config: str) -> Path: + config_path = Path(config) + if not config_path.exists(): + click.echo(f"❌ Config file not found: {config}", err=True) + sys.exit(1) + from app.config import load_config_from_file + + load_config_from_file(str(config_path)) + return config_path + + +@usage.command("seed-rates") +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option( + "--effective-from", + type=click.DateTime(formats=["%Y-%m-%d"]), + default=None, + help="Effective date for seeded rates (default: 2020-01-01)", +) +def usage_seed_rates(config: str, effective_from): + """Upsert model_pricing_rates from models.json pricing blocks.""" + _load_cli_config(config) + from app.database import SessionLocal + from app.services.usage.pricing_ops import seed_rates_from_models_json + + day = effective_from.date() if effective_from else None + db = SessionLocal() + try: + count = seed_rates_from_models_json(db, effective_from=day) + db.commit() + click.echo(f"✅ Seeded/updated {count} pricing rate row(s)") + finally: + db.close() + + +@usage.command("diff-rates") +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option( + "--effective-from", + type=click.DateTime(formats=["%Y-%m-%d"]), + default=None, + help="Compare rates at this effective_from date (default: 2020-01-01)", +) +@click.option("--json", "as_json", is_flag=True, help="Print machine-readable JSON") +def usage_diff_rates(config: str, effective_from, as_json: bool): + """Diff models.json pricing blocks vs model_pricing_rates in Postgres.""" + import json as json_module + + _load_cli_config(config) + from app.database import SessionLocal + from app.services.usage.pricing_ops import diff_models_json_vs_db + + day = effective_from.date() if effective_from else None + db = SessionLocal() + try: + report = diff_models_json_vs_db(db, effective_from=day) + finally: + db.close() + + if as_json: + click.echo(json_module.dumps(report, indent=2, default=str)) + return + + click.echo(f"effective_from: {report['effective_from']}") + click.echo( + f"models.json priced: {report['models_json_count']} | " + f"database rows: {report['database_count']} | " + f"in_sync: {report['in_sync']}" + ) + if report["only_in_models_json"]: + click.echo(f"\nOnly in models.json ({len(report['only_in_models_json'])}):") + for item in report["only_in_models_json"][:20]: + click.echo(f" - {item['model']} ({item['usage_kind']})") + if report["only_in_database"]: + click.echo(f"\nOnly in database ({len(report['only_in_database'])}):") + for item in report["only_in_database"][:20]: + click.echo(f" - {item['model']} ({item['usage_kind']})") + if report["mismatches"]: + click.echo(f"\nMismatched rates ({len(report['mismatches'])}):") + for item in report["mismatches"][:20]: + click.echo(f" - {item['model']} ({item['usage_kind']})") + for field, values in item["fields"].items(): + click.echo( + f" {field}: json={values['models_json']} db={values['database']}" + ) + missing = report["missing_pricing_blocks"] + if missing: + click.echo(f"\nmodels.json entries missing pricing blocks ({len(missing)}):") + for model in missing[:20]: + click.echo(f" - {model}") + unresolved = report["litellm_unresolved"] + if unresolved: + click.echo(f"\nLiteLLM unresolved ({len(unresolved)}):") + for item in unresolved[:20]: + click.echo(f" - {item.get('model')} ({item.get('reason', 'unresolved')})") + + +@usage.command("recompute") +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option("--organization-id", default=None, help="Scope recompute to one org UUID") +@click.option("--model", default=None, help="Scope recompute to one model") +@click.option("--usage-kind", default=None, help="Scope recompute to llm/stt/tts") +@click.option("--start-date", type=click.DateTime(formats=["%Y-%m-%d"]), default=None) +@click.option("--end-date", type=click.DateTime(formats=["%Y-%m-%d"]), default=None) +@click.option( + "--async/--sync", + "run_async", + default=True, + help="Enqueue Celery task (default) or run synchronously in this process", +) +def usage_recompute( + config: str, + organization_id: Optional[str], + model: Optional[str], + usage_kind: Optional[str], + start_date, + end_date, + run_async: bool, +): + """Backfill or recompute stored usage costs on llm_usage_daily rollups.""" + from uuid import UUID + + _load_cli_config(config) + start = start_date.date() if start_date else None + end = end_date.date() if end_date else None + org_uuid = UUID(organization_id) if organization_id else None + + if run_async: + if org_uuid is None: + click.echo( + "❌ --organization-id is required for async recompute (creates a tracked job).", + err=True, + ) + click.echo( + "💡 Use --sync to recompute in this process without an org scope, or pass --organization-id.", + err=True, + ) + sys.exit(1) + + from app.database import SessionLocal + from app.services.usage.pricing_jobs import ( + create_recompute_job, + enqueue_recompute_job, + job_to_dict, + ) + + db = SessionLocal() + try: + job = create_recompute_job( + db, + organization_id=org_uuid, + model=model, + usage_kind=usage_kind, + start_date=start, + end_date=end, + ) + enqueue_recompute_job(db, job) + db.refresh(job) + payload = job_to_dict(job) + click.echo(f"✅ Enqueued recompute job {payload['id']} (status={payload['status']})") + if payload.get("celery_task_id"): + click.echo(f" Celery task: {payload['celery_task_id']}") + except Exception as exc: + click.echo(f"❌ Failed to enqueue recompute job: {exc}", err=True) + sys.exit(1) + finally: + db.close() + return + + from app.database import SessionLocal + from app.services.usage.pricing import recompute_usage_costs + + db = SessionLocal() + try: + updated = recompute_usage_costs( + db, + organization_id=org_uuid, + model=model, + usage_kind=usage_kind, + start_date=start, + end_date=end, + ) + click.echo(f"✅ Recomputed costs for {updated} rollup row(s)") + finally: + db.close() + + +@usage.command("sync-litellm") +@click.option("--local", is_flag=True, help="Use bundled LiteLLM model_cost JSON") +@click.option( + "--write-models", + is_flag=True, + help="Merge generated pricing into app/config/models.json", +) +@click.option("--stdout", is_flag=True, help="Print pricing_catalog.json to stdout") +def usage_sync_litellm(local: bool, write_models: bool, stdout: bool): + """Fetch LiteLLM prices and regenerate pricing_catalog.json.""" + import subprocess + import sys as sys_module + + script = Path(__file__).resolve().parent.parent / "scripts" / "sync_pricing_catalog_from_litellm.py" + cmd = [sys_module.executable, str(script)] + if local: + cmd.append("--local") + if write_models: + cmd.append("--write-models") + if stdout: + cmd.append("--stdout") + subprocess.run(cmd, check=True) + + if __name__ == "__main__": main() diff --git a/app/config/models.json b/app/config/models.json index d727a14d..38f24cca 100644 --- a/app/config/models.json +++ b/app/config/models.json @@ -1,1029 +1,2025 @@ -{ - "whisper-1": { - "provider": "openai", - "model_type": "stt", - "description": "General-purpose speech recognition (verbose_json + word/segment timestamps)" - }, - "gpt-4o-transcribe": { - "provider": "openai", - "model_type": "stt", - "description": "GPT-4o speech-to-text; higher accuracy, no granular word timestamps" - }, - "gpt-4o-mini-transcribe": { - "provider": "openai", - "model_type": "stt", - "description": "Cheaper / faster GPT-4o transcription; no granular word timestamps" - }, - "gpt-4o-transcribe-diarize": { - "provider": "openai", - "model_type": "stt", - "description": "Transcription model that identifies who's speaking when" - }, - "gpt-realtime-whisper": { - "provider": "openai", - "model_type": "stt", - "description": "Streaming speech-to-text for realtime transcription" - }, - "gpt-5.6": { - "provider": "openai", - "model_type": "llm", - "description": "Preview frontier model for select partners; broad availability coming soon" - }, - "gpt-5.5": { - "provider": "openai", - "model_type": "llm", - "description": "OpenAI flagship — complex reasoning, agentic coding, 1M context" - }, - "gpt-5.5-pro": { - "provider": "openai", - "model_type": "llm", - "description": "Higher-accuracy GPT-5.5 variant with parallel test-time compute" - }, - "gpt-5.4": { - "provider": "openai", - "model_type": "llm", - "description": "Affordable frontier model for coding and professional work" - }, - "gpt-5.4-pro": { - "provider": "openai", - "model_type": "llm", - "description": "Higher-accuracy GPT-5.4 variant" - }, - "gpt-5.4-mini": { - "provider": "openai", - "model_type": "llm", - "description": "Strong mini model for coding, computer use, and subagents" - }, - "gpt-5.4-nano": { - "provider": "openai", - "model_type": "llm", - "description": "Cheapest GPT-5.4-class model for simple high-volume tasks" - }, - "gpt-5.3-codex": { - "provider": "openai", - "model_type": "llm", - "description": "Most capable agentic coding model" - }, - "gpt-5.2": { - "provider": "openai", - "model_type": "llm", - "description": "Previous frontier model for professional work with configurable reasoning" - }, - "gpt-5.2-pro": { - "provider": "openai", - "model_type": "llm", - "description": "Previous pro model for professional work" - }, - "gpt-5.2-codex": { - "provider": "openai", - "model_type": "llm", - "description": "Intelligent coding model optimized for long-horizon agentic tasks (deprecated)" - }, - "gpt-5.1": { - "provider": "openai", - "model_type": "llm", - "description": "Best model for coding and agentic tasks with configurable reasoning effort" - }, - "gpt-5.1-codex": { - "provider": "openai", - "model_type": "llm", - "description": "GPT-5.1 optimized for agentic coding in Codex (deprecated)" - }, - "gpt-5.1-codex-max": { - "provider": "openai", - "model_type": "llm", - "description": "GPT-5.1-codex optimized for long-running tasks (deprecated)" - }, - "gpt-5.1-codex-mini": { - "provider": "openai", - "model_type": "llm", - "description": "Smaller, cost-effective GPT-5.1-Codex variant (deprecated)" - }, - "gpt-5": { - "provider": "openai", - "model_type": "llm", - "description": "Intelligent reasoning model for coding and agentic tasks" - }, - "gpt-5-pro": { - "provider": "openai", - "model_type": "llm", - "description": "Higher-accuracy GPT-5 variant" - }, - "gpt-5-mini": { - "provider": "openai", - "model_type": "llm", - "description": "Near-frontier intelligence for cost-sensitive, low-latency workloads" - }, - "gpt-5-nano": { - "provider": "openai", - "model_type": "llm", - "description": "Fastest, most cost-efficient GPT-5 variant" - }, - "gpt-5-codex": { - "provider": "openai", - "model_type": "llm", - "description": "GPT-5 optimized for agentic coding in Codex (deprecated)" - }, - "codex-mini-latest": { - "provider": "openai", - "model_type": "llm", - "description": "Fast reasoning model optimized for the Codex CLI (deprecated)" - }, - "o3-pro": { - "provider": "openai", - "model_type": "llm", - "description": "o3 with more compute for better responses" - }, - "o3": { - "provider": "openai", - "model_type": "llm", - "description": "Reasoning model for complex tasks (succeeded by GPT-5)" - }, - "o3-mini": { - "provider": "openai", - "model_type": "llm", - "description": "Small o3 alternative (deprecated)" - }, - "o3-deep-research": { - "provider": "openai", - "model_type": "llm", - "description": "OpenAI flagship \u2014 complex reasoning, agentic coding, 1M context" - }, - "o4-mini": { - "provider": "openai", - "model_type": "llm", - "description": "Fast, cost-efficient reasoning model (succeeded by GPT-5 mini)" - }, - "o4-mini-deep-research": { - "provider": "openai", - "model_type": "llm", - "description": "Faster, more affordable deep research model (deprecated)" - }, - "o1-pro": { - "provider": "openai", - "model_type": "llm", - "description": "o1 with more compute for better responses (deprecated)" - }, - "o1": { - "provider": "openai", - "model_type": "llm", - "description": "Previous full o-series reasoning model (deprecated)" - }, - "o1-mini": { - "provider": "openai", - "model_type": "llm", - "description": "Small o1 alternative (deprecated)" - }, - "o1-preview": { - "provider": "openai", - "model_type": "llm", - "description": "Preview of the first o-series reasoning model (deprecated)" - }, - "gpt-4.1": { - "provider": "openai", - "model_type": "llm", - "description": "Smartest non-reasoning GPT-4.x model" - }, - "gpt-4.1-mini": { - "provider": "openai", - "model_type": "llm", - "description": "Smaller, faster GPT-4.1 variant" - }, - "gpt-4.1-nano": { - "provider": "openai", - "model_type": "llm", - "description": "Fastest, most cost-efficient GPT-4.1 variant (deprecated)" - }, - "gpt-4o": { - "provider": "openai", - "model_type": "llm", - "description": "Fast, intelligent, flexible GPT model (deprecated)" - }, - "gpt-4o-mini": { - "provider": "openai", - "model_type": "llm", - "description": "Fast, affordable small model for focused tasks (deprecated)" - }, - "gpt-4o-search-preview": { - "provider": "openai", - "model_type": "llm", - "description": "GPT-4o with web search in Chat Completions (deprecated)" - }, - "gpt-4o-mini-search-preview": { - "provider": "openai", - "model_type": "llm", - "description": "GPT-4o mini with web search in Chat Completions (deprecated)" - }, - "gpt-4-turbo": { - "provider": "openai", - "model_type": "llm", - "description": "Older high-intelligence GPT model (deprecated)" - }, - "gpt-4-turbo-preview": { - "provider": "openai", - "model_type": "llm", - "description": "Older fast GPT model preview (deprecated)" - }, - "gpt-4": { - "provider": "openai", - "model_type": "llm", - "description": "Older high-intelligence GPT model (deprecated)" - }, - "gpt-4.5-preview": { - "provider": "openai", - "model_type": "llm", - "description": "Deprecated large GPT preview model" - }, - "gpt-3.5-turbo": { - "provider": "openai", - "model_type": "llm", - "description": "Legacy GPT model for cheaper chat tasks (deprecated)" - }, - "computer-use-preview": { - "provider": "openai", - "model_type": "llm", - "description": "Specialized model for computer use tool (deprecated)" - }, - "gpt-4o-audio-preview": { - "provider": "openai", - "model_type": "llm", - "description": "GPT-4o with audio input/output on Chat Completions (deprecated)" - }, - "gpt-4o-mini-audio-preview": { - "provider": "openai", - "model_type": "llm", - "description": "GPT-4o mini with audio input/output on Chat Completions (deprecated)" - }, - "gpt-audio-1.5": { - "provider": "openai", - "model_type": "llm", - "description": "Best voice model for audio in/out with Chat Completions" - }, - "gpt-audio": { - "provider": "openai", - "model_type": "llm", - "description": "Audio inputs and outputs with Chat Completions API" - }, - "gpt-audio-mini": { - "provider": "openai", - "model_type": "llm", - "description": "Cost-efficient audio in/out with Chat Completions (deprecated)" - }, - "chat-latest": { - "provider": "openai", - "model_type": "llm", - "description": "Latest Instant model used in ChatGPT; not recommended for most API use" - }, - "gpt-5.3-chat": { - "provider": "openai", - "model_type": "llm", - "description": "GPT-5.3 Instant model used in ChatGPT (deprecated)" - }, - "gpt-5.2-chat": { - "provider": "openai", - "model_type": "llm", - "description": "GPT-5.2 model used in ChatGPT (deprecated)" - }, - "gpt-5.1-chat": { - "provider": "openai", - "model_type": "llm", - "description": "GPT-5.1 model used in ChatGPT (deprecated)" - }, - "gpt-5-chat": { - "provider": "openai", - "model_type": "llm", - "description": "GPT-5 model used in ChatGPT (deprecated)" - }, - "chatgpt-4o-latest": { - "provider": "openai", - "model_type": "llm", - "description": "GPT-4o model used in ChatGPT (deprecated)" - }, - "gpt-oss-120b": { - "provider": "openai", - "model_type": "llm", - "description": "Most powerful OpenAI open-weight model (Apache 2.0)" - }, - "gpt-oss-20b": { - "provider": "openai", - "model_type": "llm", - "description": "Medium-sized OpenAI open-weight model for low latency (Apache 2.0)" - }, - "gpt-5.6-sol": { - "provider": "openai", - "model_type": "llm", - "description": "GPT-5.6 flagship — frontier reasoning, agentic coding, and computer use (1M context)", - "featured": true, - "featured_rank": 1, - "highlights": ["Flagship", "1M context", "Agentic coding"] - }, - "gpt-5.6-terra": { - "provider": "openai", - "model_type": "llm", - "description": "GPT-5.6 balanced tier — strong everyday performance at lower cost than Sol" - }, - "gpt-5.6-luna": { - "provider": "openai", - "model_type": "llm", - "description": "GPT-5.6 fast tier — lowest-cost model for high-volume classification and extraction" - }, - "gpt-5.6": { - "provider": "openai", - "model_type": "llm", - "description": "OpenAI API alias for gpt-5.6-sol" - }, - "gpt-4o-mini-tts": { - "provider": "openai", - "model_type": "tts", - "description": "Text-to-speech powered by GPT-4o mini (deprecated)", - "featured": true, - "featured_rank": 3, - "highlights": [ - "Natural prosody", - "Multiple voices", - "Studio quality" - ] - }, - "claude-sonnet-4.5": { - "provider": "anthropic", - "model_type": "llm" - }, - "claude-opus-4.5": { - "provider": "anthropic", - "model_type": "llm" - }, - "claude-haiku-4.5": { - "provider": "anthropic", - "model_type": "llm" - }, - "claude-sonnet-4.6": { - "provider": "anthropic", - "model_type": "llm" - }, - "claude-opus-4.6": { - "provider": "anthropic", - "model_type": "llm" - }, - "claude-opus-4-7": { - "provider": "anthropic", - "model_type": "llm", - "description": "Claude Opus 4.7 \u2014 stronger coding, vision, and complex multi-step tasks" - }, - "claude-opus-4-8": { - "provider": "anthropic", - "model_type": "llm", - "description": "Claude Opus 4.8 \u2014 most capable Opus-tier model; 1M context, adaptive thinking" - }, - "claude-fable-5": { - "provider": "anthropic", - "model_type": "llm", - "description": "Claude Fable 5 \u2014 Anthropic's most capable widely released model for demanding agentic work" - }, - "claude-mythos-5": { - "provider": "anthropic", - "model_type": "llm", - "description": "Claude Mythos 5 \u2014 Fable 5 capabilities; limited availability via Project Glasswing" - }, - "claude-sonnet-5": { - "provider": "anthropic", - "model_type": "llm", - "description": "Claude Sonnet 5 \u2014 latest Sonnet tier; near-Opus quality with lower latency and cost", - "featured": true, - "featured_rank": 4, - "highlights": ["Latest Sonnet", "Agentic workflows", "Cost-efficient"] - }, - "grok-4.3": { - "provider": "xai", - "model_type": "llm", - "description": "xAI flagship \u2014 low hallucination, agentic tool calling, 1M context" - }, - "grok-build-0.1": { - "provider": "xai", - "model_type": "llm", - "description": "xAI fast coding model trained for agentic coding (early access)" - }, - "grok-4.20-0309-reasoning": { - "provider": "xai", - "model_type": "llm", - "description": "Grok 4.20 reasoning snapshot (Mar 2026)" - }, - "grok-4.20-0309-non-reasoning": { - "provider": "xai", - "model_type": "llm", - "description": "Grok 4.20 non-reasoning snapshot (Mar 2026)" - }, - "grok-4.20-multi-agent-0309": { - "provider": "xai", - "model_type": "llm", - "description": "Grok 4.20 multi-agent snapshot (Mar 2026)" - }, - "deepseek-v4-pro": { - "provider": "fireworks", - "model_type": "llm", - "description": "DeepSeek V4 Pro \u2014 frontier MoE reasoning and coding (1M context)" - }, - "deepseek-v4-flash": { - "provider": "fireworks", - "model_type": "llm", - "description": "DeepSeek V4 Flash \u2014 fast extraction, classification, and search" - }, - "kimi-k2p6": { - "provider": "fireworks", - "model_type": "llm", - "description": "Kimi K2.6 \u2014 native multimodal agentic model for long-horizon coding" - }, - "kimi-k2p5": { - "provider": "fireworks", - "model_type": "llm", - "description": "Kimi K2.5 \u2014 unified vision/text agentic model with controllable reasoning" - }, - "glm-5p1": { - "provider": "fireworks", - "model_type": "llm", - "description": "GLM 5.1 \u2014 frontier open model for reasoning and agentic workflows" - }, - "minimax-m2p7": { - "provider": "fireworks", - "model_type": "llm", - "description": "MiniMax M2.7 \u2014 MoE model for complex agent harnesses and productivity tasks" - }, - "minimax-m2p5": { - "provider": "fireworks", - "model_type": "llm", - "description": "MiniMax M2.5 \u2014 fast coding and agentic tool use at low cost" - }, - "qwen3p6-plus": { - "provider": "fireworks", - "model_type": "llm", - "description": "Qwen 3.6 Plus \u2014 flagship multimodal model (Fireworks exclusive outside Alibaba)" - }, - "gpt-oss-120b": { - "provider": "fireworks", - "model_type": "llm", - "description": "OpenAI gpt-oss-120b \u2014 high-quality open-weight model for general reasoning" - }, - "gpt-oss-20b": { - "provider": "fireworks", - "model_type": "llm", - "description": "OpenAI gpt-oss-20b \u2014 fast open-weight model for chat and classification" - }, - "firefunction-v2": { - "provider": "fireworks", - "model_type": "llm", - "description": "FireFunction V2 \u2014 Fireworks function-calling optimized model" - }, - "google-speech-v2": { - "provider": "google", - "model_type": "stt" - }, - "gemini-2.5-pro-stt": { - "provider": "google", - "model_type": "stt", - "description": "Gemini 2.5 Pro used for audio transcription via LiteLLM proxy (audio input -> text)" - }, - "gemini-2.5-flash-stt": { - "provider": "google", - "model_type": "stt", - "description": "Gemini 2.5 Flash used for audio transcription via LiteLLM proxy (fast, cost-efficient)", - "featured": true, - "featured_rank": 6, - "highlights": [ - "Fast transcription", - "Cost-efficient", - "Gemini 2.5" - ] - }, - "gemini-2.5-flash-lite-stt": { - "provider": "google", - "model_type": "stt", - "description": "Gemini 2.5 Flash Lite used for audio transcription via LiteLLM proxy (lowest cost)" - }, - "gemini-2.5-pro": { - "provider": "google", - "model_type": "llm" - }, - "gemini-2.5-flash": { - "provider": "google", - "model_type": "llm" - }, - "gemini-2.5-flash-lite": { - "provider": "google", - "model_type": "llm", - "description": "Gemini 2.5 Flash Lite \u2014 lowest-latency 2.5 model; ideal for diarisation, classification, and other low-reasoning multimodal tasks" - }, - "gemini-3-pro-preview": { - "provider": "google", - "model_type": "llm", - "description": "Gemini 3 Pro \u2014 advanced reasoning and multimodal understanding" - }, - "gemini-3-flash-preview": { - "provider": "google", - "model_type": "llm", - "description": "Gemini 3 Flash \u2014 frontier-class performance at lower cost" - }, - "gemini-3.1-pro-preview": { - "provider": "google", - "model_type": "llm", - "description": "Gemini 3.1 Pro \u2014 latest reasoning model with 1M context" - }, - "gemini-3.5-flash": { - "provider": "google", - "model_type": "llm", - "description": "Gemini 3.5 Flash \u2014 GA flagship Flash model for agentic coding and long-horizon tasks", - "featured": true, - "featured_rank": 7, - "highlights": ["GA stable", "1M context", "Agentic coding"] - }, - "gemini-3.5-flash-lite": { - "provider": "google", - "model_type": "llm", - "description": "Gemini 3.5 Flash Lite \u2014 fastest, lowest-cost 3.5 model for high-throughput extraction, classification, and subagent workflows" - }, - "gemini-3.1-flash-lite": { - "provider": "google", - "model_type": "llm", - "description": "Gemini 3.1 Flash Lite \u2014 cost-efficient stable model for high-volume lightweight tasks" - }, - "azure-speech-v1": { - "provider": "azure", - "model_type": "stt", - "description": "Azure Speech-to-text (batch and realtime)" - }, - "azure-openai-gpt4": { - "provider": "azure", - "model_type": "llm", - "description": "Legacy alias — maps to gpt-4 deployment; prefer azure-gpt-5-mini if that is your deployment name" - }, - "azure-gpt-4o": { - "provider": "azure", - "model_type": "llm", - "description": "Azure OpenAI GPT-4o — catalog key maps to deployment name gpt-4o" - }, - "azure-gpt-4o-mini": { - "provider": "azure", - "model_type": "llm", - "description": "Azure OpenAI GPT-4o mini" - }, - "azure-gpt-4.1": { - "provider": "azure", - "model_type": "llm", - "description": "Azure OpenAI GPT-4.1" - }, - "azure-gpt-4.1-mini": { - "provider": "azure", - "model_type": "llm", - "description": "Azure OpenAI GPT-4.1 mini" - }, - "azure-gpt-4.1-nano": { - "provider": "azure", - "model_type": "llm", - "description": "Azure OpenAI GPT-4.1 nano" - }, - "azure-gpt-5": { - "provider": "azure", - "model_type": "llm", - "description": "Azure OpenAI GPT-5" - }, - "azure-gpt-5-mini": { - "provider": "azure", - "model_type": "llm", - "description": "Azure OpenAI GPT-5 mini — pick this if your deployment is named gpt-5-mini" - }, - "azure-gpt-5-nano": { - "provider": "azure", - "model_type": "llm", - "description": "Azure OpenAI GPT-5 nano" - }, - "azure-gpt-5.1": { - "provider": "azure", - "model_type": "llm", - "description": "Azure OpenAI GPT-5.1" - }, - "azure-gpt-5.2": { - "provider": "azure", - "model_type": "llm", - "description": "Azure OpenAI GPT-5.2" - }, - "azure-o3": { - "provider": "azure", - "model_type": "llm", - "description": "Azure OpenAI o3 reasoning model" - }, - "azure-o3-mini": { - "provider": "azure", - "model_type": "llm", - "description": "Azure OpenAI o3 mini" - }, - "azure-o4-mini": { - "provider": "azure", - "model_type": "llm", - "description": "Azure OpenAI o4 mini" - }, - "azure-tts-v1": { - "provider": "azure", - "model_type": "tts", - "description": "Azure neural text-to-speech" - }, - "aws-transcribe": { - "provider": "aws", - "model_type": "stt" - }, - "aws-bedrock-claude": { - "provider": "aws", - "model_type": "llm" - }, - "aws-polly": { - "provider": "aws", - "model_type": "tts" - }, - "deepgram-flux": { - "provider": "deepgram", - "model_type": "stt", - "description": "Conversational STT with integrated turn detection for voice agents", - "featured": true, - "featured_rank": 4, - "highlights": [ - "Turn detection", - "Voice agents", - "Low latency" - ] - }, - "deepgram-nova-3": { - "provider": "deepgram", - "model_type": "stt", - "description": "Highest-performing general-purpose ASR model" - }, - "deepgram-nova-3-general": { - "provider": "deepgram", - "model_type": "stt" - }, - "deepgram-nova-3-general-preview-12-2025": { - "provider": "deepgram", - "model_type": "stt" - }, - "deepgram-nova-2": { - "provider": "deepgram", - "model_type": "stt", - "description": "Available for languages not yet supported by Nova-3" - }, - "pulse-v4": { - "provider": "smallest", - "model_type": "stt", - "description": "Smallest Pulse v4 speech-to-text (batch and realtime)" - }, - "cartesia-sonic-3": { - "provider": "cartesia", - "model_type": "tts", - "description": "Flagship streaming TTS with laughter, volume/speed/emotion controls, 42 languages", - "featured": true, - "featured_rank": 1, - "highlights": [ - "Ultra-low latency", - "42 languages", - "Emotion controls" - ] - }, - "cartesia-sonic-3-mini": { - "provider": "cartesia", - "model_type": "tts" - }, - "cartesia-sonic-3-nano": { - "provider": "cartesia", - "model_type": "tts" - }, - "eleven_v3": { - "provider": "elevenlabs", - "model_type": "tts", - "description": "Human-like and expressive speech generation", - "languages": "70+ languages" - }, - "eleven_ttv_v3": { - "provider": "elevenlabs", - "model_type": "tts", - "description": "Human-like and expressive voice design model (Text to Voice)", - "languages": "70+ languages" - }, - "eleven_multilingual_v2": { - "provider": "elevenlabs", - "model_type": "tts", - "description": "Most lifelike model with rich emotional expression", - "languages": "en, ja, zh, de, hi, fr, ko, pt, it, es, id, nl, tr, fil, pl, sv, bg, ro, ar, cs, el, fi, hr, ms, sk, da, ta, uk, ru" - }, - "eleven_flash_v2_5": { - "provider": "elevenlabs", - "model_type": "tts", - "description": "Ultra-fast model optimized for real-time use (~75ms)", - "languages": "en, ja, zh, de, hi, fr, ko, pt, it, es, id, nl, tr, fil, pl, sv, bg, ro, ar, cs, el, fi, hr, ms, sk, da, ta, uk, ru, hu, no, vi", - "featured": true, - "featured_rank": 2, - "highlights": [ - "~75ms latency", - "Real-time", - "Multilingual" - ] - }, - "eleven_turbo_v2_5": { - "provider": "elevenlabs", - "model_type": "tts", - "description": "High quality, low-latency model (~250ms-300ms)", - "languages": "en, ja, zh, de, hi, fr, ko, pt, it, es, id, nl, tr, fil, pl, sv, bg, ro, ar, cs, el, fi, hr, ms, sk, da, ta, uk, ru, hu, no, vi" - }, - "eleven_multilingual_sts_v2": { - "provider": "elevenlabs", - "model_type": "sts", - "description": "State-of-the-art multilingual voice changer model (Speech to Speech)", - "languages": "en, ja, zh, de, hi, fr, ko, pt, it, es, id, nl, tr, fil, pl, sv, bg, ro, ar, cs, el, fi, hr, ms, sk, da, ta, uk, ru" - }, - "eleven_multilingual_ttv_v2": { - "provider": "elevenlabs", - "model_type": "tts", - "description": "State-of-the-art multilingual voice designer model (Text to Voice)", - "languages": "en, ja, zh, de, hi, fr, ko, pt, it, es, id, nl, tr, fil, pl, sv, bg, ro, ar, cs, el, fi, hr, ms, sk, da, ta, uk, ru" - }, - "eleven_english_sts_v2": { - "provider": "elevenlabs", - "model_type": "sts", - "description": "English-only voice changer model (Speech to Speech)", - "languages": "en" - }, - "scribe_v2_realtime": { - "provider": "elevenlabs", - "model_type": "stt", - "description": "Real-time speech recognition model", - "languages": "90+ languages", - "featured": true, - "featured_rank": 5, - "highlights": [ - "Real-time STT", - "90+ languages", - "Streaming" - ] - }, - "scribe_v2": { - "provider": "elevenlabs", - "model_type": "stt", - "description": "Most accurate transcription model with keyterm prompting and entity detection", - "languages": "90+ languages" - }, - "eleven_text_to_sound_v2": { - "provider": "elevenlabs", - "model_type": "sound_effects", - "description": "Sound effects generation from text prompts" - }, - "music_v1": { - "provider": "elevenlabs", - "model_type": "music", - "description": "Studio-grade music generation from text prompts", - "languages": "en, es, de, ja, and more" - }, - "murf-falcon": { - "provider": "murf", - "model_type": "tts", - "description": "Murf FALCON \u2013 ultra-low latency streaming TTS", - "languages": "en, es, de, fr, hi, ja, ko, pt, it, zh, and 10+ more", - "voices_source_file": "murf_falcon_voices.json" - }, - "murf-gen2": { - "provider": "murf", - "model_type": "tts", - "description": "Murf GEN2 \u2013 high-quality, natural-sounding TTS", - "languages": "en, es, de, fr, hi, ja, ko, pt, it, zh, and 10+ more", - "voices_source_file": "murf_gen2_voices.json" - }, - "voicemaker-ai3": { - "provider": "voicemaker", - "model_type": "tts", - "description": "VoiceMaker AI3 neural TTS voices", - "languages": "en-US, en-GB, en-AU, es-ES, fr-FR, de-DE, ja-JP, zh-CN", - "voices_source_file": "voicemaker_voices.json" - }, - "voicemaker-proplus": { - "provider": "voicemaker", - "model_type": "tts", - "description": "VoiceMaker ProPlus expressive multilingual voices", - "languages": "multi-lang + regional accents", - "voices_source_file": "voicemaker_voices.json" - }, - "lightning-v3.1": { - "provider": "smallest", - "model_type": "tts", - "description": "Smallest Lightning v3.1 low-latency neural TTS" - }, - "sarvam-30b": { - "provider": "sarvam", - "model_type": "llm", - "description": "Sarvam 30B \u2014 balanced Indian-language + English chat model (64K context)" - }, - "sarvam-105b": { - "provider": "sarvam", - "model_type": "llm", - "description": "Sarvam 105B \u2014 flagship MoE for complex reasoning, coding, and agentic workflows (128K context)" - }, - "saaras:v3": { - "provider": "sarvam", - "model_type": "stt", - "description": "State-of-the-art Sarvam STT model with transcribe/translate modes" - }, - "bulbul:v3": { - "provider": "sarvam", - "model_type": "tts", - "description": "High quality Indian language TTS", - "voices": [ - { - "id": "aditya", - "name": "Aditya", - "gender": "Male" - }, - { - "id": "ritu", - "name": "Ritu", - "gender": "Female" - }, - { - "id": "ashutosh", - "name": "Ashutosh", - "gender": "Male" - }, - { - "id": "priya", - "name": "Priya", - "gender": "Female" - }, - { - "id": "neha", - "name": "Neha", - "gender": "Female" - }, - { - "id": "rahul", - "name": "Rahul", - "gender": "Male" - }, - { - "id": "pooja", - "name": "Pooja", - "gender": "Female" - }, - { - "id": "rohan", - "name": "Rohan", - "gender": "Male" - }, - { - "id": "simran", - "name": "Simran", - "gender": "Female" - }, - { - "id": "kavya", - "name": "Kavya", - "gender": "Female" - }, - { - "id": "amit", - "name": "Amit", - "gender": "Male" - }, - { - "id": "dev", - "name": "Dev", - "gender": "Male" - }, - { - "id": "ishita", - "name": "Ishita", - "gender": "Female" - }, - { - "id": "shreya", - "name": "Shreya", - "gender": "Female" - }, - { - "id": "ratan", - "name": "Ratan", - "gender": "Male" - }, - { - "id": "varun", - "name": "Varun", - "gender": "Male" - }, - { - "id": "manan", - "name": "Manan", - "gender": "Male" - }, - { - "id": "sumit", - "name": "Sumit", - "gender": "Male" - }, - { - "id": "roopa", - "name": "Roopa", - "gender": "Female" - }, - { - "id": "kabir", - "name": "Kabir", - "gender": "Male" - }, - { - "id": "aayan", - "name": "Aayan", - "gender": "Male" - }, - { - "id": "shubh", - "name": "Shubh", - "gender": "Male" - }, - { - "id": "advait", - "name": "Advait", - "gender": "Male" - }, - { - "id": "amelia", - "name": "Amelia", - "gender": "Female" - }, - { - "id": "sophia", - "name": "Sophia", - "gender": "Female" - }, - { - "id": "anand", - "name": "Anand", - "gender": "Male" - }, - { - "id": "tanya", - "name": "Tanya", - "gender": "Female" - }, - { - "id": "tarun", - "name": "Tarun", - "gender": "Male" - }, - { - "id": "sunny", - "name": "Sunny", - "gender": "Male" - }, - { - "id": "mani", - "name": "Mani", - "gender": "Male" - }, - { - "id": "gokul", - "name": "Gokul", - "gender": "Male" - }, - { - "id": "vijay", - "name": "Vijay", - "gender": "Male" - }, - { - "id": "shruti", - "name": "Shruti", - "gender": "Female" - }, - { - "id": "suhani", - "name": "Suhani", - "gender": "Female" - }, - { - "id": "mohit", - "name": "Mohit", - "gender": "Male" - }, - { - "id": "kavitha", - "name": "Kavitha", - "gender": "Female" - }, - { - "id": "rehan", - "name": "Rehan", - "gender": "Male" - }, - { - "id": "soham", - "name": "Soham", - "gender": "Male" - }, - { - "id": "rupali", - "name": "Rupali", - "gender": "Female" - } - ] - } -} +{ + "whisper-1": { + "provider": "openai", + "model_type": "stt", + "description": "General-purpose speech recognition (verbose_json + word/segment timestamps)", + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.006 + } + }, + "gpt-4o-transcribe": { + "provider": "openai", + "model_type": "stt", + "description": "GPT-4o speech-to-text; higher accuracy, no granular word timestamps", + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.00372 + } + }, + "gpt-4o-mini-transcribe": { + "provider": "openai", + "model_type": "stt", + "description": "Cheaper / faster GPT-4o transcription; no granular word timestamps", + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.00186 + } + }, + "gpt-4o-transcribe-diarize": { + "provider": "openai", + "model_type": "stt", + "description": "Transcription model that identifies who's speaking when", + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.00372 + } + }, + "gpt-realtime-whisper": { + "provider": "openai", + "model_type": "stt", + "description": "Streaming speech-to-text for realtime transcription", + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.01698 + } + }, + "gpt-5.6": { + "provider": "openai", + "model_type": "llm", + "description": "OpenAI API alias for gpt-5.6-sol", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 5.0, + "output_per_1m": 30.0, + "cache_read_per_1m": 0.5, + "cache_write_per_1m": 6.25 + } + }, + "gpt-5.5": { + "provider": "openai", + "model_type": "llm", + "description": "OpenAI flagship — complex reasoning, agentic coding, 1M context", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 5.0, + "output_per_1m": 30.0, + "cache_read_per_1m": 0.5 + } + }, + "gpt-5.5-pro": { + "provider": "openai", + "model_type": "llm", + "description": "Higher-accuracy GPT-5.5 variant with parallel test-time compute", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 30.0, + "output_per_1m": 180.0, + "cache_read_per_1m": 3.0 + } + }, + "gpt-5.4": { + "provider": "openai", + "model_type": "llm", + "description": "Affordable frontier model for coding and professional work", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.5, + "output_per_1m": 15.0, + "cache_read_per_1m": 0.25 + } + }, + "gpt-5.4-pro": { + "provider": "openai", + "model_type": "llm", + "description": "Higher-accuracy GPT-5.4 variant", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 30.0, + "output_per_1m": 180.0, + "cache_read_per_1m": 3.0 + } + }, + "gpt-5.4-mini": { + "provider": "openai", + "model_type": "llm", + "description": "Strong mini model for coding, computer use, and subagents", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.75, + "output_per_1m": 4.5, + "cache_read_per_1m": 0.075 + } + }, + "gpt-5.4-nano": { + "provider": "openai", + "model_type": "llm", + "description": "Cheapest GPT-5.4-class model for simple high-volume tasks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.2, + "output_per_1m": 1.25, + "cache_read_per_1m": 0.02 + } + }, + "gpt-5.3-codex": { + "provider": "openai", + "model_type": "llm", + "description": "Most capable agentic coding model", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.75, + "output_per_1m": 14.0, + "cache_read_per_1m": 0.175 + } + }, + "gpt-5.2": { + "provider": "openai", + "model_type": "llm", + "description": "Previous frontier model for professional work with configurable reasoning", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.75, + "output_per_1m": 14.0, + "cache_read_per_1m": 0.175 + } + }, + "gpt-5.2-pro": { + "provider": "openai", + "model_type": "llm", + "description": "Previous pro model for professional work", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 21.0, + "output_per_1m": 168.0 + } + }, + "gpt-5.2-codex": { + "provider": "openai", + "model_type": "llm", + "description": "Intelligent coding model optimized for long-horizon agentic tasks (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.75, + "output_per_1m": 14.0, + "cache_read_per_1m": 0.175 + } + }, + "gpt-5.1": { + "provider": "openai", + "model_type": "llm", + "description": "Best model for coding and agentic tasks with configurable reasoning effort", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.25, + "output_per_1m": 10.0, + "cache_read_per_1m": 0.125 + } + }, + "gpt-5.1-codex": { + "provider": "openai", + "model_type": "llm", + "description": "GPT-5.1 optimized for agentic coding in Codex (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.25, + "output_per_1m": 10.0, + "cache_read_per_1m": 0.125 + } + }, + "gpt-5.1-codex-max": { + "provider": "openai", + "model_type": "llm", + "description": "GPT-5.1-codex optimized for long-running tasks (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.25, + "output_per_1m": 10.0, + "cache_read_per_1m": 0.125 + } + }, + "gpt-5.1-codex-mini": { + "provider": "openai", + "model_type": "llm", + "description": "Smaller, cost-effective GPT-5.1-Codex variant (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.25, + "output_per_1m": 2.0, + "cache_read_per_1m": 0.025 + } + }, + "gpt-5": { + "provider": "openai", + "model_type": "llm", + "description": "Intelligent reasoning model for coding and agentic tasks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.25, + "output_per_1m": 10.0, + "cache_read_per_1m": 0.125 + } + }, + "gpt-5-pro": { + "provider": "openai", + "model_type": "llm", + "description": "Higher-accuracy GPT-5 variant", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 15.0, + "output_per_1m": 120.0 + } + }, + "gpt-5-mini": { + "provider": "openai", + "model_type": "llm", + "description": "Near-frontier intelligence for cost-sensitive, low-latency workloads", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.25, + "output_per_1m": 2.0, + "cache_read_per_1m": 0.025 + } + }, + "gpt-5-nano": { + "provider": "openai", + "model_type": "llm", + "description": "Fastest, most cost-efficient GPT-5 variant", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.05, + "output_per_1m": 0.4, + "cache_read_per_1m": 0.005 + } + }, + "gpt-5-codex": { + "provider": "openai", + "model_type": "llm", + "description": "GPT-5 optimized for agentic coding in Codex (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.25, + "output_per_1m": 10.0, + "cache_read_per_1m": 0.125 + } + }, + "codex-mini-latest": { + "provider": "openai", + "model_type": "llm", + "description": "Fast reasoning model optimized for the Codex CLI (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.5, + "output_per_1m": 6.0, + "cache_read_per_1m": 0.375 + } + }, + "o3-pro": { + "provider": "openai", + "model_type": "llm", + "description": "o3 with more compute for better responses", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 20.0, + "output_per_1m": 80.0 + } + }, + "o3": { + "provider": "openai", + "model_type": "llm", + "description": "Reasoning model for complex tasks (succeeded by GPT-5)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.0, + "output_per_1m": 8.0, + "cache_read_per_1m": 0.5 + } + }, + "o3-mini": { + "provider": "openai", + "model_type": "llm", + "description": "Small o3 alternative (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.1, + "output_per_1m": 4.4, + "cache_read_per_1m": 0.55 + } + }, + "o3-deep-research": { + "provider": "openai", + "model_type": "llm", + "description": "OpenAI flagship — complex reasoning, agentic coding, 1M context", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 10.0, + "output_per_1m": 40.0, + "cache_read_per_1m": 2.5 + } + }, + "o4-mini": { + "provider": "openai", + "model_type": "llm", + "description": "Fast, cost-efficient reasoning model (succeeded by GPT-5 mini)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.1, + "output_per_1m": 4.4, + "cache_read_per_1m": 0.275 + } + }, + "o4-mini-deep-research": { + "provider": "openai", + "model_type": "llm", + "description": "Faster, more affordable deep research model (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.0, + "output_per_1m": 8.0, + "cache_read_per_1m": 0.5 + } + }, + "o1-pro": { + "provider": "openai", + "model_type": "llm", + "description": "o1 with more compute for better responses (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 150.0, + "output_per_1m": 600.0 + } + }, + "o1": { + "provider": "openai", + "model_type": "llm", + "description": "Previous full o-series reasoning model (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 15.0, + "output_per_1m": 60.0, + "cache_read_per_1m": 7.5 + } + }, + "o1-mini": { + "provider": "openai", + "model_type": "llm", + "description": "Small o1 alternative (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.21, + "output_per_1m": 4.84, + "cache_read_per_1m": 0.605 + } + }, + "o1-preview": { + "provider": "openai", + "model_type": "llm", + "description": "Preview of the first o-series reasoning model (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 15.0, + "output_per_1m": 60.0, + "cache_read_per_1m": 7.5 + } + }, + "gpt-4.1": { + "provider": "openai", + "model_type": "llm", + "description": "Smartest non-reasoning GPT-4.x model", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.0, + "output_per_1m": 8.0, + "cache_read_per_1m": 0.5 + } + }, + "gpt-4.1-mini": { + "provider": "openai", + "model_type": "llm", + "description": "Smaller, faster GPT-4.1 variant", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.4, + "output_per_1m": 1.6, + "cache_read_per_1m": 0.1 + } + }, + "gpt-4.1-nano": { + "provider": "openai", + "model_type": "llm", + "description": "Fastest, most cost-efficient GPT-4.1 variant (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.1, + "output_per_1m": 0.4, + "cache_read_per_1m": 0.025 + } + }, + "gpt-4o": { + "provider": "openai", + "model_type": "llm", + "description": "Fast, intelligent, flexible GPT model (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.5, + "output_per_1m": 10.0, + "cache_read_per_1m": 1.25 + } + }, + "gpt-4o-mini": { + "provider": "openai", + "model_type": "llm", + "description": "Fast, affordable small model for focused tasks (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.15, + "output_per_1m": 0.6, + "cache_read_per_1m": 0.075 + } + }, + "gpt-4o-search-preview": { + "provider": "openai", + "model_type": "llm", + "description": "GPT-4o with web search in Chat Completions (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.5, + "output_per_1m": 10.0, + "cache_read_per_1m": 1.25 + } + }, + "gpt-4o-mini-search-preview": { + "provider": "openai", + "model_type": "llm", + "description": "GPT-4o mini with web search in Chat Completions (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.15, + "output_per_1m": 0.6, + "cache_read_per_1m": 0.075 + } + }, + "gpt-4-turbo": { + "provider": "openai", + "model_type": "llm", + "description": "Older high-intelligence GPT model (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 10.0, + "output_per_1m": 30.0 + } + }, + "gpt-4-turbo-preview": { + "provider": "openai", + "model_type": "llm", + "description": "Older fast GPT model preview (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 10.0, + "output_per_1m": 30.0 + } + }, + "gpt-4": { + "provider": "openai", + "model_type": "llm", + "description": "Older high-intelligence GPT model (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 30.0, + "output_per_1m": 60.0 + } + }, + "gpt-4.5-preview": { + "provider": "openai", + "model_type": "llm", + "description": "Deprecated large GPT preview model", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 75.0, + "output_per_1m": 150.0, + "cache_read_per_1m": 37.5 + } + }, + "gpt-3.5-turbo": { + "provider": "openai", + "model_type": "llm", + "description": "Legacy GPT model for cheaper chat tasks (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.5, + "output_per_1m": 1.5 + } + }, + "computer-use-preview": { + "provider": "openai", + "model_type": "llm", + "description": "Specialized model for computer use tool (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 3.0, + "output_per_1m": 12.0 + } + }, + "gpt-4o-audio-preview": { + "provider": "openai", + "model_type": "llm", + "description": "GPT-4o with audio input/output on Chat Completions (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.5, + "output_per_1m": 10.0 + } + }, + "gpt-4o-mini-audio-preview": { + "provider": "openai", + "model_type": "llm", + "description": "GPT-4o mini with audio input/output on Chat Completions (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.15, + "output_per_1m": 0.6 + } + }, + "gpt-audio-1.5": { + "provider": "openai", + "model_type": "llm", + "description": "Best voice model for audio in/out with Chat Completions", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.5, + "output_per_1m": 10.0 + } + }, + "gpt-audio": { + "provider": "openai", + "model_type": "llm", + "description": "Audio inputs and outputs with Chat Completions API", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.5, + "output_per_1m": 10.0 + } + }, + "gpt-audio-mini": { + "provider": "openai", + "model_type": "llm", + "description": "Cost-efficient audio in/out with Chat Completions (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.6, + "output_per_1m": 2.4 + } + }, + "chat-latest": { + "provider": "openai", + "model_type": "llm", + "description": "Latest Instant model used in ChatGPT; not recommended for most API use", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.25, + "output_per_1m": 10.0, + "cache_read_per_1m": 0.125 + } + }, + "gpt-5.3-chat": { + "provider": "openai", + "model_type": "llm", + "description": "GPT-5.3 Instant model used in ChatGPT (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.75, + "output_per_1m": 14.0, + "cache_read_per_1m": 0.175 + } + }, + "gpt-5.2-chat": { + "provider": "openai", + "model_type": "llm", + "description": "GPT-5.2 model used in ChatGPT (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.75, + "output_per_1m": 14.0, + "cache_read_per_1m": 0.175 + } + }, + "gpt-5.1-chat": { + "provider": "openai", + "model_type": "llm", + "description": "GPT-5.1 model used in ChatGPT (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.38, + "output_per_1m": 11.0, + "cache_read_per_1m": 0.14 + } + }, + "gpt-5-chat": { + "provider": "openai", + "model_type": "llm", + "description": "GPT-5 model used in ChatGPT (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.25, + "output_per_1m": 10.0, + "cache_read_per_1m": 0.125 + } + }, + "chatgpt-4o-latest": { + "provider": "openai", + "model_type": "llm", + "description": "GPT-4o model used in ChatGPT (deprecated)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 5.0, + "output_per_1m": 15.0 + } + }, + "gpt-oss-120b": { + "provider": "fireworks", + "model_type": "llm", + "description": "OpenAI gpt-oss-120b — high-quality open-weight model for general reasoning", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.15, + "output_per_1m": 0.6, + "cache_read_per_1m": 0.015 + } + }, + "gpt-oss-20b": { + "provider": "fireworks", + "model_type": "llm", + "description": "OpenAI gpt-oss-20b — fast open-weight model for chat and classification", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.07, + "output_per_1m": 0.3, + "cache_read_per_1m": 0.035 + } + }, + "gpt-5.6-sol": { + "provider": "openai", + "model_type": "llm", + "description": "GPT-5.6 flagship — frontier reasoning, agentic coding, and computer use (1M context)", + "featured": true, + "featured_rank": 1, + "highlights": [ + "Flagship", + "1M context", + "Agentic coding" + ], + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 5.0, + "output_per_1m": 30.0, + "cache_read_per_1m": 0.5, + "cache_write_per_1m": 6.25 + } + }, + "gpt-5.6-terra": { + "provider": "openai", + "model_type": "llm", + "description": "GPT-5.6 balanced tier — strong everyday performance at lower cost than Sol", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.0, + "output_per_1m": 12.0, + "cache_read_per_1m": 0.2, + "cache_write_per_1m": 2.5 + } + }, + "gpt-5.6-luna": { + "provider": "openai", + "model_type": "llm", + "description": "GPT-5.6 fast tier — lowest-cost model for high-volume classification and extraction", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.2, + "output_per_1m": 1.2, + "cache_read_per_1m": 0.02, + "cache_write_per_1m": 0.25 + } + }, + "gpt-4o-mini-tts": { + "provider": "openai", + "model_type": "tts", + "description": "Text-to-speech powered by GPT-4o mini (deprecated)", + "featured": true, + "featured_rank": 3, + "highlights": [ + "Natural prosody", + "Multiple voices", + "Studio quality" + ], + "pricing": { + "source": "litellm_import", + "usage_kind": "tts", + "tts_per_1m_characters": 2.5 + } + }, + "claude-sonnet-4.5": { + "provider": "anthropic", + "model_type": "llm", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 3.0, + "output_per_1m": 15.0 + } + }, + "claude-opus-4.5": { + "provider": "anthropic", + "model_type": "llm", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 5.0, + "output_per_1m": 25.0 + } + }, + "claude-haiku-4.5": { + "provider": "anthropic", + "model_type": "llm", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.0, + "output_per_1m": 5.0, + "cache_read_per_1m": 0.1, + "cache_write_per_1m": 1.25 + } + }, + "claude-sonnet-4.6": { + "provider": "anthropic", + "model_type": "llm", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 3.0, + "output_per_1m": 15.0, + "cache_read_per_1m": 0.3, + "cache_write_per_1m": 3.75 + } + }, + "claude-opus-4.6": { + "provider": "anthropic", + "model_type": "llm", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 5.0, + "output_per_1m": 25.0, + "cache_read_per_1m": 0.5, + "cache_write_per_1m": 6.25 + } + }, + "claude-opus-4-7": { + "provider": "anthropic", + "model_type": "llm", + "description": "Claude Opus 4.7 — stronger coding, vision, and complex multi-step tasks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 5.0, + "output_per_1m": 25.0, + "cache_read_per_1m": 0.5, + "cache_write_per_1m": 6.25 + } + }, + "claude-opus-4-8": { + "provider": "anthropic", + "model_type": "llm", + "description": "Claude Opus 4.8 — most capable Opus-tier model; 1M context, adaptive thinking", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 5.0, + "output_per_1m": 25.0, + "cache_read_per_1m": 0.5, + "cache_write_per_1m": 6.25 + } + }, + "claude-fable-5": { + "provider": "anthropic", + "model_type": "llm", + "description": "Claude Fable 5 — Anthropic's most capable widely released model for demanding agentic work", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 10.0, + "output_per_1m": 50.0, + "cache_read_per_1m": 1.0, + "cache_write_per_1m": 12.5 + } + }, + "claude-mythos-5": { + "provider": "anthropic", + "model_type": "llm", + "description": "Claude Mythos 5 — Fable 5 capabilities; limited availability via Project Glasswing", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 10.0, + "output_per_1m": 50.0, + "cache_read_per_1m": 1.0, + "cache_write_per_1m": 12.5 + } + }, + "claude-sonnet-5": { + "provider": "anthropic", + "model_type": "llm", + "description": "Claude Sonnet 5 — latest Sonnet tier; near-Opus quality with lower latency and cost", + "featured": true, + "featured_rank": 4, + "highlights": [ + "Latest Sonnet", + "Agentic workflows", + "Cost-efficient" + ], + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.0, + "output_per_1m": 10.0, + "cache_read_per_1m": 0.2, + "cache_write_per_1m": 2.5 + } + }, + "grok-4.3": { + "provider": "xai", + "model_type": "llm", + "description": "xAI flagship — low hallucination, agentic tool calling, 1M context", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.25, + "output_per_1m": 2.5, + "cache_read_per_1m": 0.2 + } + }, + "grok-build-0.1": { + "provider": "xai", + "model_type": "llm", + "description": "xAI fast coding model trained for agentic coding (early access)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 3.0, + "output_per_1m": 15.0, + "cache_read_per_1m": 0.75 + } + }, + "grok-4.20-0309-reasoning": { + "provider": "xai", + "model_type": "llm", + "description": "Grok 4.20 reasoning snapshot (Mar 2026)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.25, + "output_per_1m": 2.5, + "cache_read_per_1m": 0.2 + } + }, + "grok-4.20-0309-non-reasoning": { + "provider": "xai", + "model_type": "llm", + "description": "Grok 4.20 non-reasoning snapshot (Mar 2026)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.2, + "output_per_1m": 0.5, + "cache_read_per_1m": 0.05 + } + }, + "grok-4.20-multi-agent-0309": { + "provider": "xai", + "model_type": "llm", + "description": "Grok 4.20 multi-agent snapshot (Mar 2026)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 3.0, + "output_per_1m": 15.0 + } + }, + "deepseek-v4-pro": { + "provider": "fireworks", + "model_type": "llm", + "description": "DeepSeek V4 Pro — frontier MoE reasoning and coding (1M context)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.74, + "output_per_1m": 3.48, + "cache_read_per_1m": 0.145 + } + }, + "deepseek-v4-flash": { + "provider": "fireworks", + "model_type": "llm", + "description": "DeepSeek V4 Flash — fast extraction, classification, and search", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.14, + "output_per_1m": 0.28, + "cache_read_per_1m": 0.028 + } + }, + "kimi-k2p6": { + "provider": "fireworks", + "model_type": "llm", + "description": "Kimi K2.6 — native multimodal agentic model for long-horizon coding", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.95, + "output_per_1m": 4.0, + "cache_read_per_1m": 0.16 + } + }, + "kimi-k2p5": { + "provider": "fireworks", + "model_type": "llm", + "description": "Kimi K2.5 — unified vision/text agentic model with controllable reasoning", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.6, + "output_per_1m": 3.0, + "cache_read_per_1m": 0.1 + } + }, + "glm-5p1": { + "provider": "fireworks", + "model_type": "llm", + "description": "GLM 5.1 — frontier open model for reasoning and agentic workflows", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.4, + "output_per_1m": 4.4, + "cache_read_per_1m": 0.26 + } + }, + "minimax-m2p7": { + "provider": "fireworks", + "model_type": "llm", + "description": "MiniMax M2.7 — MoE model for complex agent harnesses and productivity tasks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.3, + "output_per_1m": 1.2, + "cache_read_per_1m": 0.06 + } + }, + "minimax-m2p5": { + "provider": "fireworks", + "model_type": "llm", + "description": "MiniMax M2.5 — fast coding and agentic tool use at low cost", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.3, + "output_per_1m": 1.2, + "cache_read_per_1m": 0.06 + } + }, + "qwen3p6-plus": { + "provider": "fireworks", + "model_type": "llm", + "description": "Qwen 3.6 Plus — flagship multimodal model (Fireworks exclusive outside Alibaba)", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.325, + "output_per_1m": 1.95 + } + }, + "firefunction-v2": { + "provider": "fireworks", + "model_type": "llm", + "description": "FireFunction V2 — Fireworks function-calling optimized model", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.9, + "output_per_1m": 0.9 + } + }, + "google-speech-v2": { + "provider": "google", + "model_type": "stt", + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.0015 + } + }, + "gemini-2.5-pro-stt": { + "provider": "google", + "model_type": "stt", + "description": "Gemini 2.5 Pro used for audio transcription via LiteLLM proxy (audio input -> text)", + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.0015 + } + }, + "gemini-2.5-flash-stt": { + "provider": "google", + "model_type": "stt", + "description": "Gemini 2.5 Flash used for audio transcription via LiteLLM proxy (fast, cost-efficient)", + "featured": true, + "featured_rank": 6, + "highlights": [ + "Fast transcription", + "Cost-efficient", + "Gemini 2.5" + ], + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.0015 + } + }, + "gemini-2.5-flash-lite-stt": { + "provider": "google", + "model_type": "stt", + "description": "Gemini 2.5 Flash Lite used for audio transcription via LiteLLM proxy (lowest cost)", + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.00042 + } + }, + "gemini-2.5-pro": { + "provider": "google", + "model_type": "llm", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.25, + "output_per_1m": 10.0, + "cache_read_per_1m": 0.125 + } + }, + "gemini-2.5-flash": { + "provider": "google", + "model_type": "llm", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.3, + "output_per_1m": 2.5, + "cache_read_per_1m": 0.03, + "reasoning_per_1m": 2.5 + } + }, + "gemini-2.5-flash-lite": { + "provider": "google", + "model_type": "llm", + "description": "Gemini 2.5 Flash Lite — lowest-latency 2.5 model; ideal for diarisation, classification, and other low-reasoning multimodal tasks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.1, + "output_per_1m": 0.4, + "cache_read_per_1m": 0.01, + "reasoning_per_1m": 0.4 + } + }, + "gemini-3-pro-preview": { + "provider": "google", + "model_type": "llm", + "description": "Gemini 3 Pro — advanced reasoning and multimodal understanding", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.0, + "output_per_1m": 12.0, + "cache_read_per_1m": 0.2 + } + }, + "gemini-3-flash-preview": { + "provider": "google", + "model_type": "llm", + "description": "Gemini 3 Flash — frontier-class performance at lower cost", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.5, + "output_per_1m": 3.0, + "cache_read_per_1m": 0.05, + "reasoning_per_1m": 3.0 + } + }, + "gemini-3.1-pro-preview": { + "provider": "google", + "model_type": "llm", + "description": "Gemini 3.1 Pro — latest reasoning model with 1M context", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.0, + "output_per_1m": 12.0, + "cache_read_per_1m": 0.2 + } + }, + "gemini-3.5-flash": { + "provider": "google", + "model_type": "llm", + "description": "Gemini 3.5 Flash — GA flagship Flash model for agentic coding and long-horizon tasks", + "featured": true, + "featured_rank": 7, + "highlights": [ + "GA stable", + "1M context", + "Agentic coding" + ], + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.5, + "output_per_1m": 9.0, + "cache_read_per_1m": 0.15, + "reasoning_per_1m": 9.0 + } + }, + "gemini-3.5-flash-lite": { + "provider": "google", + "model_type": "llm", + "description": "Gemini 3.5 Flash Lite — fastest, lowest-cost 3.5 model for high-throughput extraction, classification, and subagent workflows", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.3, + "output_per_1m": 2.5, + "cache_read_per_1m": 0.03, + "reasoning_per_1m": 2.5 + } + }, + "gemini-3.1-flash-lite": { + "provider": "google", + "model_type": "llm", + "description": "Gemini 3.1 Flash Lite — cost-efficient stable model for high-volume lightweight tasks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.25, + "output_per_1m": 1.5, + "cache_read_per_1m": 0.025, + "reasoning_per_1m": 1.5 + } + }, + "azure-speech-v1": { + "provider": "azure", + "model_type": "stt", + "description": "Azure Speech-to-text (batch and realtime)", + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.01668 + } + }, + "azure-openai-gpt4": { + "provider": "azure", + "model_type": "llm", + "description": "Legacy alias — maps to gpt-4 deployment; prefer azure-gpt-5-mini if that is your deployment name", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 30.0, + "output_per_1m": 60.0 + } + }, + "azure-gpt-4o": { + "provider": "azure", + "model_type": "llm", + "description": "Azure OpenAI GPT-4o — catalog key maps to deployment name gpt-4o", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.5, + "output_per_1m": 10.0, + "cache_read_per_1m": 1.25 + } + }, + "azure-gpt-4o-mini": { + "provider": "azure", + "model_type": "llm", + "description": "Azure OpenAI GPT-4o mini", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.165, + "output_per_1m": 0.66, + "cache_read_per_1m": 0.075 + } + }, + "azure-gpt-4.1": { + "provider": "azure", + "model_type": "llm", + "description": "Azure OpenAI GPT-4.1", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.0, + "output_per_1m": 8.0, + "cache_read_per_1m": 0.5 + } + }, + "azure-gpt-4.1-mini": { + "provider": "azure", + "model_type": "llm", + "description": "Azure OpenAI GPT-4.1 mini", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.4, + "output_per_1m": 1.6, + "cache_read_per_1m": 0.1 + } + }, + "azure-gpt-4.1-nano": { + "provider": "azure", + "model_type": "llm", + "description": "Azure OpenAI GPT-4.1 nano", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.1, + "output_per_1m": 0.4, + "cache_read_per_1m": 0.025 + } + }, + "azure-gpt-5": { + "provider": "azure", + "model_type": "llm", + "description": "Azure OpenAI GPT-5", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.25, + "output_per_1m": 10.0, + "cache_read_per_1m": 0.125 + } + }, + "azure-gpt-5-mini": { + "provider": "azure", + "model_type": "llm", + "description": "Azure OpenAI GPT-5 mini — pick this if your deployment is named gpt-5-mini", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.25, + "output_per_1m": 2.0, + "cache_read_per_1m": 0.025 + } + }, + "azure-gpt-5-nano": { + "provider": "azure", + "model_type": "llm", + "description": "Azure OpenAI GPT-5 nano", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.05, + "output_per_1m": 0.4, + "cache_read_per_1m": 0.005 + } + }, + "azure-gpt-5.1": { + "provider": "azure", + "model_type": "llm", + "description": "Azure OpenAI GPT-5.1", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.25, + "output_per_1m": 10.0, + "cache_read_per_1m": 0.125 + } + }, + "azure-gpt-5.2": { + "provider": "azure", + "model_type": "llm", + "description": "Azure OpenAI GPT-5.2", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.75, + "output_per_1m": 14.0, + "cache_read_per_1m": 0.175 + } + }, + "azure-o3": { + "provider": "azure", + "model_type": "llm", + "description": "Azure OpenAI o3 reasoning model", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.0, + "output_per_1m": 8.0, + "cache_read_per_1m": 0.5 + } + }, + "azure-o3-mini": { + "provider": "azure", + "model_type": "llm", + "description": "Azure OpenAI o3 mini", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.1, + "output_per_1m": 4.4, + "cache_read_per_1m": 0.55 + } + }, + "azure-o4-mini": { + "provider": "azure", + "model_type": "llm", + "description": "Azure OpenAI o4 mini", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.1, + "output_per_1m": 4.4, + "cache_read_per_1m": 0.275 + } + }, + "azure-tts-v1": { + "provider": "azure", + "model_type": "tts", + "description": "Azure neural text-to-speech", + "pricing": { + "source": "litellm_import", + "usage_kind": "tts", + "tts_per_1m_characters": 15.0 + } + }, + "aws-transcribe": { + "provider": "aws", + "model_type": "stt", + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.006 + } + }, + "aws-bedrock-claude": { + "provider": "aws", + "model_type": "llm", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 3.0, + "output_per_1m": 15.0, + "cache_read_per_1m": 0.3, + "cache_write_per_1m": 3.75 + } + }, + "aws-polly": { + "provider": "aws", + "model_type": "tts", + "pricing": { + "source": "litellm_import", + "usage_kind": "tts", + "tts_per_1m_characters": 16.0 + } + }, + "deepgram-flux": { + "provider": "deepgram", + "model_type": "stt", + "description": "Conversational STT with integrated turn detection for voice agents", + "featured": true, + "featured_rank": 4, + "highlights": [ + "Turn detection", + "Voice agents", + "Low latency" + ], + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.00432 + } + }, + "deepgram-nova-3": { + "provider": "deepgram", + "model_type": "stt", + "description": "Highest-performing general-purpose ASR model", + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.00432 + } + }, + "deepgram-nova-3-general": { + "provider": "deepgram", + "model_type": "stt", + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.00432 + } + }, + "deepgram-nova-3-general-preview-12-2025": { + "provider": "deepgram", + "model_type": "stt", + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.00432 + } + }, + "deepgram-nova-2": { + "provider": "deepgram", + "model_type": "stt", + "description": "Available for languages not yet supported by Nova-3", + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.00432 + } + }, + "pulse-v4": { + "provider": "smallest", + "model_type": "stt", + "description": "Smallest Pulse v4 speech-to-text (batch and realtime)", + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.006 + } + }, + "cartesia-sonic-3": { + "provider": "cartesia", + "model_type": "tts", + "description": "Flagship streaming TTS with laughter, volume/speed/emotion controls, 42 languages", + "featured": true, + "featured_rank": 1, + "highlights": [ + "Ultra-low latency", + "42 languages", + "Emotion controls" + ], + "pricing": { + "source": "cartesia.ai credits ~$50/1M chars (Pro tier, 1 credit/char)", + "usage_kind": "tts", + "tts_per_1m_characters": 50.0 + } + }, + "cartesia-sonic-3-mini": { + "provider": "cartesia", + "model_type": "tts", + "pricing": { + "source": "cartesia.ai ~$40/1M chars (mid-tier estimate)", + "usage_kind": "tts", + "tts_per_1m_characters": 40.0 + } + }, + "cartesia-sonic-3-nano": { + "provider": "cartesia", + "model_type": "tts", + "pricing": { + "source": "cartesia.ai Scale ~$37/1M chars", + "usage_kind": "tts", + "tts_per_1m_characters": 37.0 + } + }, + "eleven_v3": { + "provider": "elevenlabs", + "model_type": "tts", + "description": "Human-like and expressive speech generation", + "languages": "70+ languages", + "pricing": { + "source": "litellm_import", + "usage_kind": "tts", + "tts_per_1m_characters": 180.0 + } + }, + "eleven_ttv_v3": { + "provider": "elevenlabs", + "model_type": "tts", + "description": "Human-like and expressive voice design model (Text to Voice)", + "languages": "70+ languages", + "pricing": { + "source": "litellm_import", + "usage_kind": "tts", + "tts_per_1m_characters": 180.0 + } + }, + "eleven_multilingual_v2": { + "provider": "elevenlabs", + "model_type": "tts", + "description": "Most lifelike model with rich emotional expression", + "languages": "en, ja, zh, de, hi, fr, ko, pt, it, es, id, nl, tr, fil, pl, sv, bg, ro, ar, cs, el, fi, hr, ms, sk, da, ta, uk, ru", + "pricing": { + "source": "litellm_import", + "usage_kind": "tts", + "tts_per_1m_characters": 180.0 + } + }, + "eleven_flash_v2_5": { + "provider": "elevenlabs", + "model_type": "tts", + "description": "Ultra-fast model optimized for real-time use (~75ms)", + "languages": "en, ja, zh, de, hi, fr, ko, pt, it, es, id, nl, tr, fil, pl, sv, bg, ro, ar, cs, el, fi, hr, ms, sk, da, ta, uk, ru, hu, no, vi", + "featured": true, + "featured_rank": 2, + "highlights": [ + "~75ms latency", + "Real-time", + "Multilingual" + ], + "pricing": { + "source": "litellm_import", + "usage_kind": "tts", + "tts_per_1m_characters": 180.0 + } + }, + "eleven_turbo_v2_5": { + "provider": "elevenlabs", + "model_type": "tts", + "description": "High quality, low-latency model (~250ms-300ms)", + "languages": "en, ja, zh, de, hi, fr, ko, pt, it, es, id, nl, tr, fil, pl, sv, bg, ro, ar, cs, el, fi, hr, ms, sk, da, ta, uk, ru, hu, no, vi", + "pricing": { + "source": "litellm_import", + "usage_kind": "tts", + "tts_per_1m_characters": 180.0 + } + }, + "eleven_multilingual_sts_v2": { + "provider": "elevenlabs", + "model_type": "sts", + "description": "State-of-the-art multilingual voice changer model (Speech to Speech)", + "languages": "en, ja, zh, de, hi, fr, ko, pt, it, es, id, nl, tr, fil, pl, sv, bg, ro, ar, cs, el, fi, hr, ms, sk, da, ta, uk, ru", + "pricing": { + "source": "litellm_import", + "usage_kind": "tts", + "tts_per_1m_characters": 180.0 + } + }, + "eleven_multilingual_ttv_v2": { + "provider": "elevenlabs", + "model_type": "tts", + "description": "State-of-the-art multilingual voice designer model (Text to Voice)", + "languages": "en, ja, zh, de, hi, fr, ko, pt, it, es, id, nl, tr, fil, pl, sv, bg, ro, ar, cs, el, fi, hr, ms, sk, da, ta, uk, ru", + "pricing": { + "source": "litellm_import", + "usage_kind": "tts", + "tts_per_1m_characters": 180.0 + } + }, + "eleven_english_sts_v2": { + "provider": "elevenlabs", + "model_type": "sts", + "description": "English-only voice changer model (Speech to Speech)", + "languages": "en", + "pricing": { + "source": "litellm_import", + "usage_kind": "tts", + "tts_per_1m_characters": 180.0 + } + }, + "scribe_v2_realtime": { + "provider": "elevenlabs", + "model_type": "stt", + "description": "Real-time speech recognition model", + "languages": "90+ languages", + "featured": true, + "featured_rank": 5, + "highlights": [ + "Real-time STT", + "90+ languages", + "Streaming" + ], + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.00366 + } + }, + "scribe_v2": { + "provider": "elevenlabs", + "model_type": "stt", + "description": "Most accurate transcription model with keyterm prompting and entity detection", + "languages": "90+ languages", + "pricing": { + "source": "litellm_import", + "usage_kind": "stt", + "audio_per_minute": 0.00366 + } + }, + "eleven_text_to_sound_v2": { + "provider": "elevenlabs", + "model_type": "sound_effects", + "description": "Sound effects generation from text prompts", + "pricing": { + "source": "litellm_import", + "usage_kind": "tts", + "tts_per_1m_characters": 180.0 + } + }, + "music_v1": { + "provider": "elevenlabs", + "model_type": "music", + "description": "Studio-grade music generation from text prompts", + "languages": "en, es, de, ja, and more", + "pricing": { + "source": "litellm_import", + "usage_kind": "tts", + "tts_per_1m_characters": 180.0 + } + }, + "murf-falcon": { + "provider": "murf", + "model_type": "tts", + "description": "Murf FALCON – ultra-low latency streaming TTS", + "languages": "en, es, de, fr, hi, ja, ko, pt, it, zh, and 10+ more", + "voices_source_file": "murf_falcon_voices.json", + "pricing": { + "source": "murf.ai API $0.01/1K chars", + "usage_kind": "tts", + "tts_per_1m_characters": 10.0 + } + }, + "murf-gen2": { + "provider": "murf", + "model_type": "tts", + "description": "Murf GEN2 – high-quality, natural-sounding TTS", + "languages": "en, es, de, fr, hi, ja, ko, pt, it, zh, and 10+ more", + "voices_source_file": "murf_gen2_voices.json", + "pricing": { + "source": "murf.ai API $0.03/1K chars", + "usage_kind": "tts", + "tts_per_1m_characters": 30.0 + } + }, + "voicemaker-ai3": { + "provider": "voicemaker", + "model_type": "tts", + "description": "VoiceMaker AI3 neural TTS voices", + "languages": "en-US, en-GB, en-AU, es-ES, fr-FR, de-DE, ja-JP, zh-CN", + "voices_source_file": "voicemaker_voices.json", + "pricing": { + "source": "developer.voicemaker.in $25/1M chars × 1× (AI3)", + "usage_kind": "tts", + "tts_per_1m_characters": 25.0 + } + }, + "voicemaker-proplus": { + "provider": "voicemaker", + "model_type": "tts", + "description": "VoiceMaker ProPlus expressive multilingual voices", + "languages": "multi-lang + regional accents", + "voices_source_file": "voicemaker_voices.json", + "pricing": { + "source": "developer.voicemaker.in $25/1M chars × 2× (ProPlus Turbo)", + "usage_kind": "tts", + "tts_per_1m_characters": 50.0 + } + }, + "lightning-v3.1": { + "provider": "smallest", + "model_type": "tts", + "description": "Smallest Lightning v3.1 low-latency neural TTS", + "pricing": { + "source": "smallest.ai $0.175/10K chars", + "usage_kind": "tts", + "tts_per_1m_characters": 17.5 + } + }, + "sarvam-30b": { + "provider": "sarvam", + "model_type": "llm", + "description": "Sarvam 30B — balanced Indian-language + English chat model (64K context)", + "pricing": { + "source": "sarvam.ai ₹2.5 / ₹1.5 / ₹10 per 1M tokens @ 83 INR/USD", + "usage_kind": "llm", + "input_per_1m": 0.03012, + "output_per_1m": 0.120482, + "cache_read_per_1m": 0.018072 + } + }, + "sarvam-105b": { + "provider": "sarvam", + "model_type": "llm", + "description": "Sarvam 105B — flagship MoE for complex reasoning, coding, and agentic workflows (128K context)", + "pricing": { + "source": "sarvam.ai ₹4 / ₹2.5 / ₹16 per 1M tokens @ 83 INR/USD", + "usage_kind": "llm", + "input_per_1m": 0.048193, + "output_per_1m": 0.192771, + "cache_read_per_1m": 0.03012 + } + }, + "saaras:v3": { + "provider": "sarvam", + "model_type": "stt", + "description": "State-of-the-art Sarvam STT model with transcribe/translate modes", + "pricing": { + "source": "sarvam.ai ₹30/hour audio @ 83 INR/USD", + "usage_kind": "stt", + "audio_per_minute": 0.006 + } + }, + "bulbul:v3": { + "provider": "sarvam", + "model_type": "tts", + "description": "High quality Indian language TTS", + "voices": [ + { + "id": "aditya", + "name": "Aditya", + "gender": "Male" + }, + { + "id": "ritu", + "name": "Ritu", + "gender": "Female" + }, + { + "id": "ashutosh", + "name": "Ashutosh", + "gender": "Male" + }, + { + "id": "priya", + "name": "Priya", + "gender": "Female" + }, + { + "id": "neha", + "name": "Neha", + "gender": "Female" + }, + { + "id": "rahul", + "name": "Rahul", + "gender": "Male" + }, + { + "id": "pooja", + "name": "Pooja", + "gender": "Female" + }, + { + "id": "rohan", + "name": "Rohan", + "gender": "Male" + }, + { + "id": "simran", + "name": "Simran", + "gender": "Female" + }, + { + "id": "kavya", + "name": "Kavya", + "gender": "Female" + }, + { + "id": "amit", + "name": "Amit", + "gender": "Male" + }, + { + "id": "dev", + "name": "Dev", + "gender": "Male" + }, + { + "id": "ishita", + "name": "Ishita", + "gender": "Female" + }, + { + "id": "shreya", + "name": "Shreya", + "gender": "Female" + }, + { + "id": "ratan", + "name": "Ratan", + "gender": "Male" + }, + { + "id": "varun", + "name": "Varun", + "gender": "Male" + }, + { + "id": "manan", + "name": "Manan", + "gender": "Male" + }, + { + "id": "sumit", + "name": "Sumit", + "gender": "Male" + }, + { + "id": "roopa", + "name": "Roopa", + "gender": "Female" + }, + { + "id": "kabir", + "name": "Kabir", + "gender": "Male" + }, + { + "id": "aayan", + "name": "Aayan", + "gender": "Male" + }, + { + "id": "shubh", + "name": "Shubh", + "gender": "Male" + }, + { + "id": "advait", + "name": "Advait", + "gender": "Male" + }, + { + "id": "amelia", + "name": "Amelia", + "gender": "Female" + }, + { + "id": "sophia", + "name": "Sophia", + "gender": "Female" + }, + { + "id": "anand", + "name": "Anand", + "gender": "Male" + }, + { + "id": "tanya", + "name": "Tanya", + "gender": "Female" + }, + { + "id": "tarun", + "name": "Tarun", + "gender": "Male" + }, + { + "id": "sunny", + "name": "Sunny", + "gender": "Male" + }, + { + "id": "mani", + "name": "Mani", + "gender": "Male" + }, + { + "id": "gokul", + "name": "Gokul", + "gender": "Male" + }, + { + "id": "vijay", + "name": "Vijay", + "gender": "Male" + }, + { + "id": "shruti", + "name": "Shruti", + "gender": "Female" + }, + { + "id": "suhani", + "name": "Suhani", + "gender": "Female" + }, + { + "id": "mohit", + "name": "Mohit", + "gender": "Male" + }, + { + "id": "kavitha", + "name": "Kavitha", + "gender": "Female" + }, + { + "id": "rehan", + "name": "Rehan", + "gender": "Male" + }, + { + "id": "soham", + "name": "Soham", + "gender": "Male" + }, + { + "id": "rupali", + "name": "Rupali", + "gender": "Female" + } + ], + "pricing": { + "source": "sarvam.ai ₹30/10K chars @ 83 INR/USD", + "usage_kind": "tts", + "tts_per_1m_characters": 36.144578 + } + }, + "voice-agent-call": { + "provider": "telephony", + "model_type": "llm", + "description": "External voice-agent call duration (Vapi, Retell, ElevenLabs, Smallest); billed per minute", + "pricing": { + "source": "manual", + "usage_kind": "llm", + "audio_per_minute": 0.05 + } + }, + "unknown": { + "provider": "internal", + "model_type": "llm", + "description": "Fallback when model name is missing from usage records", + "pricing": { + "source": "manual", + "usage_kind": "llm", + "input_per_1m": 0, + "output_per_1m": 0 + } + } +} diff --git a/app/migrations/069_usage_pricing.py b/app/migrations/069_usage_pricing.py new file mode 100644 index 00000000..0d8f8a76 --- /dev/null +++ b/app/migrations/069_usage_pricing.py @@ -0,0 +1,190 @@ +"""Migration: usage pricing catalog, org overrides, and cost columns on rollups.""" + +from __future__ import annotations + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Add model_pricing_catalog, org_model_pricing_overrides, usage_pricing_mode, " + "and cost columns on llm_usage_daily" +) + + +def _table_exists(db: Session, table: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.tables + WHERE table_name = :table_name + """ + ), + {"table_name": table}, + ).first() + is not None + ) + + +def _column_exists(db: Session, table: str, column: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table, "column_name": column}, + ).first() + is not None + ) + + +def _seed_pricing_catalog(db: Session) -> int: + from app.services.usage.pricing import DEFAULT_CATALOG_EFFECTIVE_FROM, seed_pricing_catalog + + return seed_pricing_catalog(db, effective_from=DEFAULT_CATALOG_EFFECTIVE_FROM) + + +def upgrade(db: Session): + if not _column_exists(db, "organizations", "usage_pricing_mode"): + db.execute( + text( + """ + ALTER TABLE organizations + ADD COLUMN usage_pricing_mode VARCHAR(32) NOT NULL DEFAULT 'platform_managed' + """ + ) + ) + print("Added organizations.usage_pricing_mode") + + if not _table_exists(db, "model_pricing_catalog"): + db.execute( + text( + """ + CREATE TABLE model_pricing_catalog ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + model VARCHAR(255) NOT NULL, + usage_kind VARCHAR(16) NOT NULL DEFAULT 'llm', + effective_from DATE NOT NULL DEFAULT CURRENT_DATE, + effective_to DATE, + input_micro_usd_per_million BIGINT NOT NULL DEFAULT 0, + output_micro_usd_per_million BIGINT NOT NULL DEFAULT 0, + cache_read_micro_usd_per_million BIGINT NOT NULL DEFAULT 0, + cache_creation_micro_usd_per_million BIGINT NOT NULL DEFAULT 0, + reasoning_micro_usd_per_million BIGINT NOT NULL DEFAULT 0, + audio_micro_usd_per_second BIGINT NOT NULL DEFAULT 0, + tts_micro_usd_per_million_chars BIGINT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT uq_model_pricing_catalog_model_kind_from + UNIQUE (model, usage_kind, effective_from) + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX ix_model_pricing_catalog_lookup + ON model_pricing_catalog (model, usage_kind, effective_from DESC) + """ + ) + ) + print("Created model_pricing_catalog") + + if not _table_exists(db, "org_model_pricing_overrides"): + db.execute( + text( + """ + CREATE TABLE org_model_pricing_overrides ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id UUID NOT NULL + REFERENCES organizations(id) ON DELETE CASCADE, + model VARCHAR(255) NOT NULL, + usage_kind VARCHAR(16) NOT NULL DEFAULT 'llm', + effective_from DATE NOT NULL DEFAULT CURRENT_DATE, + effective_to DATE, + input_micro_usd_per_million BIGINT, + output_micro_usd_per_million BIGINT, + cache_read_micro_usd_per_million BIGINT, + cache_creation_micro_usd_per_million BIGINT, + reasoning_micro_usd_per_million BIGINT, + audio_micro_usd_per_second BIGINT, + tts_micro_usd_per_million_chars BIGINT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT uq_org_model_pricing_override + UNIQUE (organization_id, model, usage_kind, effective_from) + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX ix_org_model_pricing_overrides_lookup + ON org_model_pricing_overrides ( + organization_id, model, usage_kind, effective_from DESC + ) + """ + ) + ) + print("Created org_model_pricing_overrides") + + cost_columns = [ + ("input_cost_micro_usd", "BIGINT NOT NULL DEFAULT 0"), + ("output_cost_micro_usd", "BIGINT NOT NULL DEFAULT 0"), + ("cache_read_cost_micro_usd", "BIGINT NOT NULL DEFAULT 0"), + ("cache_creation_cost_micro_usd", "BIGINT NOT NULL DEFAULT 0"), + ("reasoning_cost_micro_usd", "BIGINT NOT NULL DEFAULT 0"), + ("audio_cost_micro_usd", "BIGINT NOT NULL DEFAULT 0"), + ("tts_cost_micro_usd", "BIGINT NOT NULL DEFAULT 0"), + ("total_cost_micro_usd", "BIGINT NOT NULL DEFAULT 0"), + ("pricing_rate_source", "VARCHAR(16)"), + ("pricing_rate_id", "UUID"), + ] + if _table_exists(db, "llm_usage_daily"): + for column, col_type in cost_columns: + if not _column_exists(db, "llm_usage_daily", column): + db.execute( + text( + f"ALTER TABLE llm_usage_daily ADD COLUMN {column} {col_type}" + ) + ) + print("Added cost columns to llm_usage_daily") + + db.commit() + + if _table_exists(db, "model_pricing_catalog"): + seeded = _seed_pricing_catalog(db) + if seeded: + print(f"Seeded {seeded} model pricing catalog row(s)") + db.commit() + + +def downgrade(db: Session): + if _table_exists(db, "llm_usage_daily"): + for column in ( + "input_cost_micro_usd", + "output_cost_micro_usd", + "cache_read_cost_micro_usd", + "cache_creation_cost_micro_usd", + "reasoning_cost_micro_usd", + "audio_cost_micro_usd", + "tts_cost_micro_usd", + "total_cost_micro_usd", + "pricing_rate_source", + "pricing_rate_id", + ): + if _column_exists(db, "llm_usage_daily", column): + db.execute(text(f"ALTER TABLE llm_usage_daily DROP COLUMN {column}")) + + db.execute(text("DROP TABLE IF EXISTS org_model_pricing_overrides")) + db.execute(text("DROP TABLE IF EXISTS model_pricing_catalog")) + + if _column_exists(db, "organizations", "usage_pricing_mode"): + db.execute(text("ALTER TABLE organizations DROP COLUMN usage_pricing_mode")) + + db.commit() diff --git a/app/migrations/070_usage_margin_multiplier.py b/app/migrations/070_usage_margin_multiplier.py new file mode 100644 index 00000000..9a6c3e03 --- /dev/null +++ b/app/migrations/070_usage_margin_multiplier.py @@ -0,0 +1,45 @@ +"""Migration: org-level usage margin multiplier for priced rollups.""" + +from __future__ import annotations + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add organizations.usage_margin_multiplier for usage cost markup" + + +def _column_exists(db: Session, table: str, column: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table, "column_name": column}, + ).first() + is not None + ) + + +def upgrade(db: Session): + if not _column_exists(db, "organizations", "usage_margin_multiplier"): + db.execute( + text( + """ + ALTER TABLE organizations + ADD COLUMN usage_margin_multiplier DOUBLE PRECISION NOT NULL DEFAULT 1.0 + """ + ) + ) + print("Added organizations.usage_margin_multiplier") + db.commit() + + +def downgrade(db: Session): + if _column_exists(db, "organizations", "usage_margin_multiplier"): + db.execute( + text("ALTER TABLE organizations DROP COLUMN usage_margin_multiplier") + ) + db.commit() diff --git a/app/migrations/071_reseed_pricing_catalog.py b/app/migrations/071_reseed_pricing_catalog.py new file mode 100644 index 00000000..ad17a360 --- /dev/null +++ b/app/migrations/071_reseed_pricing_catalog.py @@ -0,0 +1,37 @@ +"""Migration: re-seed model_pricing_rates from models.json after effective_from fix.""" + +from __future__ import annotations + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.services.usage.pricing import DEFAULT_RATES_EFFECTIVE_FROM, seed_pricing_rates + +description = "Re-seed model_pricing_rates from models.json pricing blocks" + + +def upgrade(db: Session): + table = "model_pricing_rates" + exists = db.execute( + text("SELECT to_regclass('public.model_pricing_rates')") + ).scalar() + if not exists: + table = "model_pricing_catalog" + db.execute( + text( + f""" + UPDATE {table} + SET effective_from = CAST(:effective_from AS date) + WHERE effective_from > CAST(:effective_from AS date) + """ + ), + {"effective_from": DEFAULT_RATES_EFFECTIVE_FROM.isoformat()}, + ) + seeded = seed_pricing_rates(db, effective_from=DEFAULT_RATES_EFFECTIVE_FROM) + db.commit() + if seeded: + print(f"Re-seeded {seeded} model pricing rate row(s)") + + +def downgrade(db: Session): + pass diff --git a/app/migrations/072_usage_pricing_phase1.py b/app/migrations/072_usage_pricing_phase1.py new file mode 100644 index 00000000..68254deb --- /dev/null +++ b/app/migrations/072_usage_pricing_phase1.py @@ -0,0 +1,202 @@ +"""Migration: Phase 1 plan alignment — model_pricing_rates, buffer costs, strip extras.""" + +from __future__ import annotations + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Rename model_pricing_catalog to model_pricing_rates, add currency/source, " + "cost columns on usage_pending_buffer, drop margin/BYOK org columns" +) + + +def _table_exists(db: Session, table: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.tables + WHERE table_name = :table_name + """ + ), + {"table_name": table}, + ).first() + is not None + ) + + +def _column_exists(db: Session, table: str, column: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table, "column_name": column}, + ).first() + is not None + ) + + +def _ensure_model_pricing_rates(db: Session) -> None: + if _table_exists(db, "model_pricing_catalog") and not _table_exists( + db, "model_pricing_rates" + ): + db.execute( + text("ALTER TABLE model_pricing_catalog RENAME TO model_pricing_rates") + ) + db.execute( + text( + """ + ALTER INDEX IF EXISTS uq_model_pricing_catalog_model_kind_from + RENAME TO uq_model_pricing_rates_model_kind_from + """ + ) + ) + db.execute( + text( + """ + ALTER INDEX IF EXISTS ix_model_pricing_catalog_lookup + RENAME TO ix_model_pricing_rates_lookup + """ + ) + ) + print("Renamed model_pricing_catalog -> model_pricing_rates") + + if not _table_exists(db, "model_pricing_rates"): + db.execute( + text( + """ + CREATE TABLE model_pricing_rates ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + model VARCHAR(255) NOT NULL, + usage_kind VARCHAR(16) NOT NULL DEFAULT 'llm', + effective_from DATE NOT NULL DEFAULT CURRENT_DATE, + effective_to DATE, + currency VARCHAR(8) NOT NULL DEFAULT 'USD', + source VARCHAR(32) NOT NULL DEFAULT 'catalog', + input_micro_usd_per_million BIGINT NOT NULL DEFAULT 0, + output_micro_usd_per_million BIGINT NOT NULL DEFAULT 0, + cache_read_micro_usd_per_million BIGINT NOT NULL DEFAULT 0, + cache_creation_micro_usd_per_million BIGINT NOT NULL DEFAULT 0, + reasoning_micro_usd_per_million BIGINT NOT NULL DEFAULT 0, + audio_micro_usd_per_second BIGINT NOT NULL DEFAULT 0, + tts_micro_usd_per_million_chars BIGINT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT uq_model_pricing_rates_model_kind_from + UNIQUE (model, usage_kind, effective_from) + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX ix_model_pricing_rates_lookup + ON model_pricing_rates (model, usage_kind, effective_from DESC) + """ + ) + ) + print("Created model_pricing_rates") + + if not _column_exists(db, "model_pricing_rates", "currency"): + db.execute( + text( + """ + ALTER TABLE model_pricing_rates + ADD COLUMN currency VARCHAR(8) NOT NULL DEFAULT 'USD' + """ + ) + ) + if not _column_exists(db, "model_pricing_rates", "source"): + db.execute( + text( + """ + ALTER TABLE model_pricing_rates + ADD COLUMN source VARCHAR(32) NOT NULL DEFAULT 'catalog' + """ + ) + ) + + +def _add_buffer_cost_columns(db: Session) -> None: + if not _table_exists(db, "usage_pending_buffer"): + return + cost_columns = [ + ("input_cost_micro_usd", "BIGINT NOT NULL DEFAULT 0"), + ("output_cost_micro_usd", "BIGINT NOT NULL DEFAULT 0"), + ("cache_read_cost_micro_usd", "BIGINT NOT NULL DEFAULT 0"), + ("cache_creation_cost_micro_usd", "BIGINT NOT NULL DEFAULT 0"), + ("reasoning_cost_micro_usd", "BIGINT NOT NULL DEFAULT 0"), + ("audio_cost_micro_usd", "BIGINT NOT NULL DEFAULT 0"), + ("tts_cost_micro_usd", "BIGINT NOT NULL DEFAULT 0"), + ("total_cost_micro_usd", "BIGINT NOT NULL DEFAULT 0"), + ("pricing_rate_source", "VARCHAR(16)"), + ("pricing_rate_id", "UUID"), + ] + for column, col_type in cost_columns: + if not _column_exists(db, "usage_pending_buffer", column): + db.execute( + text( + f"ALTER TABLE usage_pending_buffer ADD COLUMN {column} {col_type}" + ) + ) + print("Ensured cost columns on usage_pending_buffer") + + +def _strip_org_extras(db: Session) -> None: + if _column_exists(db, "organizations", "usage_margin_multiplier"): + db.execute( + text("ALTER TABLE organizations DROP COLUMN usage_margin_multiplier") + ) + print("Dropped organizations.usage_margin_multiplier") + if _column_exists(db, "organizations", "usage_pricing_mode"): + db.execute(text("ALTER TABLE organizations DROP COLUMN usage_pricing_mode")) + print("Dropped organizations.usage_pricing_mode") + + +def upgrade(db: Session): + _ensure_model_pricing_rates(db) + _add_buffer_cost_columns(db) + _strip_org_extras(db) + db.commit() + + from app.services.usage.pricing import DEFAULT_RATES_EFFECTIVE_FROM, seed_pricing_rates + + seeded = seed_pricing_rates(db, effective_from=DEFAULT_RATES_EFFECTIVE_FROM) + db.commit() + if seeded: + print(f"Seeded {seeded} model_pricing_rates row(s)") + + +def downgrade(db: Session): + if _table_exists(db, "usage_pending_buffer"): + for column in ( + "input_cost_micro_usd", + "output_cost_micro_usd", + "cache_read_cost_micro_usd", + "cache_creation_cost_micro_usd", + "reasoning_cost_micro_usd", + "audio_cost_micro_usd", + "tts_cost_micro_usd", + "total_cost_micro_usd", + "pricing_rate_source", + "pricing_rate_id", + ): + if _column_exists(db, "usage_pending_buffer", column): + db.execute( + text(f"ALTER TABLE usage_pending_buffer DROP COLUMN {column}") + ) + + if _table_exists(db, "model_pricing_rates") and not _table_exists( + db, "model_pricing_catalog" + ): + db.execute( + text("ALTER TABLE model_pricing_rates RENAME TO model_pricing_catalog") + ) + + db.commit() diff --git a/app/migrations/073_usage_cost_recompute_jobs.py b/app/migrations/073_usage_cost_recompute_jobs.py new file mode 100644 index 00000000..55f18192 --- /dev/null +++ b/app/migrations/073_usage_cost_recompute_jobs.py @@ -0,0 +1,72 @@ +"""Migration: async usage cost recompute job tracking.""" + +from __future__ import annotations + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add usage_cost_recompute_jobs table for async cost backfill/recompute" + + +def _table_exists(db: Session, table: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.tables + WHERE table_name = :table_name + """ + ), + {"table_name": table}, + ).first() + is not None + ) + + +def upgrade(db: Session) -> None: + if _table_exists(db, "usage_cost_recompute_jobs"): + print("usage_cost_recompute_jobs already exists, skipping...") + return + + db.execute( + text( + """ + CREATE TABLE usage_cost_recompute_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id UUID NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + model VARCHAR(255), + usage_kind VARCHAR(16), + start_date DATE, + end_date DATE, + updated_rows BIGINT NOT NULL DEFAULT 0, + error_message TEXT, + celery_task_id VARCHAR(255), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_usage_cost_recompute_jobs_organization_id + ON usage_cost_recompute_jobs(organization_id) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_usage_cost_recompute_jobs_org_status + ON usage_cost_recompute_jobs(organization_id, status) + """ + ) + ) + print("Created usage_cost_recompute_jobs table") + + +def downgrade(db: Session) -> None: + db.execute(text("DROP TABLE IF EXISTS usage_cost_recompute_jobs")) diff --git a/app/migrations/074_pricing_rates_source_and_effective_from.py b/app/migrations/074_pricing_rates_source_and_effective_from.py new file mode 100644 index 00000000..6f78f8b4 --- /dev/null +++ b/app/migrations/074_pricing_rates_source_and_effective_from.py @@ -0,0 +1,81 @@ +"""Migration: widen pricing source column and normalize effective_from baseline.""" + +from __future__ import annotations + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.services.usage.pricing import DEFAULT_RATES_EFFECTIVE_FROM, seed_pricing_rates + +description = ( + "Widen model_pricing_rates.source to VARCHAR(255) and normalize effective_from " + "to 2020-01-01 baseline" +) + +_BASELINE = DEFAULT_RATES_EFFECTIVE_FROM.isoformat() + + +def _table_exists(db: Session, table: str) -> bool: + return ( + db.execute( + text("SELECT to_regclass(:table_name)"), + {"table_name": f"public.{table}"}, + ).scalar() + is not None + ) + + +def upgrade(db: Session) -> None: + table = "model_pricing_rates" + if not _table_exists(db, table): + table = "model_pricing_catalog" + if not _table_exists(db, table): + print("No pricing rates table found, skipping") + return + + db.execute( + text( + f""" + ALTER TABLE {table} + ALTER COLUMN source TYPE VARCHAR(255) + """ + ) + ) + print(f"Widened {table}.source to VARCHAR(255)") + + db.execute( + text( + f""" + DELETE FROM {table} newer + USING {table} baseline + WHERE newer.model = baseline.model + AND newer.usage_kind = baseline.usage_kind + AND newer.effective_from > CAST(:baseline AS date) + AND baseline.effective_from = CAST(:baseline AS date) + """ + ), + {"baseline": _BASELINE}, + ) + db.execute( + text( + f""" + UPDATE {table} + SET effective_from = CAST(:baseline AS date) + WHERE effective_from > CAST(:baseline AS date) + """ + ), + {"baseline": _BASELINE}, + ) + print(f"Normalized {table} effective_from to {_BASELINE}") + + seeded = seed_pricing_rates(db, effective_from=DEFAULT_RATES_EFFECTIVE_FROM) + from app.services.usage.pricing_cache import invalidate_all_pricing_cache + + invalidate_all_pricing_cache() + db.commit() + if seeded: + print(f"Re-seeded {seeded} pricing rate row(s) at {_BASELINE}") + + +def downgrade(db: Session) -> None: + pass diff --git a/app/models/database.py b/app/models/database.py index 00b99fdb..99fbbbba 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -2779,6 +2779,31 @@ class JudgeRun(Base): dataset = relationship("JudgeDataset", back_populates="runs") +class UsageCostRecomputeJob(Base): + """Async job tracking for retroactive usage cost recompute.""" + + __tablename__ = "usage_cost_recompute_jobs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + status = Column(String(32), nullable=False, default="pending", server_default="pending") + model = Column(String(255), nullable=True) + usage_kind = Column(String(16), nullable=True) + start_date = Column(Date, nullable=True) + end_date = Column(Date, nullable=True) + updated_rows = Column(BigInteger, nullable=False, default=0, server_default="0") + error_message = Column(String, nullable=True) + celery_task_id = Column(String(255), nullable=True, index=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + completed_at = Column(DateTime(timezone=True), nullable=True) + + class LLMUsageDaily(Base): """Daily LLM/STT usage rollups for org-scoped Usage reporting.""" @@ -2812,6 +2837,22 @@ class LLMUsageDaily(Base): audio_seconds = Column(BigInteger, nullable=False, default=0, server_default="0") tts_characters = Column(BigInteger, nullable=False, default=0, server_default="0") call_count = Column(BigInteger, nullable=False, default=0, server_default="0") + input_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") + output_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") + cache_read_cost_micro_usd = Column( + BigInteger, nullable=False, default=0, server_default="0" + ) + cache_creation_cost_micro_usd = Column( + BigInteger, nullable=False, default=0, server_default="0" + ) + reasoning_cost_micro_usd = Column( + BigInteger, nullable=False, default=0, server_default="0" + ) + audio_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") + tts_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") + total_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") + pricing_rate_source = Column(String(16), nullable=True) + pricing_rate_id = Column(UUID(as_uuid=True), nullable=True) created_at = Column(DateTime(timezone=True), server_default=func.now()) updated_at = Column( DateTime(timezone=True), server_default=func.now(), onupdate=func.now() diff --git a/app/services/ai/stt_clients/google.py b/app/services/ai/stt_clients/google.py index 2bc100a1..84caf40d 100644 --- a/app/services/ai/stt_clients/google.py +++ b/app/services/ai/stt_clients/google.py @@ -83,6 +83,28 @@ def _build_transcription_prompt(language: Optional[str]) -> str: ) +def _gemini_stt_usage_ctx(config_model: str): + """Merge Gemini multimodal STT tags into the active usage context.""" + from app.services.usage.context import LLMUsageContext, get_usage_context + + tags = { + "stt_backend": "gemini_multimodal", + "config_model": (config_model or "").strip(), + "usage_split": "llm_tokens_and_stt_seconds", + } + base = get_usage_context() + if base is None: + return None + return LLMUsageContext( + organization_id=base.organization_id, + workspace_id=base.workspace_id, + product_section=base.product_section, + resource_id=base.resource_id, + resource_type=base.resource_type, + extra={**(base.extra or {}), **tags}, + ) + + def transcribe_google( audio_file_path: str, model: str, @@ -158,10 +180,12 @@ def transcribe_google( from app.services.usage.normalize import normalize_llm_usage gemini_model = _strip_stt_suffix(model) + usage_ctx = _gemini_stt_usage_ctx(model) record_llm_usage( gemini_model, normalize_llm_usage(raw_response=response), organization_id=organization_id, + ctx=usage_ctx, ) from app.services.usage.llm_usage import probe_audio_seconds, record_stt_usage @@ -171,6 +195,7 @@ def transcribe_google( gemini_model, audio_seconds=audio_seconds, organization_id=organization_id, + ctx=usage_ctx, count_call=False, ) except Exception as exc: diff --git a/app/services/call_import_user_insights.py b/app/services/call_import_user_insights.py index bf723e6a..7484a34b 100644 --- a/app/services/call_import_user_insights.py +++ b/app/services/call_import_user_insights.py @@ -21,6 +21,8 @@ Metric, ModelProvider, ) +from app.services.usage.context import LLMUsageContext + from app.models.schemas import ( EvaluationUserInsightItem, EvaluationUserInsightsState, @@ -222,17 +224,28 @@ def _call_llm( *, temperature: float, max_tokens: int, + usage_ctx: Optional[LLMUsageContext] = None, ) -> str: - result = llm_service.generate_response( - messages=messages, - llm_provider=provider, - llm_model=model, - organization_id=organization_id, - db=db, - temperature=temperature, - max_tokens=max_tokens, - ) - return str(result.get("text") or "") + from app.services.usage.context import get_usage_context, llm_usage_context + + effective_ctx = usage_ctx or get_usage_context() + + def _run() -> str: + result = llm_service.generate_response( + messages=messages, + llm_provider=provider, + llm_model=model, + organization_id=organization_id, + db=db, + temperature=temperature, + max_tokens=max_tokens, + ) + return str(result.get("text") or "") + + if effective_ctx is not None: + with llm_usage_context(effective_ctx): + return _run() + return _run() def run_extraction_batch( diff --git a/app/services/usage/llm_usage.py b/app/services/usage/llm_usage.py index b992b9e5..0fdb48f1 100644 --- a/app/services/usage/llm_usage.py +++ b/app/services/usage/llm_usage.py @@ -240,14 +240,18 @@ def _buffer_to_postgres( logger.warning("usage postgres fallback unavailable: {}", exc) return + usage_date = bucket["usage_date"] + if isinstance(usage_date, str): + usage_date = date.fromisoformat(usage_date) + usage_kind = bucket.get("usage_kind") or USAGE_KIND_LLM params = { "organization_id": str(organization_id), "workspace_id": str(bucket["workspace_id"]) if bucket.get("workspace_id") else None, "product_section": bucket["product_section"], "model": bucket["model"], "context": json.dumps(bucket.get("context") or {}), - "usage_date": bucket["usage_date"].isoformat(), - "usage_kind": bucket.get("usage_kind") or USAGE_KIND_LLM, + "usage_date": usage_date.isoformat(), + "usage_kind": usage_kind, "prompt_tokens": int(deltas.get("prompt_tokens", 0)), "completion_tokens": int(deltas.get("completion_tokens", 0)), "cache_read_tokens": int(deltas.get("cache_read_tokens", 0)), @@ -259,6 +263,18 @@ def _buffer_to_postgres( } db = SessionLocal() try: + from app.services.usage.pricing import cost_fields_from_deltas + + params.update( + cost_fields_from_deltas( + deltas, + organization_id=organization_id, + model=bucket["model"], + usage_kind=usage_kind, + usage_date=usage_date, + db=db, + ) + ) db.execute( text( """ @@ -267,7 +283,12 @@ def _buffer_to_postgres( context, usage_date, usage_kind, prompt_tokens, completion_tokens, cache_read_tokens, cache_creation_tokens, reasoning_tokens, audio_seconds, - tts_characters, call_count, created_at + tts_characters, call_count, + input_cost_micro_usd, output_cost_micro_usd, + cache_read_cost_micro_usd, cache_creation_cost_micro_usd, + reasoning_cost_micro_usd, audio_cost_micro_usd, tts_cost_micro_usd, + total_cost_micro_usd, pricing_rate_source, pricing_rate_id, + created_at ) VALUES ( gen_random_uuid(), CAST(:organization_id AS uuid), CAST(:workspace_id AS uuid), :product_section, :model, @@ -275,7 +296,13 @@ def _buffer_to_postgres( CAST(:usage_date AS date), :usage_kind, :prompt_tokens, :completion_tokens, :cache_read_tokens, :cache_creation_tokens, :reasoning_tokens, :audio_seconds, - :tts_characters, :call_count, now() + :tts_characters, :call_count, + :input_cost_micro_usd, :output_cost_micro_usd, + :cache_read_cost_micro_usd, :cache_creation_cost_micro_usd, + :reasoning_cost_micro_usd, :audio_cost_micro_usd, :tts_cost_micro_usd, + :total_cost_micro_usd, :pricing_rate_source, + CAST(:pricing_rate_id AS uuid), + now() ) """ ), @@ -715,6 +742,8 @@ def _upsert_bucket( organization_id: UUID, bucket: Dict[str, Any], deltas: Dict[str, int], + *, + pricing_resolver: Any = None, ) -> None: context = bucket.get("context") or {} params = { @@ -772,64 +801,75 @@ def _upsert_bucket( text(f"UPDATE llm_usage_daily SET {update_set} {exact_context_where}"), params, ) - if result.rowcount: - return - - result = db.execute( - text(f"UPDATE llm_usage_daily SET {update_set} {legacy_context_where}"), - params, - ) - if result.rowcount: - return - - db.execute(text("SAVEPOINT llm_usage_bucket_insert")) - try: - db.execute( - text( - """ - INSERT INTO llm_usage_daily ( - id, organization_id, workspace_id, product_section, model, - context, usage_date, usage_kind, - prompt_tokens, completion_tokens, cache_read_tokens, - cache_creation_tokens, reasoning_tokens, audio_seconds, - tts_characters, call_count, - created_at, updated_at - ) VALUES ( - gen_random_uuid(), CAST(:organization_id AS uuid), - CAST(:workspace_id AS uuid), :product_section, :model, - CAST(:context AS jsonb), CAST(:usage_date AS date), - :usage_kind, - :prompt_tokens, :completion_tokens, :cache_read_tokens, - :cache_creation_tokens, :reasoning_tokens, :audio_seconds, - :tts_characters, :call_count, - now(), now() - ) - """ - ), - params, - ) - db.execute(text("RELEASE SAVEPOINT llm_usage_bucket_insert")) - except IntegrityError as exc: - if not _is_unique_violation(exc): - db.execute(text("ROLLBACK TO SAVEPOINT llm_usage_bucket_insert")) - db.execute(text("RELEASE SAVEPOINT llm_usage_bucket_insert")) - raise - db.execute(text("ROLLBACK TO SAVEPOINT llm_usage_bucket_insert")) + if not result.rowcount: result = db.execute( text(f"UPDATE llm_usage_daily SET {update_set} {legacy_context_where}"), params, ) if not result.rowcount: - result = db.execute( - text(f"UPDATE llm_usage_daily SET {update_set} {exact_context_where}"), - params, - ) - db.execute(text("RELEASE SAVEPOINT llm_usage_bucket_insert")) - if not result.rowcount: - logger.warning( - "llm usage upsert unique conflict but no matching bucket for org {}", - organization_id, - ) + db.execute(text("SAVEPOINT llm_usage_bucket_insert")) + try: + db.execute( + text( + """ + INSERT INTO llm_usage_daily ( + id, organization_id, workspace_id, product_section, model, + context, usage_date, usage_kind, + prompt_tokens, completion_tokens, cache_read_tokens, + cache_creation_tokens, reasoning_tokens, audio_seconds, + tts_characters, call_count, + created_at, updated_at + ) VALUES ( + gen_random_uuid(), CAST(:organization_id AS uuid), + CAST(:workspace_id AS uuid), :product_section, :model, + CAST(:context AS jsonb), CAST(:usage_date AS date), + :usage_kind, + :prompt_tokens, :completion_tokens, :cache_read_tokens, + :cache_creation_tokens, :reasoning_tokens, :audio_seconds, + :tts_characters, :call_count, + now(), now() + ) + """ + ), + params, + ) + db.execute(text("RELEASE SAVEPOINT llm_usage_bucket_insert")) + except IntegrityError as exc: + if not _is_unique_violation(exc): + db.execute(text("ROLLBACK TO SAVEPOINT llm_usage_bucket_insert")) + db.execute(text("RELEASE SAVEPOINT llm_usage_bucket_insert")) + raise + db.execute(text("ROLLBACK TO SAVEPOINT llm_usage_bucket_insert")) + result = db.execute( + text(f"UPDATE llm_usage_daily SET {update_set} {legacy_context_where}"), + params, + ) + if not result.rowcount: + result = db.execute( + text(f"UPDATE llm_usage_daily SET {update_set} {exact_context_where}"), + params, + ) + db.execute(text("RELEASE SAVEPOINT llm_usage_bucket_insert")) + if not result.rowcount: + logger.warning( + "llm usage upsert unique conflict but no matching bucket for org {}", + organization_id, + ) + + try: + from app.services.usage.pricing import PricingResolver, apply_cost_to_bucket + + resolver = pricing_resolver + if resolver is None: + resolver = PricingResolver(db) + apply_cost_to_bucket( + db, + organization_id=organization_id, + bucket=bucket, + resolver=resolver, + ) + except Exception as exc: + logger.warning("usage cost apply failed: {}", exc) def _is_unique_violation(exc: BaseException) -> bool: @@ -854,8 +894,13 @@ def _is_missing_organization_fk(exc: BaseException) -> bool: return "llm_usage_daily_organization_id_fkey" in text_blob -def _flush_pending_buffer(db: Session, organization_id: UUID) -> int: +def _flush_pending_buffer( + db: Session, organization_id: UUID, *, pricing_resolver: Any = None +) -> int: """Drain Postgres write-ahead rows into llm_usage_daily.""" + from app.services.usage.pricing import PricingResolver + + resolver = pricing_resolver or PricingResolver(db) try: rows = db.execute( text( @@ -910,6 +955,7 @@ def _flush_pending_buffer(db: Session, organization_id: UUID) -> int: organization_id, bucket, deltas, + pricing_resolver=resolver, ) ids.append(str(row["id"])) flushed += 1 @@ -966,11 +1012,14 @@ def _catalog_flush_recently(organization_id: UUID) -> bool: def flush_usage_to_catalog(db: Session, organization_id: UUID, *, force: bool = False) -> int: """Claim Redis deltas + drain PG buffer into llm_usage_daily.""" + from app.services.usage.pricing import PricingResolver + _recover_orphaned_claims() skip_redis_flush = not force and _catalog_flush_recently(organization_id) if skip_redis_flush and _has_pending_usage(organization_id): skip_redis_flush = False flushed = 0 + pricing_resolver = PricingResolver(db) if not skip_redis_flush: redis_locked = _acquire_flush_lock(organization_id) claim_key = None @@ -995,6 +1044,7 @@ def flush_usage_to_catalog(db: Session, organization_id: UUID, *, force: bool = organization_id, parsed, deltas, + pricing_resolver=pricing_resolver, ) flushed += 1 if claim_key: @@ -1032,8 +1082,8 @@ def flush_usage_to_catalog(db: Session, organization_id: UUID, *, force: bool = _client().delete(claim_key) except redis.RedisError: pass - claim_key = None - return _flush_pending_buffer(db, organization_id) + claim_key = None + return _flush_pending_buffer(db, organization_id) if skipped: _restore_buckets_to_pending(organization_id, skipped) if claim_key: diff --git a/app/services/usage/pricing.py b/app/services/usage/pricing.py new file mode 100644 index 00000000..762add7d --- /dev/null +++ b/app/services/usage/pricing.py @@ -0,0 +1,912 @@ +"""Usage pricing: rate resolution, cost computation, seed, and rollup backfill.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import date +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Tuple +from uuid import UUID + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.services.usage.pricing_cache import ( + get_cached_rate_payload, + pricing_cache_key, + set_cached_rate_payload, +) + +DEFAULT_RATES_EFFECTIVE_FROM = date(2020, 1, 1) +# Back-compat alias for older migrations. +DEFAULT_CATALOG_EFFECTIVE_FROM = DEFAULT_RATES_EFFECTIVE_FROM + +USAGE_KIND_LLM = "llm" +USAGE_KIND_STT = "stt" +USAGE_KIND_TTS = "tts" + +MICRO_USD_PER_UNIT = 1_000_000 +RATE_SOURCE_CATALOG = "catalog" +RATE_SOURCE_OVERRIDE = "override" + +_MODELS_JSON_PATH = ( + Path(__file__).resolve().parent.parent.parent / "config" / "models.json" +) + +_RATES_TABLE = "model_pricing_rates" +_RATES_TABLE_CACHE: Optional[str] = None + + +def _rates_table(db: Session) -> str: + global _RATES_TABLE_CACHE + if _RATES_TABLE_CACHE: + return _RATES_TABLE_CACHE + row = db.execute(text("SELECT to_regclass('public.model_pricing_rates')")).scalar() + if row: + _RATES_TABLE_CACHE = "model_pricing_rates" + return _RATES_TABLE_CACHE + row = db.execute(text("SELECT to_regclass('public.model_pricing_catalog')")).scalar() + if row: + _RATES_TABLE_CACHE = "model_pricing_catalog" + return _RATES_TABLE_CACHE + _RATES_TABLE_CACHE = _RATES_TABLE + return _RATES_TABLE_CACHE + + +@dataclass(frozen=True) +class RateCard: + source: str + rate_id: UUID + input_micro_usd_per_million: int = 0 + output_micro_usd_per_million: int = 0 + cache_read_micro_usd_per_million: int = 0 + cache_creation_micro_usd_per_million: int = 0 + reasoning_micro_usd_per_million: int = 0 + audio_micro_usd_per_second: int = 0 + tts_micro_usd_per_million_chars: int = 0 + + +@dataclass(frozen=True) +class CostBreakdown: + input_cost_micro_usd: int = 0 + output_cost_micro_usd: int = 0 + cache_read_cost_micro_usd: int = 0 + cache_creation_cost_micro_usd: int = 0 + reasoning_cost_micro_usd: int = 0 + audio_cost_micro_usd: int = 0 + tts_cost_micro_usd: int = 0 + total_cost_micro_usd: int = 0 + pricing_rate_source: Optional[str] = None + pricing_rate_id: Optional[UUID] = None + + +@dataclass(frozen=True) +class UsageMetrics: + prompt_tokens: int = 0 + completion_tokens: int = 0 + cache_read_tokens: int = 0 + cache_creation_tokens: int = 0 + reasoning_tokens: int = 0 + audio_seconds: int = 0 + tts_characters: int = 0 + + +def _int(value: Any, default: int = 0) -> int: + try: + return int(value or 0) + except (TypeError, ValueError): + return default + + +def _normalize_rate_source(value: Any) -> str: + source = str(value or "catalog").strip() or "catalog" + if len(source) > 255: + return source[:255] + return source + + +def _usage_kind_for_model_type(model_type: Optional[str]) -> str: + if model_type == "stt": + return USAGE_KIND_STT + if model_type in {"tts", "sts", "sound_effects", "music"}: + return USAGE_KIND_TTS + return USAGE_KIND_LLM + + +def _usd_per_million_to_micro(value: Any) -> int: + if value is None: + return 0 + try: + return int(round(float(value) * MICRO_USD_PER_UNIT)) + except (TypeError, ValueError): + return 0 + + +def _usd_per_minute_to_micro_per_second(value: Any) -> int: + if value is None: + return 0 + try: + return int(round(float(value) * MICRO_USD_PER_UNIT / 60.0)) + except (TypeError, ValueError): + return 0 + + +def _normalize_pricing_block( + pricing: Dict[str, Any], *, model_type: Optional[str] +) -> Dict[str, Any]: + """Convert models.json pricing block (plan USD fields or legacy micro fields) to DB shape.""" + usage_kind = pricing.get("usage_kind") or _usage_kind_for_model_type(model_type) + source = str(pricing.get("source") or pricing.get("_price_source") or "catalog") + + def micro(field_micro: str, field_usd: str) -> int: + if pricing.get(field_micro) is not None: + return _int(pricing.get(field_micro)) + return _usd_per_million_to_micro(pricing.get(field_usd)) + + audio_micro = _int(pricing.get("audio_micro_usd_per_second")) + if not audio_micro: + audio_micro = _usd_per_minute_to_micro_per_second(pricing.get("audio_per_minute")) + + tts_micro = _int(pricing.get("tts_micro_usd_per_million_chars")) + if not tts_micro: + tts_micro = _usd_per_million_to_micro(pricing.get("tts_per_1m_characters")) + + return { + "usage_kind": usage_kind, + "source": source, + "currency": str(pricing.get("currency") or "USD"), + "input_micro_usd_per_million": micro( + "input_micro_usd_per_million", "input_per_1m" + ), + "output_micro_usd_per_million": micro( + "output_micro_usd_per_million", "output_per_1m" + ), + "cache_read_micro_usd_per_million": micro( + "cache_read_micro_usd_per_million", "cache_read_per_1m" + ), + "cache_creation_micro_usd_per_million": micro( + "cache_creation_micro_usd_per_million", "cache_write_per_1m" + ), + "reasoning_micro_usd_per_million": micro( + "reasoning_micro_usd_per_million", "reasoning_per_1m" + ), + "audio_micro_usd_per_second": audio_micro, + "tts_micro_usd_per_million_chars": tts_micro, + } + + +def _catalog_lookup_models(model: str, usage_kind: str) -> Tuple[str, ...]: + candidates: List[str] = [] + for item in ( + model, + f"azure-{model}" if not model.startswith("azure-") else model[len("azure-") :], + ): + if item and item not in candidates: + candidates.append(item) + return tuple(candidates) + + +def _scaled_cost(units: int, rate_per_million: int) -> int: + if not units or not rate_per_million: + return 0 + return (int(units) * int(rate_per_million)) // MICRO_USD_PER_UNIT + + +def metrics_from_deltas(deltas: Dict[str, Any]) -> UsageMetrics: + return UsageMetrics( + prompt_tokens=_int(deltas.get("prompt_tokens")), + completion_tokens=_int(deltas.get("completion_tokens")), + cache_read_tokens=_int(deltas.get("cache_read_tokens")), + cache_creation_tokens=_int(deltas.get("cache_creation_tokens")), + reasoning_tokens=_int(deltas.get("reasoning_tokens")), + audio_seconds=_int(deltas.get("audio_seconds")), + tts_characters=_int(deltas.get("tts_characters")), + ) + + +def cost_fields_from_deltas( + deltas: Dict[str, Any], + *, + organization_id: UUID, + model: str, + usage_kind: str, + usage_date: date, + db: Session, + resolver: Optional["PricingResolver"] = None, +) -> Dict[str, Any]: + """Compute persisted cost columns for one pending-buffer delta row.""" + pricing = resolver or PricingResolver(db) + rate = pricing.resolve_rate( + organization_id=organization_id, + model=model, + usage_kind=usage_kind, + usage_date=usage_date, + ) + costs = compute_cost(metrics_from_deltas(deltas), rate) + return { + "input_cost_micro_usd": costs.input_cost_micro_usd, + "output_cost_micro_usd": costs.output_cost_micro_usd, + "cache_read_cost_micro_usd": costs.cache_read_cost_micro_usd, + "cache_creation_cost_micro_usd": costs.cache_creation_cost_micro_usd, + "reasoning_cost_micro_usd": costs.reasoning_cost_micro_usd, + "audio_cost_micro_usd": costs.audio_cost_micro_usd, + "tts_cost_micro_usd": costs.tts_cost_micro_usd, + "total_cost_micro_usd": costs.total_cost_micro_usd, + "pricing_rate_source": costs.pricing_rate_source, + "pricing_rate_id": str(costs.pricing_rate_id) + if costs.pricing_rate_id + else None, + } + + +def compute_cost(metrics: UsageMetrics, rate: Optional[RateCard]) -> CostBreakdown: + if rate is None: + return CostBreakdown() + + input_cost = _scaled_cost(metrics.prompt_tokens, rate.input_micro_usd_per_million) + output_cost = _scaled_cost( + metrics.completion_tokens, rate.output_micro_usd_per_million + ) + cache_read_cost = _scaled_cost( + metrics.cache_read_tokens, rate.cache_read_micro_usd_per_million + ) + cache_creation_cost = _scaled_cost( + metrics.cache_creation_tokens, rate.cache_creation_micro_usd_per_million + ) + reasoning_cost = _scaled_cost( + metrics.reasoning_tokens, rate.reasoning_micro_usd_per_million + ) + audio_cost = 0 + if metrics.audio_seconds and rate.audio_micro_usd_per_second: + audio_cost = int(metrics.audio_seconds) * int(rate.audio_micro_usd_per_second) + tts_cost = _scaled_cost( + metrics.tts_characters, rate.tts_micro_usd_per_million_chars + ) + total = ( + input_cost + + output_cost + + cache_read_cost + + cache_creation_cost + + reasoning_cost + + audio_cost + + tts_cost + ) + return CostBreakdown( + input_cost_micro_usd=input_cost, + output_cost_micro_usd=output_cost, + cache_read_cost_micro_usd=cache_read_cost, + cache_creation_cost_micro_usd=cache_creation_cost, + reasoning_cost_micro_usd=reasoning_cost, + audio_cost_micro_usd=audio_cost, + tts_cost_micro_usd=tts_cost, + total_cost_micro_usd=total, + pricing_rate_source=rate.source, + pricing_rate_id=rate.rate_id, + ) + + +def _rate_card_from_row(row: Any, *, source: str) -> RateCard: + return RateCard( + source=source, + rate_id=row["id"], + input_micro_usd_per_million=_int(row["input_micro_usd_per_million"]), + output_micro_usd_per_million=_int(row["output_micro_usd_per_million"]), + cache_read_micro_usd_per_million=_int(row["cache_read_micro_usd_per_million"]), + cache_creation_micro_usd_per_million=_int( + row["cache_creation_micro_usd_per_million"] + ), + reasoning_micro_usd_per_million=_int(row["reasoning_micro_usd_per_million"]), + audio_micro_usd_per_second=_int(row["audio_micro_usd_per_second"]), + tts_micro_usd_per_million_chars=_int(row["tts_micro_usd_per_million_chars"]), + ) + + +def _merge_override_with_catalog( + override_row: Any, catalog: Optional[RateCard] +) -> RateCard: + def pick(column: str, attr: str) -> int: + value = override_row.get(column) + if value is not None: + return _int(value) + if catalog is not None: + return getattr(catalog, attr) + return 0 + + return RateCard( + source=RATE_SOURCE_OVERRIDE, + rate_id=override_row["id"], + input_micro_usd_per_million=pick( + "input_micro_usd_per_million", "input_micro_usd_per_million" + ), + output_micro_usd_per_million=pick( + "output_micro_usd_per_million", "output_micro_usd_per_million" + ), + cache_read_micro_usd_per_million=pick( + "cache_read_micro_usd_per_million", "cache_read_micro_usd_per_million" + ), + cache_creation_micro_usd_per_million=pick( + "cache_creation_micro_usd_per_million", + "cache_creation_micro_usd_per_million", + ), + reasoning_micro_usd_per_million=pick( + "reasoning_micro_usd_per_million", "reasoning_micro_usd_per_million" + ), + audio_micro_usd_per_second=pick( + "audio_micro_usd_per_second", "audio_micro_usd_per_second" + ), + tts_micro_usd_per_million_chars=pick( + "tts_micro_usd_per_million_chars", "tts_micro_usd_per_million_chars" + ), + ) + + +def _rate_card_to_cache_payload(card: RateCard) -> Dict[str, Any]: + return { + "source": card.source, + "rate_id": str(card.rate_id), + "input_micro_usd_per_million": card.input_micro_usd_per_million, + "output_micro_usd_per_million": card.output_micro_usd_per_million, + "cache_read_micro_usd_per_million": card.cache_read_micro_usd_per_million, + "cache_creation_micro_usd_per_million": card.cache_creation_micro_usd_per_million, + "reasoning_micro_usd_per_million": card.reasoning_micro_usd_per_million, + "audio_micro_usd_per_second": card.audio_micro_usd_per_second, + "tts_micro_usd_per_million_chars": card.tts_micro_usd_per_million_chars, + } + + +def _rate_card_from_cache_payload(payload: Dict[str, Any]) -> RateCard: + return RateCard( + source=str(payload["source"]), + rate_id=UUID(str(payload["rate_id"])), + input_micro_usd_per_million=_int(payload.get("input_micro_usd_per_million")), + output_micro_usd_per_million=_int(payload.get("output_micro_usd_per_million")), + cache_read_micro_usd_per_million=_int( + payload.get("cache_read_micro_usd_per_million") + ), + cache_creation_micro_usd_per_million=_int( + payload.get("cache_creation_micro_usd_per_million") + ), + reasoning_micro_usd_per_million=_int( + payload.get("reasoning_micro_usd_per_million") + ), + audio_micro_usd_per_second=_int(payload.get("audio_micro_usd_per_second")), + tts_micro_usd_per_million_chars=_int( + payload.get("tts_micro_usd_per_million_chars") + ), + ) + + +class PricingResolver: + """Cached pricing lookups for flush/recompute batches.""" + + def __init__(self, db: Session): + self._db = db + self._memory_cache: Dict[Tuple[str, ...], Optional[RateCard]] = {} + + def resolve_rate( + self, + *, + organization_id: UUID, + model: str, + usage_kind: str, + usage_date: date, + ) -> Optional[RateCard]: + kind = usage_kind or USAGE_KIND_LLM + memory_key = (str(organization_id), model, kind, usage_date.isoformat()) + if memory_key in self._memory_cache: + return self._memory_cache[memory_key] + + redis_key = pricing_cache_key( + organization_id=organization_id, + model=model, + usage_kind=kind, + usage_date=usage_date, + ) + cached = get_cached_rate_payload(redis_key) + if cached: + card = _rate_card_from_cache_payload(cached) + self._memory_cache[memory_key] = card + return card + + override = self._load_override_rate(organization_id, model, kind, usage_date) + if override is not None: + set_cached_rate_payload(redis_key, _rate_card_to_cache_payload(override)) + self._memory_cache[memory_key] = override + return override + + catalog = self._resolve_catalog(model, kind, usage_date) + set_cached_rate_payload( + redis_key, + _rate_card_to_cache_payload(catalog) if catalog else None, + ) + self._memory_cache[memory_key] = catalog + return catalog + + def _resolve_catalog( + self, model: str, usage_kind: str, usage_date: date + ) -> Optional[RateCard]: + for candidate in _catalog_lookup_models(model, usage_kind): + card = self._load_catalog_rate(candidate, usage_kind, usage_date) + if card is not None: + return card + return None + + def _load_catalog_rate( + self, model: str, usage_kind: str, usage_date: date + ) -> Optional[RateCard]: + table = _rates_table(self._db) + row = self._db.execute( + text( + f""" + SELECT + id, + input_micro_usd_per_million, + output_micro_usd_per_million, + cache_read_micro_usd_per_million, + cache_creation_micro_usd_per_million, + reasoning_micro_usd_per_million, + audio_micro_usd_per_second, + tts_micro_usd_per_million_chars + FROM {table} + WHERE model = :model + AND usage_kind = :usage_kind + AND effective_from <= :usage_date + AND (effective_to IS NULL OR effective_to >= :usage_date) + ORDER BY effective_from DESC + LIMIT 1 + """ + ), + { + "model": model, + "usage_kind": usage_kind, + "usage_date": usage_date.isoformat(), + }, + ).mappings().first() + if not row: + return None + return _rate_card_from_row(row, source=RATE_SOURCE_CATALOG) + + def _load_override_rate( + self, + organization_id: UUID, + model: str, + usage_kind: str, + usage_date: date, + ) -> Optional[RateCard]: + row = self._db.execute( + text( + """ + SELECT + id, + input_micro_usd_per_million, + output_micro_usd_per_million, + cache_read_micro_usd_per_million, + cache_creation_micro_usd_per_million, + reasoning_micro_usd_per_million, + audio_micro_usd_per_second, + tts_micro_usd_per_million_chars + FROM org_model_pricing_overrides + WHERE organization_id = CAST(:organization_id AS uuid) + AND model = :model + AND usage_kind = :usage_kind + AND effective_from <= :usage_date + AND (effective_to IS NULL OR effective_to >= :usage_date) + ORDER BY effective_from DESC + LIMIT 1 + """ + ), + { + "organization_id": str(organization_id), + "model": model, + "usage_kind": usage_kind, + "usage_date": usage_date.isoformat(), + }, + ).mappings().first() + if not row: + return None + catalog = self._resolve_catalog(model, usage_kind, usage_date) + return _merge_override_with_catalog(row, catalog) + + +def _pricing_entries_from_models_json() -> Dict[str, Dict[str, Any]]: + if not _MODELS_JSON_PATH.exists(): + return {} + try: + with open(_MODELS_JSON_PATH, "r", encoding="utf-8") as handle: + payload = json.load(handle) + except (OSError, json.JSONDecodeError): + return {} + entries: Dict[str, Dict[str, Any]] = {} + for model_name, config in payload.items(): + if model_name.startswith("_") or not isinstance(config, dict): + continue + pricing = config.get("pricing") + if not isinstance(pricing, dict): + continue + entries[model_name] = _normalize_pricing_block( + pricing, model_type=config.get("model_type") + ) + return entries + + +def seed_pricing_rates(db: Session, *, effective_from: Optional[date] = None) -> int: + """Upsert global rates from models.json pricing blocks (seed only; DB is runtime truth).""" + day = effective_from or DEFAULT_RATES_EFFECTIVE_FROM + table = _rates_table(db) + has_currency = ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.columns + WHERE table_name = :table_name AND column_name = 'currency' + """ + ), + {"table_name": table}, + ).first() + is not None + ) + inserted = 0 + for model_name, pricing in _pricing_entries_from_models_json().items(): + usage_kind = pricing.get("usage_kind") or USAGE_KIND_LLM + base_params = { + "model": model_name, + "usage_kind": usage_kind, + "effective_from": day.isoformat(), + "input_micro_usd_per_million": _int( + pricing.get("input_micro_usd_per_million") + ), + "output_micro_usd_per_million": _int( + pricing.get("output_micro_usd_per_million") + ), + "cache_read_micro_usd_per_million": _int( + pricing.get("cache_read_micro_usd_per_million") + ), + "cache_creation_micro_usd_per_million": _int( + pricing.get("cache_creation_micro_usd_per_million") + ), + "reasoning_micro_usd_per_million": _int( + pricing.get("reasoning_micro_usd_per_million") + ), + "audio_micro_usd_per_second": _int( + pricing.get("audio_micro_usd_per_second") + ), + "tts_micro_usd_per_million_chars": _int( + pricing.get("tts_micro_usd_per_million_chars") + ), + } + if has_currency: + sql = f""" + INSERT INTO {table} ( + id, model, usage_kind, effective_from, currency, source, + input_micro_usd_per_million, + output_micro_usd_per_million, + cache_read_micro_usd_per_million, + cache_creation_micro_usd_per_million, + reasoning_micro_usd_per_million, + audio_micro_usd_per_second, + tts_micro_usd_per_million_chars, + created_at, updated_at + ) VALUES ( + gen_random_uuid(), :model, :usage_kind, CAST(:effective_from AS date), + :currency, :source, + :input_micro_usd_per_million, + :output_micro_usd_per_million, + :cache_read_micro_usd_per_million, + :cache_creation_micro_usd_per_million, + :reasoning_micro_usd_per_million, + :audio_micro_usd_per_second, + :tts_micro_usd_per_million_chars, + now(), now() + ) + ON CONFLICT (model, usage_kind, effective_from) DO UPDATE SET + currency = EXCLUDED.currency, + source = EXCLUDED.source, + input_micro_usd_per_million = EXCLUDED.input_micro_usd_per_million, + output_micro_usd_per_million = EXCLUDED.output_micro_usd_per_million, + cache_read_micro_usd_per_million = EXCLUDED.cache_read_micro_usd_per_million, + cache_creation_micro_usd_per_million = EXCLUDED.cache_creation_micro_usd_per_million, + reasoning_micro_usd_per_million = EXCLUDED.reasoning_micro_usd_per_million, + audio_micro_usd_per_second = EXCLUDED.audio_micro_usd_per_second, + tts_micro_usd_per_million_chars = EXCLUDED.tts_micro_usd_per_million_chars, + updated_at = now() + """ + params = { + **base_params, + "currency": pricing.get("currency") or "USD", + "source": _normalize_rate_source(pricing.get("source")), + } + else: + sql = f""" + INSERT INTO {table} ( + id, model, usage_kind, effective_from, + input_micro_usd_per_million, + output_micro_usd_per_million, + cache_read_micro_usd_per_million, + cache_creation_micro_usd_per_million, + reasoning_micro_usd_per_million, + audio_micro_usd_per_second, + tts_micro_usd_per_million_chars, + created_at, updated_at + ) VALUES ( + gen_random_uuid(), :model, :usage_kind, CAST(:effective_from AS date), + :input_micro_usd_per_million, + :output_micro_usd_per_million, + :cache_read_micro_usd_per_million, + :cache_creation_micro_usd_per_million, + :reasoning_micro_usd_per_million, + :audio_micro_usd_per_second, + :tts_micro_usd_per_million_chars, + now(), now() + ) + ON CONFLICT (model, usage_kind, effective_from) DO UPDATE SET + input_micro_usd_per_million = EXCLUDED.input_micro_usd_per_million, + output_micro_usd_per_million = EXCLUDED.output_micro_usd_per_million, + cache_read_micro_usd_per_million = EXCLUDED.cache_read_micro_usd_per_million, + cache_creation_micro_usd_per_million = EXCLUDED.cache_creation_micro_usd_per_million, + reasoning_micro_usd_per_million = EXCLUDED.reasoning_micro_usd_per_million, + audio_micro_usd_per_second = EXCLUDED.audio_micro_usd_per_second, + tts_micro_usd_per_million_chars = EXCLUDED.tts_micro_usd_per_million_chars, + updated_at = now() + """ + params = base_params + result = db.execute(text(sql), params) + if result.rowcount: + inserted += 1 + if inserted: + from app.services.usage.pricing_cache import invalidate_all_pricing_cache + + invalidate_all_pricing_cache() + return inserted + + +def seed_pricing_catalog(db: Session, *, effective_from: Optional[date] = None) -> int: + """Back-compat alias.""" + return seed_pricing_rates(db, effective_from=effective_from) + + +def apply_cost_to_bucket( + db: Session, + *, + organization_id: UUID, + bucket: Dict[str, Any], + resolver: Optional[PricingResolver] = None, +) -> bool: + """Recompute and persist cost columns for one rollup bucket from row totals.""" + context = bucket.get("context") or {} + workspace_id = bucket.get("workspace_id") + usage_kind = bucket.get("usage_kind") or USAGE_KIND_LLM + usage_date = bucket["usage_date"] + if isinstance(usage_date, str): + usage_date = date.fromisoformat(usage_date) + + base_params = { + "organization_id": str(organization_id), + "workspace_id": str(workspace_id) if workspace_id else None, + "product_section": bucket["product_section"], + "model": bucket["model"], + "usage_date": usage_date.isoformat(), + "usage_kind": usage_kind, + } + exact_params = { + **base_params, + "context": json.dumps(context), + "context_resource_id": str(context.get("resource_id") or ""), + "context_resource_type": str(context.get("resource_type") or ""), + } + row = db.execute( + text( + """ + SELECT + prompt_tokens, completion_tokens, cache_read_tokens, + cache_creation_tokens, reasoning_tokens, audio_seconds, + tts_characters + FROM llm_usage_daily + WHERE organization_id = CAST(:organization_id AS uuid) + AND product_section = :product_section + AND model = :model + AND usage_date = CAST(:usage_date AS date) + AND usage_kind = :usage_kind + AND workspace_id IS NOT DISTINCT FROM CAST(:workspace_id AS uuid) + AND context = CAST(:context AS jsonb) + """ + ), + exact_params, + ).mappings().first() + if not row: + row = db.execute( + text( + """ + SELECT + prompt_tokens, completion_tokens, cache_read_tokens, + cache_creation_tokens, reasoning_tokens, audio_seconds, + tts_characters + FROM llm_usage_daily + WHERE organization_id = CAST(:organization_id AS uuid) + AND product_section = :product_section + AND model = :model + AND usage_date = CAST(:usage_date AS date) + AND usage_kind = :usage_kind + AND workspace_id IS NOT DISTINCT FROM CAST(:workspace_id AS uuid) + AND COALESCE(context->>'resource_id', '') = :context_resource_id + AND COALESCE(context->>'resource_type', '') = :context_resource_type + """ + ), + exact_params, + ).mappings().first() + if not row: + return False + + pricing = resolver or PricingResolver(db) + rate = pricing.resolve_rate( + organization_id=organization_id, + model=bucket["model"], + usage_kind=usage_kind, + usage_date=usage_date, + ) + costs = compute_cost( + UsageMetrics( + prompt_tokens=_int(row["prompt_tokens"]), + completion_tokens=_int(row["completion_tokens"]), + cache_read_tokens=_int(row["cache_read_tokens"]), + cache_creation_tokens=_int(row["cache_creation_tokens"]), + reasoning_tokens=_int(row["reasoning_tokens"]), + audio_seconds=_int(row["audio_seconds"]), + tts_characters=_int(row["tts_characters"]), + ), + rate, + ) + update_params = { + **exact_params, + "input_cost_micro_usd": costs.input_cost_micro_usd, + "output_cost_micro_usd": costs.output_cost_micro_usd, + "cache_read_cost_micro_usd": costs.cache_read_cost_micro_usd, + "cache_creation_cost_micro_usd": costs.cache_creation_cost_micro_usd, + "reasoning_cost_micro_usd": costs.reasoning_cost_micro_usd, + "audio_cost_micro_usd": costs.audio_cost_micro_usd, + "tts_cost_micro_usd": costs.tts_cost_micro_usd, + "total_cost_micro_usd": costs.total_cost_micro_usd, + "pricing_rate_source": costs.pricing_rate_source, + "pricing_rate_id": str(costs.pricing_rate_id) + if costs.pricing_rate_id + else None, + } + result = db.execute( + text( + """ + UPDATE llm_usage_daily SET + input_cost_micro_usd = :input_cost_micro_usd, + output_cost_micro_usd = :output_cost_micro_usd, + cache_read_cost_micro_usd = :cache_read_cost_micro_usd, + cache_creation_cost_micro_usd = :cache_creation_cost_micro_usd, + reasoning_cost_micro_usd = :reasoning_cost_micro_usd, + audio_cost_micro_usd = :audio_cost_micro_usd, + tts_cost_micro_usd = :tts_cost_micro_usd, + total_cost_micro_usd = :total_cost_micro_usd, + pricing_rate_source = :pricing_rate_source, + pricing_rate_id = CAST(:pricing_rate_id AS uuid), + updated_at = now() + WHERE organization_id = CAST(:organization_id AS uuid) + AND product_section = :product_section + AND model = :model + AND usage_date = CAST(:usage_date AS date) + AND usage_kind = :usage_kind + AND workspace_id IS NOT DISTINCT FROM CAST(:workspace_id AS uuid) + AND context = CAST(:context AS jsonb) + """ + ), + update_params, + ) + if result.rowcount: + return True + result = db.execute( + text( + """ + UPDATE llm_usage_daily SET + input_cost_micro_usd = :input_cost_micro_usd, + output_cost_micro_usd = :output_cost_micro_usd, + cache_read_cost_micro_usd = :cache_read_cost_micro_usd, + cache_creation_cost_micro_usd = :cache_creation_cost_micro_usd, + reasoning_cost_micro_usd = :reasoning_cost_micro_usd, + audio_cost_micro_usd = :audio_cost_micro_usd, + tts_cost_micro_usd = :tts_cost_micro_usd, + total_cost_micro_usd = :total_cost_micro_usd, + pricing_rate_source = :pricing_rate_source, + pricing_rate_id = CAST(:pricing_rate_id AS uuid), + updated_at = now() + WHERE organization_id = CAST(:organization_id AS uuid) + AND product_section = :product_section + AND model = :model + AND usage_date = CAST(:usage_date AS date) + AND usage_kind = :usage_kind + AND workspace_id IS NOT DISTINCT FROM CAST(:workspace_id AS uuid) + AND COALESCE(context->>'resource_id', '') = :context_resource_id + AND COALESCE(context->>'resource_type', '') = :context_resource_type + """ + ), + update_params, + ) + return bool(result.rowcount) + + +def recompute_usage_costs( + db: Session, + *, + organization_id: Optional[UUID] = None, + model: Optional[str] = None, + usage_kind: Optional[str] = None, + start_date: Optional[date] = None, + end_date: Optional[date] = None, + batch_size: int = 500, + on_progress: Optional[Callable[[int], None]] = None, +) -> int: + """Recompute stored costs for existing rollup rows (backfill / override changes).""" + resolver = PricingResolver(db) + updated = 0 + last_id: Optional[str] = None + + while True: + params: Dict[str, Any] = {"batch_size": batch_size} + filters = ["1=1"] + if organization_id is not None: + filters.append("organization_id = CAST(:organization_id AS uuid)") + params["organization_id"] = str(organization_id) + if model is not None: + filters.append("model = :model") + params["model"] = model + if usage_kind is not None: + filters.append("usage_kind = :usage_kind") + params["usage_kind"] = usage_kind + if start_date is not None: + filters.append("usage_date >= CAST(:start_date AS date)") + params["start_date"] = start_date.isoformat() + if end_date is not None: + filters.append("usage_date <= CAST(:end_date AS date)") + params["end_date"] = end_date.isoformat() + if last_id is not None: + filters.append("id > CAST(:last_id AS uuid)") + params["last_id"] = last_id + + rows = db.execute( + text( + f""" + SELECT + id, organization_id, workspace_id, product_section, model, + context, usage_date, usage_kind + FROM llm_usage_daily + WHERE {' AND '.join(filters)} + ORDER BY id + LIMIT :batch_size + """ + ), + params, + ).mappings().all() + if not rows: + break + + for row in rows: + bucket = { + "workspace_id": row["workspace_id"], + "product_section": row["product_section"], + "model": row["model"], + "context": row["context"] or {}, + "usage_date": row["usage_date"], + "usage_kind": row["usage_kind"] or USAGE_KIND_LLM, + } + if apply_cost_to_bucket( + db, + organization_id=row["organization_id"], + bucket=bucket, + resolver=resolver, + ): + updated += 1 + last_id = str(row["id"]) + + db.commit() + if on_progress is not None: + on_progress(updated) + + if on_progress is not None: + on_progress(updated) + + return updated diff --git a/app/services/usage/pricing_cache.py b/app/services/usage/pricing_cache.py new file mode 100644 index 00000000..dd2ebb0f --- /dev/null +++ b/app/services/usage/pricing_cache.py @@ -0,0 +1,87 @@ +"""Redis cache for resolved usage pricing rates.""" + +from __future__ import annotations + +import json +from datetime import date +from typing import Optional +from uuid import UUID + +import redis +from loguru import logger + +from app.config import settings + +PRICING_CACHE_TTL_SEC = 3600 +PRICING_NULL_CACHE_TTL_SEC = 300 +PRICING_CACHE_PREFIX = "usage:pricing" + +_redis: redis.Redis | None = None + + +def _client() -> redis.Redis: + global _redis + if _redis is None: + _redis = redis.from_url(settings.REDIS_URL, decode_responses=True) + return _redis + + +def pricing_cache_key( + *, + organization_id: UUID, + model: str, + usage_kind: str, + usage_date: date, +) -> str: + return ( + f"{PRICING_CACHE_PREFIX}:{organization_id}:{model}:" + f"{usage_kind}:{usage_date.isoformat()}" + ) + + +def get_cached_rate_payload(key: str) -> Optional[dict]: + try: + raw = _client().get(key) + except redis.RedisError as exc: + logger.debug("pricing cache read skipped: {}", exc) + return None + if raw is None: + return None + if raw == "__null__": + return {} + try: + payload = json.loads(raw) + except json.JSONDecodeError: + return None + return payload if isinstance(payload, dict) else None + + +def set_cached_rate_payload(key: str, payload: Optional[dict]) -> None: + try: + value = "__null__" if not payload else json.dumps(payload) + ttl = PRICING_NULL_CACHE_TTL_SEC if not payload else PRICING_CACHE_TTL_SEC + _client().setex(key, ttl, value) + except redis.RedisError as exc: + logger.debug("pricing cache write skipped: {}", exc) + + +def invalidate_org_pricing_cache(organization_id: UUID) -> None: + _invalidate_pricing_cache_pattern(f"{PRICING_CACHE_PREFIX}:{organization_id}:*") + + +def invalidate_all_pricing_cache() -> None: + _invalidate_pricing_cache_pattern(f"{PRICING_CACHE_PREFIX}:*") + + +def _invalidate_pricing_cache_pattern(pattern: str) -> None: + try: + client = _client() + cursor = 0 + while True: + cursor, keys = client.scan(cursor=cursor, match=pattern, count=200) + if keys: + client.delete(*keys) + if cursor == 0: + break + except redis.RedisError as exc: + logger.debug("pricing cache invalidate skipped: {}", exc) diff --git a/app/services/usage/pricing_jobs.py b/app/services/usage/pricing_jobs.py new file mode 100644 index 00000000..5e01947a --- /dev/null +++ b/app/services/usage/pricing_jobs.py @@ -0,0 +1,145 @@ +"""Usage cost recompute job lifecycle.""" + +from __future__ import annotations + +from datetime import date, datetime, timezone +from typing import Any, Dict, Optional +from uuid import UUID + +from fastapi import HTTPException +from sqlalchemy.orm import Session + +from app.models.database import UsageCostRecomputeJob + +ACTIVE_JOB_STATUSES = ("pending", "running") +TERMINAL_JOB_STATUSES = ("completed", "failed") + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def get_recompute_job( + db: Session, + *, + organization_id: UUID, + job_id: UUID, +) -> UsageCostRecomputeJob: + job = ( + db.query(UsageCostRecomputeJob) + .filter( + UsageCostRecomputeJob.id == job_id, + UsageCostRecomputeJob.organization_id == organization_id, + ) + .first() + ) + if job is None: + raise HTTPException(status_code=404, detail="Recompute job not found") + return job + + +def create_recompute_job( + db: Session, + *, + organization_id: UUID, + model: Optional[str] = None, + usage_kind: Optional[str] = None, + start_date: Optional[date] = None, + end_date: Optional[date] = None, +) -> UsageCostRecomputeJob: + active = ( + db.query(UsageCostRecomputeJob) + .filter( + UsageCostRecomputeJob.organization_id == organization_id, + UsageCostRecomputeJob.status.in_(ACTIVE_JOB_STATUSES), + ) + .first() + ) + if active is not None: + raise HTTPException( + status_code=409, + detail="A usage cost recompute job is already in progress", + ) + + job = UsageCostRecomputeJob( + organization_id=organization_id, + status="pending", + model=model, + usage_kind=usage_kind, + start_date=start_date, + end_date=end_date, + ) + db.add(job) + db.commit() + db.refresh(job) + return job + + +def enqueue_recompute_job(db: Session, job: UsageCostRecomputeJob) -> str: + from app.workers.tasks import recompute_usage_costs_task + + result = recompute_usage_costs_task.delay(job_id=str(job.id)) + job.celery_task_id = result.id + job.updated_at = _utcnow() + db.commit() + return result.id + + +def mark_job_running(db: Session, job_id: UUID) -> None: + job = db.query(UsageCostRecomputeJob).filter(UsageCostRecomputeJob.id == job_id).first() + if job is None: + return + job.status = "running" + job.updated_at = _utcnow() + db.commit() + + +def update_job_progress(db: Session, job_id: UUID, updated_rows: int) -> None: + job = db.query(UsageCostRecomputeJob).filter(UsageCostRecomputeJob.id == job_id).first() + if job is None: + return + job.updated_rows = updated_rows + job.updated_at = _utcnow() + db.commit() + + +def mark_job_completed(db: Session, job_id: UUID, updated_rows: int) -> None: + job = db.query(UsageCostRecomputeJob).filter(UsageCostRecomputeJob.id == job_id).first() + if job is None: + return + now = _utcnow() + job.status = "completed" + job.updated_rows = updated_rows + job.updated_at = now + job.completed_at = now + db.commit() + + +def mark_job_failed(db: Session, job_id: UUID, error_message: str) -> None: + job = db.query(UsageCostRecomputeJob).filter(UsageCostRecomputeJob.id == job_id).first() + if job is None: + return + now = _utcnow() + job.status = "failed" + job.error_message = error_message[:4000] + job.updated_at = now + job.completed_at = now + db.commit() + + +def job_to_dict(job: UsageCostRecomputeJob) -> Dict[str, Any]: + return { + "id": job.id, + "organization_id": job.organization_id, + "status": job.status, + "model": job.model, + "usage_kind": job.usage_kind, + "start_date": job.start_date, + "end_date": job.end_date, + "updated_rows": int(job.updated_rows or 0), + "error_message": job.error_message, + "celery_task_id": job.celery_task_id, + "created_at": job.created_at, + "updated_at": job.updated_at, + "completed_at": job.completed_at, + } diff --git a/app/services/usage/pricing_ops.py b/app/services/usage/pricing_ops.py new file mode 100644 index 00000000..abd55d4b --- /dev/null +++ b/app/services/usage/pricing_ops.py @@ -0,0 +1,137 @@ +"""Ops helpers for usage pricing catalog maintenance.""" + +from __future__ import annotations + +import json +from datetime import date +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.services.usage.pricing import ( + DEFAULT_RATES_EFFECTIVE_FROM, + _pricing_entries_from_models_json, + _rates_table, + seed_pricing_rates, +) + +_MODELS_JSON_PATH = ( + Path(__file__).resolve().parent.parent.parent / "config" / "models.json" +) +_CATALOG_JSON_PATH = ( + Path(__file__).resolve().parent.parent.parent / "config" / "pricing_catalog.json" +) + +_RATE_COMPARE_COLUMNS = ( + "input_micro_usd_per_million", + "output_micro_usd_per_million", + "cache_read_micro_usd_per_million", + "cache_creation_micro_usd_per_million", + "reasoning_micro_usd_per_million", + "audio_micro_usd_per_second", + "tts_micro_usd_per_million_chars", +) + + +def models_missing_pricing_blocks() -> List[str]: + if not _MODELS_JSON_PATH.exists(): + return [] + payload = json.loads(_MODELS_JSON_PATH.read_text(encoding="utf-8")) + missing: List[str] = [] + for model_name, config in payload.items(): + if model_name.startswith("_") or not isinstance(config, dict): + continue + pricing = config.get("pricing") + if not isinstance(pricing, dict): + missing.append(model_name) + return sorted(missing) + + +def litellm_unresolved_models() -> List[Dict[str, Any]]: + if not _CATALOG_JSON_PATH.exists(): + return [] + payload = json.loads(_CATALOG_JSON_PATH.read_text(encoding="utf-8")) + meta = payload.get("_metadata") + if not isinstance(meta, dict): + return [] + unresolved = meta.get("unresolved") + return unresolved if isinstance(unresolved, list) else [] + + +def _load_db_rates( + db: Session, *, effective_from: date +) -> Dict[Tuple[str, str], Dict[str, Any]]: + table = _rates_table(db) + rows = db.execute( + text( + f""" + SELECT model, usage_kind, {_rate_columns_sql()} + FROM {table} + WHERE effective_from = CAST(:effective_from AS date) + """ + ), + {"effective_from": effective_from.isoformat()}, + ).mappings().all() + return {(row["model"], row["usage_kind"]): dict(row) for row in rows} + + +def _rate_columns_sql() -> str: + return ", ".join(_RATE_COMPARE_COLUMNS) + + +def diff_models_json_vs_db( + db: Session, *, effective_from: Optional[date] = None +) -> Dict[str, Any]: + day = effective_from or DEFAULT_RATES_EFFECTIVE_FROM + json_rates = _pricing_entries_from_models_json() + db_rates = _load_db_rates(db, effective_from=day) + + json_keys = {(model, entry.get("usage_kind") or "llm") for model, entry in json_rates.items()} + db_keys = set(db_rates.keys()) + + only_in_json = sorted(json_keys - db_keys) + only_in_db = sorted(db_keys - json_keys) + mismatches: List[Dict[str, Any]] = [] + + for key in sorted(json_keys & db_keys): + model, usage_kind = key + expected = json_rates[model] + actual = db_rates[key] + field_diffs: Dict[str, Dict[str, int]] = {} + for column in _RATE_COMPARE_COLUMNS: + left = int(expected.get(column) or 0) + right = int(actual.get(column) or 0) + if left != right: + field_diffs[column] = {"models_json": left, "database": right} + if field_diffs: + mismatches.append( + { + "model": model, + "usage_kind": usage_kind, + "fields": field_diffs, + } + ) + + return { + "effective_from": day.isoformat(), + "models_json_count": len(json_rates), + "database_count": len(db_rates), + "only_in_models_json": [ + {"model": model, "usage_kind": kind} for model, kind in only_in_json + ], + "only_in_database": [ + {"model": model, "usage_kind": kind} for model, kind in only_in_db + ], + "mismatches": mismatches, + "missing_pricing_blocks": models_missing_pricing_blocks(), + "litellm_unresolved": litellm_unresolved_models(), + "in_sync": not only_in_json and not only_in_db and not mismatches, + } + + +def seed_rates_from_models_json( + db: Session, *, effective_from: Optional[date] = None +) -> int: + return seed_pricing_rates(db, effective_from=effective_from) diff --git a/app/services/usage/pricing_overrides.py b/app/services/usage/pricing_overrides.py new file mode 100644 index 00000000..7f55ef01 --- /dev/null +++ b/app/services/usage/pricing_overrides.py @@ -0,0 +1,530 @@ +"""Org-level usage pricing overrides.""" + +from __future__ import annotations + +from datetime import date, timedelta +from typing import Any, Dict, List, Optional, Set +from uuid import UUID + +from fastapi import HTTPException +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.services.usage.pricing import ( + PricingResolver, + USAGE_KIND_LLM, + USAGE_KIND_STT, + USAGE_KIND_TTS, + _int, + _pricing_entries_from_models_json, + _rates_table, + _usd_per_million_to_micro, + _usd_per_minute_to_micro_per_second, +) +from app.services.usage.pricing_cache import invalidate_org_pricing_cache +from app.services.usage.pricing_jobs import create_recompute_job, enqueue_recompute_job +from app.services.usage.usage_costs import micro_to_usd + +RATE_COLUMNS = ( + "input_micro_usd_per_million", + "output_micro_usd_per_million", + "cache_read_micro_usd_per_million", + "cache_creation_micro_usd_per_million", + "reasoning_micro_usd_per_million", + "audio_micro_usd_per_second", + "tts_micro_usd_per_million_chars", +) + +USD_RATE_FIELDS = { + "input_per_1m": "input_micro_usd_per_million", + "output_per_1m": "output_micro_usd_per_million", + "cache_read_per_1m": "cache_read_micro_usd_per_million", + "cache_write_per_1m": "cache_creation_micro_usd_per_million", + "reasoning_per_1m": "reasoning_micro_usd_per_million", + "audio_per_minute": "audio_micro_usd_per_second", + "tts_per_1m_characters": "tts_micro_usd_per_million_chars", +} + + +def _known_models(db: Session) -> Set[str]: + models = set(_pricing_entries_from_models_json().keys()) + table = _rates_table(db) + rows = db.execute(text(f"SELECT DISTINCT model FROM {table}")).scalars().all() + models.update(rows) + return models + + +def validate_model_name(db: Session, model: str) -> None: + if model not in _known_models(db): + raise HTTPException(status_code=400, detail=f"Unknown model: {model}") + + +def _validate_usage_kind(usage_kind: str) -> str: + kind = usage_kind or USAGE_KIND_LLM + if kind not in {USAGE_KIND_LLM, USAGE_KIND_STT, USAGE_KIND_TTS}: + raise HTTPException(status_code=400, detail=f"Invalid usage_kind: {kind}") + return kind + + +def _micro_to_optional_usd_per_1m(micro: Optional[int]) -> Optional[float]: + if micro is None: + return None + return micro_to_usd(_int(micro)) + + +def _micro_to_optional_usd_per_minute(micro_per_second: Optional[int]) -> Optional[float]: + if micro_per_second is None: + return None + return micro_to_usd(_int(micro_per_second) * 60) + + +def _rates_usd_from_row(row: Dict[str, Any]) -> Dict[str, Optional[float]]: + return { + "input_per_1m": _micro_to_optional_usd_per_1m(row.get("input_micro_usd_per_million")), + "output_per_1m": _micro_to_optional_usd_per_1m(row.get("output_micro_usd_per_million")), + "cache_read_per_1m": _micro_to_optional_usd_per_1m( + row.get("cache_read_micro_usd_per_million") + ), + "cache_write_per_1m": _micro_to_optional_usd_per_1m( + row.get("cache_creation_micro_usd_per_million") + ), + "reasoning_per_1m": _micro_to_optional_usd_per_1m( + row.get("reasoning_micro_usd_per_million") + ), + "audio_per_minute": _micro_to_optional_usd_per_minute( + row.get("audio_micro_usd_per_second") + ), + "tts_per_1m_characters": _micro_to_optional_usd_per_1m( + row.get("tts_micro_usd_per_million_chars") + ), + } + + +def _rate_card_to_usd_dict(card) -> Dict[str, float]: + return { + "input_per_1m": micro_to_usd(card.input_micro_usd_per_million), + "output_per_1m": micro_to_usd(card.output_micro_usd_per_million), + "cache_read_per_1m": micro_to_usd(card.cache_read_micro_usd_per_million), + "cache_write_per_1m": micro_to_usd(card.cache_creation_micro_usd_per_million), + "reasoning_per_1m": micro_to_usd(card.reasoning_micro_usd_per_million), + "audio_per_minute": micro_to_usd(card.audio_micro_usd_per_second * 60), + "tts_per_1m_characters": micro_to_usd(card.tts_micro_usd_per_million_chars), + } + + +def _override_row_to_dict(row: Any) -> Dict[str, Any]: + payload = dict(row) + for key in ("id", "organization_id"): + if payload.get(key) is not None: + payload[key] = str(payload[key]) + rates = _rates_usd_from_row(payload) + return { + "id": payload["id"], + "organization_id": payload["organization_id"], + "model": payload["model"], + "usage_kind": payload["usage_kind"], + "effective_from": payload["effective_from"], + "effective_to": payload.get("effective_to"), + "rates": rates, + "created_at": payload.get("created_at"), + "updated_at": payload.get("updated_at"), + } + + +def _usd_payload_to_micro_columns(payload: Dict[str, Any]) -> Dict[str, Optional[int]]: + columns: Dict[str, Optional[int]] = {} + for usd_field, micro_column in USD_RATE_FIELDS.items(): + if usd_field not in payload: + continue + value = payload.get(usd_field) + if value is None: + columns[micro_column] = None + continue + if usd_field == "audio_per_minute": + columns[micro_column] = _usd_per_minute_to_micro_per_second(value) + else: + columns[micro_column] = _usd_per_million_to_micro(value) + return columns + + +def list_overrides( + db: Session, + *, + organization_id: UUID, + model: Optional[str] = None, + usage_kind: Optional[str] = None, +) -> List[Dict[str, Any]]: + filters = ["organization_id = CAST(:organization_id AS uuid)"] + params: Dict[str, Any] = {"organization_id": str(organization_id)} + if model is not None: + filters.append("model = :model") + params["model"] = model + if usage_kind is not None: + filters.append("usage_kind = :usage_kind") + params["usage_kind"] = _validate_usage_kind(usage_kind) + + rows = db.execute( + text( + f""" + SELECT * + FROM org_model_pricing_overrides + WHERE {' AND '.join(filters)} + ORDER BY model ASC, usage_kind ASC, effective_from DESC + """ + ), + params, + ).mappings().all() + return [_override_row_to_dict(row) for row in rows] + + +def get_effective_rate( + db: Session, + *, + organization_id: UUID, + model: str, + usage_kind: str, + as_of: date, +) -> Dict[str, Any]: + validate_model_name(db, model) + kind = _validate_usage_kind(usage_kind) + resolver = PricingResolver(db) + effective = resolver.resolve_rate( + organization_id=organization_id, + model=model, + usage_kind=kind, + usage_date=as_of, + ) + catalog = resolver._resolve_catalog(model, kind, as_of) + override_row = db.execute( + text( + """ + SELECT * + FROM org_model_pricing_overrides + WHERE organization_id = CAST(:organization_id AS uuid) + AND model = :model + AND usage_kind = :usage_kind + AND effective_from <= :as_of + AND (effective_to IS NULL OR effective_to >= :as_of) + ORDER BY effective_from DESC + LIMIT 1 + """ + ), + { + "organization_id": str(organization_id), + "model": model, + "usage_kind": kind, + "as_of": as_of.isoformat(), + }, + ).mappings().first() + + return { + "model": model, + "usage_kind": kind, + "as_of": as_of, + "catalog_rates": _rate_card_to_usd_dict(catalog) if catalog else None, + "catalog_rate_id": str(catalog.rate_id) if catalog else None, + "override": _override_row_to_dict(override_row) if override_row else None, + "effective_rates": _rate_card_to_usd_dict(effective) if effective else None, + "effective_source": effective.source if effective else None, + "effective_rate_id": str(effective.rate_id) if effective else None, + "has_override": override_row is not None, + } + + +def list_effective_pricing( + db: Session, + *, + organization_id: UUID, + usage_kind: Optional[str] = None, + model: Optional[str] = None, + as_of: date, + limit: int = 200, +) -> List[Dict[str, Any]]: + if model: + return [ + get_effective_rate( + db, + organization_id=organization_id, + model=model, + usage_kind=usage_kind or USAGE_KIND_LLM, + as_of=as_of, + ) + ] + + keys: Set[tuple[str, str]] = set() + for entry in list_overrides(db, organization_id=organization_id, usage_kind=usage_kind): + keys.add((entry["model"], entry["usage_kind"])) + + table = _rates_table(db) + rate_filters = ["effective_from <= :as_of", "(effective_to IS NULL OR effective_to >= :as_of)"] + params: Dict[str, Any] = {"as_of": as_of.isoformat(), "limit": limit} + if usage_kind is not None: + rate_filters.append("usage_kind = :usage_kind") + params["usage_kind"] = _validate_usage_kind(usage_kind) + rate_rows = db.execute( + text( + f""" + SELECT DISTINCT model, usage_kind + FROM {table} + WHERE {' AND '.join(rate_filters)} + ORDER BY model ASC + LIMIT :limit + """ + ), + params, + ).mappings().all() + for row in rate_rows: + keys.add((row["model"], row["usage_kind"])) + + results: List[Dict[str, Any]] = [] + for model_name, kind in sorted(keys)[:limit]: + results.append( + get_effective_rate( + db, + organization_id=organization_id, + model=model_name, + usage_kind=kind, + as_of=as_of, + ) + ) + return results + + +def upsert_override( + db: Session, + *, + organization_id: UUID, + model: str, + usage_kind: str, + effective_from: date, + effective_to: Optional[date] = None, + rates: Dict[str, Any], + recompute: bool = True, +) -> Dict[str, Any]: + validate_model_name(db, model) + kind = _validate_usage_kind(usage_kind) + if effective_to is not None and effective_to < effective_from: + raise HTTPException(status_code=400, detail="effective_to must be >= effective_from") + + micro_columns = _usd_payload_to_micro_columns(rates) + if not micro_columns: + raise HTTPException(status_code=400, detail="At least one rate field is required") + + params: Dict[str, Any] = { + "organization_id": str(organization_id), + "model": model, + "usage_kind": kind, + "effective_from": effective_from.isoformat(), + "effective_to": effective_to.isoformat() if effective_to else None, + } + for column in RATE_COLUMNS: + params[column] = micro_columns.get(column) + + row = db.execute( + text( + """ + INSERT INTO org_model_pricing_overrides ( + organization_id, model, usage_kind, effective_from, effective_to, + input_micro_usd_per_million, output_micro_usd_per_million, + cache_read_micro_usd_per_million, cache_creation_micro_usd_per_million, + reasoning_micro_usd_per_million, audio_micro_usd_per_second, + tts_micro_usd_per_million_chars + ) VALUES ( + CAST(:organization_id AS uuid), :model, :usage_kind, + CAST(:effective_from AS date), CAST(:effective_to AS date), + :input_micro_usd_per_million, :output_micro_usd_per_million, + :cache_read_micro_usd_per_million, :cache_creation_micro_usd_per_million, + :reasoning_micro_usd_per_million, :audio_micro_usd_per_second, + :tts_micro_usd_per_million_chars + ) + ON CONFLICT (organization_id, model, usage_kind, effective_from) + DO UPDATE SET + effective_to = EXCLUDED.effective_to, + input_micro_usd_per_million = COALESCE( + EXCLUDED.input_micro_usd_per_million, + org_model_pricing_overrides.input_micro_usd_per_million + ), + output_micro_usd_per_million = COALESCE( + EXCLUDED.output_micro_usd_per_million, + org_model_pricing_overrides.output_micro_usd_per_million + ), + cache_read_micro_usd_per_million = COALESCE( + EXCLUDED.cache_read_micro_usd_per_million, + org_model_pricing_overrides.cache_read_micro_usd_per_million + ), + cache_creation_micro_usd_per_million = COALESCE( + EXCLUDED.cache_creation_micro_usd_per_million, + org_model_pricing_overrides.cache_creation_micro_usd_per_million + ), + reasoning_micro_usd_per_million = COALESCE( + EXCLUDED.reasoning_micro_usd_per_million, + org_model_pricing_overrides.reasoning_micro_usd_per_million + ), + audio_micro_usd_per_second = COALESCE( + EXCLUDED.audio_micro_usd_per_second, + org_model_pricing_overrides.audio_micro_usd_per_second + ), + tts_micro_usd_per_million_chars = COALESCE( + EXCLUDED.tts_micro_usd_per_million_chars, + org_model_pricing_overrides.tts_micro_usd_per_million_chars + ), + updated_at = now() + RETURNING * + """ + ), + params, + ).mappings().first() + db.commit() + + invalidate_org_pricing_cache(organization_id) + + recompute_job_id = None + recompute_enqueued = False + if recompute: + recompute_job_id = _enqueue_override_recompute( + db, + organization_id=organization_id, + model=model, + usage_kind=kind, + start_date=effective_from, + end_date=effective_to, + ) + recompute_enqueued = recompute_job_id is not None + + result = _override_row_to_dict(row) + result["recompute_enqueued"] = recompute_enqueued + result["recompute_job_id"] = recompute_job_id + return result + + +def delete_override( + db: Session, + *, + organization_id: UUID, + model: str, + usage_kind: str, + effective_from: Optional[date] = None, + recompute: bool = True, +) -> Dict[str, Any]: + validate_model_name(db, model) + kind = _validate_usage_kind(usage_kind) + filters = [ + "organization_id = CAST(:organization_id AS uuid)", + "model = :model", + "usage_kind = :usage_kind", + ] + params: Dict[str, Any] = { + "organization_id": str(organization_id), + "model": model, + "usage_kind": kind, + } + + if effective_from is not None: + filters.append("effective_from = CAST(:effective_from AS date)") + params["effective_from"] = effective_from.isoformat() + result = db.execute( + text( + f""" + DELETE FROM org_model_pricing_overrides + WHERE {' AND '.join(filters)} + RETURNING effective_from, effective_to + """ + ), + params, + ).mappings().first() + if not result: + raise HTTPException(status_code=404, detail="Override not found") + recompute_from = effective_from + recompute_to = result.get("effective_to") + db.commit() + else: + active = db.execute( + text( + f""" + SELECT id, effective_from, effective_to + FROM org_model_pricing_overrides + WHERE {' AND '.join(filters)} + AND effective_from <= CURRENT_DATE + AND (effective_to IS NULL OR effective_to >= CURRENT_DATE) + ORDER BY effective_from DESC + LIMIT 1 + """ + ), + params, + ).mappings().first() + if not active: + raise HTTPException(status_code=404, detail="No active override found") + recompute_from = active["effective_from"] + recompute_to = active.get("effective_to") + yesterday = date.today() - timedelta(days=1) + if active["effective_from"] > yesterday: + db.execute( + text( + """ + DELETE FROM org_model_pricing_overrides + WHERE id = CAST(:id AS uuid) + """ + ), + {"id": str(active["id"])}, + ) + else: + db.execute( + text( + """ + UPDATE org_model_pricing_overrides + SET effective_to = CAST(:effective_to AS date), updated_at = now() + WHERE id = CAST(:id AS uuid) + """ + ), + {"id": str(active["id"]), "effective_to": yesterday.isoformat()}, + ) + db.commit() + + invalidate_org_pricing_cache(organization_id) + + recompute_job_id = None + recompute_enqueued = False + if recompute: + recompute_job_id = _enqueue_override_recompute( + db, + organization_id=organization_id, + model=model, + usage_kind=kind, + start_date=recompute_from, + end_date=recompute_to, + ) + recompute_enqueued = recompute_job_id is not None + + return { + "deleted": True, + "model": model, + "usage_kind": kind, + "recompute_enqueued": recompute_enqueued, + "recompute_job_id": recompute_job_id, + } + + +def _enqueue_override_recompute( + db: Session, + *, + organization_id: UUID, + model: str, + usage_kind: str, + start_date: date, + end_date: Optional[date], +) -> Optional[str]: + try: + job = create_recompute_job( + db, + organization_id=organization_id, + model=model, + usage_kind=usage_kind, + start_date=start_date, + end_date=end_date, + ) + enqueue_recompute_job(db, job) + return str(job.id) + except HTTPException as exc: + if exc.status_code == 409: + return None + raise diff --git a/app/services/usage/usage_costs.py b/app/services/usage/usage_costs.py new file mode 100644 index 00000000..914de0b1 --- /dev/null +++ b/app/services/usage/usage_costs.py @@ -0,0 +1,51 @@ +"""Usage cost presentation helpers for API responses.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +MICRO_USD_PER_DOLLAR = 1_000_000 + + +def micro_to_usd(micro: int) -> float: + return micro / MICRO_USD_PER_DOLLAR + + +def costs_from_micro( + *, + input_cost_micro_usd: int = 0, + output_cost_micro_usd: int = 0, + cache_read_cost_micro_usd: int = 0, + cache_creation_cost_micro_usd: int = 0, + reasoning_cost_micro_usd: int = 0, + audio_cost_micro_usd: int = 0, + tts_cost_micro_usd: int = 0, + total_cost_micro_usd: Optional[int] = None, + has_unpriced_usage: bool = False, + currency: str = "USD", +) -> Dict[str, Any]: + total_micro = ( + total_cost_micro_usd + if total_cost_micro_usd is not None + else ( + input_cost_micro_usd + + output_cost_micro_usd + + cache_read_cost_micro_usd + + cache_creation_cost_micro_usd + + reasoning_cost_micro_usd + + audio_cost_micro_usd + + tts_cost_micro_usd + ) + ) + return { + "input_cost_usd": micro_to_usd(input_cost_micro_usd), + "output_cost_usd": micro_to_usd(output_cost_micro_usd), + "cache_read_cost_usd": micro_to_usd(cache_read_cost_micro_usd), + "cache_write_cost_usd": micro_to_usd(cache_creation_cost_micro_usd), + "reasoning_cost_usd": micro_to_usd(reasoning_cost_micro_usd), + "audio_cost_usd": micro_to_usd(audio_cost_micro_usd), + "tts_cost_usd": micro_to_usd(tts_cost_micro_usd), + "total_cost_usd": micro_to_usd(total_micro), + "currency": currency, + "has_unpriced_usage": has_unpriced_usage, + } diff --git a/app/workers/config.py b/app/workers/config.py index 3448429d..e06336d0 100644 --- a/app/workers/config.py +++ b/app/workers/config.py @@ -74,6 +74,7 @@ # Queues consumed by the dedicated call-import / evaluation worker. IMPORTS_WORKER_QUEUES = "imports,diarization,eval-control,evaluations" EVAL_CONTROL_QUEUE = "eval-control" +USAGE_WORKER_QUEUE = "usage" # Create Celery app celery_app = Celery( @@ -137,13 +138,16 @@ "generate_evaluation_prompt_improvements": {"queue": "evaluations"}, "generate_agent_flowchart": {"queue": "celery"}, "map_agent_flowchart_prompt_sections": {"queue": "celery"}, - "flush_usage_counters": {"queue": "celery"}, + "flush_usage_counters": {"queue": USAGE_WORKER_QUEUE}, + "recompute_usage_costs": {"queue": USAGE_WORKER_QUEUE}, } -# Periodic flush of Redis LLM usage counters into catalog rollups. +# Periodic flush of Redis usage counters into catalog rollups (usage queue). +_flush_interval = float(os.environ.get("USAGE_FLUSH_BEAT_SECONDS", "120")) celery_app.conf.beat_schedule = { "flush-llm-usage-counters": { "task": "flush_usage_counters", - "schedule": 120.0, + "schedule": _flush_interval, + "options": {"queue": USAGE_WORKER_QUEUE}, }, } diff --git a/app/workers/tasks/__init__.py b/app/workers/tasks/__init__.py index 04eb0380..499d9a5d 100644 --- a/app/workers/tasks/__init__.py +++ b/app/workers/tasks/__init__.py @@ -23,6 +23,7 @@ from . import finalize_telephony_recording from . import call_import_bulk_ops from . import flush_usage_counters +from . import recompute_usage_costs from app.workers.concurrency import eval_dispatch from app.workers.concurrency import fair_dispatch from app.workers.concurrency import fair_diarization_dispatch @@ -117,3 +118,4 @@ call_import_bulk_ops.materialize_call_import_evaluation_task ) flush_usage_counters_task = flush_usage_counters.flush_usage_counters_task +recompute_usage_costs_task = recompute_usage_costs.recompute_usage_costs_task diff --git a/app/workers/tasks/recompute_usage_costs.py b/app/workers/tasks/recompute_usage_costs.py new file mode 100644 index 00000000..8340f797 --- /dev/null +++ b/app/workers/tasks/recompute_usage_costs.py @@ -0,0 +1,79 @@ +"""Celery task: recompute stored usage costs for rollup rows.""" + +from __future__ import annotations + +from datetime import date +from typing import Optional +from uuid import UUID + +from loguru import logger + +from app.database import SessionLocal +from app.workers.config import celery_app + + +@celery_app.task(name="recompute_usage_costs") +def recompute_usage_costs_task( + job_id: Optional[str] = None, + organization_id: Optional[str] = None, + model: Optional[str] = None, + usage_kind: Optional[str] = None, + start_date: Optional[str] = None, + end_date: Optional[str] = None, +) -> dict: + from app.models.database import UsageCostRecomputeJob + from app.services.usage.pricing import recompute_usage_costs + from app.services.usage.pricing_jobs import ( + mark_job_completed, + mark_job_failed, + mark_job_running, + update_job_progress, + ) + + db = SessionLocal() + try: + if job_id: + job = ( + db.query(UsageCostRecomputeJob) + .filter(UsageCostRecomputeJob.id == UUID(job_id)) + .first() + ) + if job is None: + return {"updated_rows": 0, "error": "job not found"} + mark_job_running(db, job.id) + org_uuid = job.organization_id + model = job.model + usage_kind = job.usage_kind + start = job.start_date + end = job.end_date + progress_job_id = job.id + else: + org_uuid = UUID(organization_id) if organization_id else None + start = date.fromisoformat(start_date) if start_date else None + end = date.fromisoformat(end_date) if end_date else None + progress_job_id = None + + def _on_progress(updated_rows: int) -> None: + if progress_job_id is not None: + update_job_progress(db, progress_job_id, updated_rows) + + updated = recompute_usage_costs( + db, + organization_id=org_uuid, + model=model, + usage_kind=usage_kind, + start_date=start, + end_date=end, + on_progress=_on_progress if progress_job_id else None, + ) + if progress_job_id is not None: + mark_job_completed(db, progress_job_id, updated) + if updated: + logger.info("Recomputed usage costs for {} rollup row(s)", updated) + return {"updated_rows": updated, "job_id": job_id} + except Exception as exc: + if job_id: + mark_job_failed(db, UUID(job_id), str(exc)) + raise + finally: + db.close() diff --git a/docker-compose.observability.yml b/docker-compose.observability.yml index 402ac2e0..35b84190 100644 --- a/docker-compose.observability.yml +++ b/docker-compose.observability.yml @@ -62,6 +62,17 @@ services: loki-max-backoff: "800ms" loki-external-labels: "service=worker-imports,environment=docker" + worker-usage: + depends_on: + - loki + logging: + driver: loki + options: + loki-url: "http://localhost:3100/loki/api/v1/push" + loki-retries: "5" + loki-max-backoff: "800ms" + loki-external-labels: "service=worker-usage,environment=docker" + # Pin image tags instead of :latest to reduce exposure to stale third-party # binaries (e.g. Go stdlib CVEs in observability tooling). loki: diff --git a/docker-compose.yml b/docker-compose.yml index 552884f3..13be6295 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -193,5 +193,62 @@ services: # (each task may hold catalog + shard connections for tens of seconds). command: eai worker --config /app/config.yml --loglevel info --queues imports,diarization,eval-control,evaluations --pool threads --concurrency 12 + # Low-priority usage pricing: Redis flush + cost recompute/backfill. + worker-usage: + image: ghcr.io/efficientai-tech/efficientai-worker:${EFFICIENTAI_VERSION:-latest} + build: + context: . + dockerfile: docker/Dockerfile.worker + args: + INSTALL_EXTRAS: "" + container_name: efficientai_worker_usage + env_file: + - .env + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-efficientai}:${POSTGRES_PASSWORD:-password}@db:5432/${POSTGRES_DB:-efficientai} + REDIS_URL: redis://redis:6379/0 + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + ENCRYPTION_KEY: ${ENCRYPTION_KEY:-} + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + volumes: + - ./uploads:/app/uploads + - ./.data:/app/.data + - ./config.docker.yml:/app/config.yml:ro + command: eai worker --config /app/config.yml --loglevel info --queues usage --pool threads --concurrency 4 + + beat: + image: ghcr.io/efficientai-tech/efficientai-worker:${EFFICIENTAI_VERSION:-latest} + build: + context: . + dockerfile: docker/Dockerfile.worker + args: + INSTALL_EXTRAS: "" + container_name: efficientai_beat + env_file: + - .env + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-efficientai}:${POSTGRES_PASSWORD:-password}@db:5432/${POSTGRES_DB:-efficientai} + REDIS_URL: redis://redis:6379/0 + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + ENCRYPTION_KEY: ${ENCRYPTION_KEY:-} + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + worker-usage: + condition: service_started + volumes: + - ./uploads:/app/uploads + - ./.data:/app/.data + - ./config.docker.yml:/app/config.yml:ro + command: celery -A app.workers.celery_app beat --loglevel info + volumes: postgres_data: diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index d18118b0..287bdd2d 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -13,6 +13,7 @@ import Dashboard from './pages/dashboard/Dashboard' // Usage import Usage from './pages/usage/Usage' +import UsagePricing from './pages/usage/UsagePricing' // Prompt Partials import PromptPartials from './pages/promptPartials/PromptPartials' @@ -186,6 +187,7 @@ function App() { } /> } /> } /> + } /> } diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 7e2c4369..6dfe8744 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -3017,6 +3017,26 @@ class ApiClient { audio_seconds: number tts_characters: number call_count: number + input_cost_micro_usd?: number + output_cost_micro_usd?: number + cache_read_cost_micro_usd?: number + cache_creation_cost_micro_usd?: number + reasoning_cost_micro_usd?: number + audio_cost_micro_usd?: number + tts_cost_micro_usd?: number + total_cost_micro_usd?: number + costs?: { + input_cost_usd: number + output_cost_usd: number + cache_read_cost_usd: number + cache_write_cost_usd: number + reasoning_cost_usd: number + audio_cost_usd: number + tts_cost_usd: number + total_cost_usd: number + currency: string + has_unpriced_usage: boolean + } } last_updated_at?: string | null }> { @@ -3067,6 +3087,26 @@ class ApiClient { audio_seconds: number tts_characters: number call_count: number + input_cost_micro_usd?: number + output_cost_micro_usd?: number + cache_read_cost_micro_usd?: number + cache_creation_cost_micro_usd?: number + reasoning_cost_micro_usd?: number + audio_cost_micro_usd?: number + tts_cost_micro_usd?: number + total_cost_micro_usd?: number + costs?: { + input_cost_usd: number + output_cost_usd: number + cache_read_cost_usd: number + cache_write_cost_usd: number + reasoning_cost_usd: number + audio_cost_usd: number + tts_cost_usd: number + total_cost_usd: number + currency: string + has_unpriced_usage: boolean + } }> total_count: number truncated_at_limit?: boolean @@ -3078,6 +3118,135 @@ class ApiClient { return response.data } + async listUsagePricingOverrides(params?: { + model?: string + usage_kind?: string + }): Promise< + Array<{ + id: string + organization_id: string + model: string + usage_kind: string + effective_from: string + effective_to?: string | null + rates: { + input_per_1m?: number | null + output_per_1m?: number | null + cache_read_per_1m?: number | null + cache_write_per_1m?: number | null + reasoning_per_1m?: number | null + audio_per_minute?: number | null + tts_per_1m_characters?: number | null + } + recompute_enqueued?: boolean + recompute_job_id?: string | null + }> + > { + const response = await this.client.get('/api/v1/organizations/usage/pricing/overrides', { + params, + }) + return response.data + } + + async upsertUsagePricingOverride( + model: string, + body: { + usage_kind?: string + effective_from: string + effective_to?: string + rates: { + input_per_1m?: number + output_per_1m?: number + cache_read_per_1m?: number + cache_write_per_1m?: number + reasoning_per_1m?: number + audio_per_minute?: number + tts_per_1m_characters?: number + } + recompute?: boolean + } + ): Promise<{ + id: string + model: string + usage_kind: string + effective_from: string + effective_to?: string | null + rates: Record + recompute_enqueued?: boolean + recompute_job_id?: string | null + }> { + const response = await this.client.put( + `/api/v1/organizations/usage/pricing/overrides/${encodeURIComponent(model)}`, + body + ) + return response.data + } + + async deleteUsagePricingOverride( + model: string, + params?: { usage_kind?: string; effective_from?: string; recompute?: boolean } + ): Promise<{ + deleted: boolean + model: string + usage_kind: string + recompute_enqueued?: boolean + recompute_job_id?: string | null + }> { + const response = await this.client.delete( + `/api/v1/organizations/usage/pricing/overrides/${encodeURIComponent(model)}`, + { params } + ) + return response.data + } + + async triggerUsageCostRecompute(body?: { + start_date?: string + end_date?: string + model?: string + usage_kind?: string + }): Promise<{ + id: string + organization_id: string + status: string + model?: string | null + usage_kind?: string | null + start_date?: string | null + end_date?: string | null + updated_rows: number + error_message?: string | null + celery_task_id?: string | null + created_at?: string | null + updated_at?: string | null + completed_at?: string | null + }> { + const response = await this.client.post( + '/api/v1/organizations/usage/pricing/recompute', + body ?? {} + ) + return response.data + } + + async getUsageCostRecomputeJob(jobId: string): Promise<{ + id: string + organization_id: string + status: string + model?: string | null + usage_kind?: string | null + start_date?: string | null + end_date?: string | null + updated_rows: number + error_message?: string | null + celery_task_id?: string | null + created_at?: string | null + updated_at?: string | null + completed_at?: string | null + }> { + const response = await this.client.get( + `/api/v1/organizations/usage/pricing/recompute/${jobId}` + ) + return response.data + } + async getOrgUsageFilters(params?: { start?: string end?: string diff --git a/frontend/src/pages/usage/SearchableSelect.tsx b/frontend/src/pages/usage/SearchableSelect.tsx index 66f625dd..61da046a 100644 --- a/frontend/src/pages/usage/SearchableSelect.tsx +++ b/frontend/src/pages/usage/SearchableSelect.tsx @@ -27,7 +27,9 @@ export default function SearchableSelect({ const [search, setSearch] = useState('') const rootRef = useRef(null) - const selected = options.find((o) => o.id === value) + const selected = options.find( + (o) => o.id === value || o.id.toLowerCase() === value.toLowerCase(), + ) const filtered = useMemo(() => { const q = search.trim().toLowerCase() @@ -102,7 +104,8 @@ export default function SearchableSelect({ + ) : null} + {isAdmin ? ( + + Pricing overrides + + ) : null} +
+ {summary?.last_updated_at ? ( +

+ Updated {new Date(summary.last_updated_at).toLocaleString()} +

+ ) : null} +
- - - - -
+
+ + + + +
- {(showAudio || - showTts || - totals?.cache_read_tokens || - totals?.cache_creation_tokens || - totals?.reasoning_tokens) ? ( -
- {showAudio ? ( +
+
+ {showAudio ? ( + + ) : null} + {showTts ? ( + + ) : null} + {(totals?.cache_read_tokens || 0) > 0 ? ( + + ) : null} + {(totals?.cache_creation_tokens || 0) > 0 ? ( + + ) : null} + {(totals?.reasoning_tokens || 0) > 0 ? ( + + ) : null} +
+
- ) : null} - {showTts ? ( - - ) : null} - {(totals?.cache_read_tokens || 0) > 0 ? ( - - ) : null} - {(totals?.cache_creation_tokens || 0) > 0 ? ( - - ) : null} - {(totals?.reasoning_tokens || 0) > 0 ? ( - - ) : null} +
- ) : null} +
setParams({ start: s, end: e })} @@ -1068,7 +1172,20 @@ export default function Usage() { }) } onEvaluationChange={(id) => { - const resource = filterOptions?.resources?.find((r) => r.id === id) + if (id.startsWith(USAGE_SECTION_SOURCE_PREFIX)) { + const section = id.slice(USAGE_SECTION_SOURCE_PREFIX.length) + setParams({ + product_section: section || null, + resource_id: null, + call_import_id: null, + model: null, + usage_kind: null, + }) + return + } + const resource = filterOptions?.resources?.find( + (r) => idKey(r.id) === idKey(id), + ) setParams({ resource_id: id || null, product_section: id ? resource?.product_section || null : null, @@ -1122,6 +1239,7 @@ export default function Usage() { Input tokens Output tokens Total tokens + Est. cost STT audio TTS chars Cache read @@ -1189,6 +1307,9 @@ export default function Usage() { {formatNumber(row.total_tokens)} + + {formatCostUsd(rowCostUsd(row))} + {row.audio_seconds ? formatAudio(row.audio_seconds) : '—'} @@ -1207,28 +1328,46 @@ export default function Usage() { )} + + setCostBreakdownOpen(false)} + costs={totals?.costs} + scopeLabel={scopeSubtitle} + />
) } +const statCardClass = 'border border-gray-200 ring-1 ring-[#fde047]/25 shadow-sm' + function StatCard({ label, value, valueLabel, loading, + emphasize = false, + className = '', }: { label: string value?: number valueLabel?: string loading?: boolean + emphasize?: boolean + className?: string }) { return ( - + -

- {label} -

-

+

{label}

+

{loading ? '—' : valueLabel ?? formatNumber(value || 0)}

diff --git a/frontend/src/pages/usage/UsageCostBreakdownModal.tsx b/frontend/src/pages/usage/UsageCostBreakdownModal.tsx new file mode 100644 index 00000000..9277de53 --- /dev/null +++ b/frontend/src/pages/usage/UsageCostBreakdownModal.tsx @@ -0,0 +1,133 @@ +import { X } from 'lucide-react' + +type UsageCosts = { + input_cost_usd: number + output_cost_usd: number + cache_read_cost_usd: number + cache_write_cost_usd: number + reasoning_cost_usd: number + audio_cost_usd: number + tts_cost_usd: number + total_cost_usd: number + has_unpriced_usage?: boolean +} + +type Props = { + isOpen: boolean + onClose: () => void + costs?: UsageCosts | null + scopeLabel?: string +} + +function formatCostUsd(usd?: number | null): string { + const amount = Number(usd || 0) + if (!amount) return '$0.00' + return new Intl.NumberFormat(undefined, { + style: 'currency', + currency: 'USD', + minimumFractionDigits: 2, + maximumFractionDigits: 4, + }).format(amount) +} + +const LINE_ITEMS: Array<{ key: keyof UsageCosts; label: string }> = [ + { key: 'input_cost_usd', label: 'Input tokens' }, + { key: 'output_cost_usd', label: 'Output tokens' }, + { key: 'cache_read_cost_usd', label: 'Cache read' }, + { key: 'cache_write_cost_usd', label: 'Cache write' }, + { key: 'reasoning_cost_usd', label: 'Reasoning' }, + { key: 'audio_cost_usd', label: 'Audio / STT' }, + { key: 'tts_cost_usd', label: 'TTS' }, +] + +export default function UsageCostBreakdownModal({ + isOpen, + onClose, + costs, + scopeLabel, +}: Props) { + if (!isOpen) return null + + const rows = LINE_ITEMS.filter((item) => Number(costs?.[item.key] || 0) > 0) + + return ( +
+
+
+
+
+
+

+ Cost breakdown +

+ {scopeLabel ? ( +

{scopeLabel}

+ ) : null} +
+ +
+ + {costs?.has_unpriced_usage ? ( +

+ Some usage in this range has no pricing rate — estimated cost may be understated. +

+ ) : null} + +
+
+ Estimated total + + {formatCostUsd(costs?.total_cost_usd)} + +
+ {rows.length > 0 ? ( +
    + {rows.map((item) => ( +
  • + {item.label} + + {formatCostUsd(costs?.[item.key] as number)} + +
    • + ))} +
+ ) : ( +

+ No per-metric cost detail for this range. +

+ )} +
+ +
+ +
+
+
+
+ ) +} diff --git a/frontend/src/pages/usage/UsageFiltersBar.tsx b/frontend/src/pages/usage/UsageFiltersBar.tsx index 747fc2ed..eb736667 100644 --- a/frontend/src/pages/usage/UsageFiltersBar.tsx +++ b/frontend/src/pages/usage/UsageFiltersBar.tsx @@ -4,6 +4,7 @@ import { ChevronDown, Filter, SlidersHorizontal } from 'lucide-react' import SearchableSelect from './SearchableSelect' import UsageDateRangePicker from './UsageDateRangePicker' import { usageTheme } from './usageTheme' +import { CALL_IMPORT_PRODUCT_SECTIONS, USAGE_SECTION_SOURCE_PREFIX } from './usageProductHints' type Kind = '' | 'llm' | 'stt' | 'tts' @@ -12,6 +13,7 @@ type FilterOptions = { call_imports: Array<{ id: string; label: string }> evaluations: Array<{ id: string; label: string }> resources?: Array<{ id: string; label: string; type?: string; product_section?: string }> + product_sections?: Array<{ id: string; label: string }> models: string[] usage_kinds: Array<{ id: string; label: string }> datasets?: string[] @@ -30,6 +32,7 @@ type UsageFiltersBarProps = { tagId: string usageKind: Kind model: string + productSection?: string options?: FilterOptions filtersLoading?: boolean onDateApply: (start: string, end: string) => void @@ -60,6 +63,7 @@ export default function UsageFiltersBar({ tagId, usageKind, model, + productSection = '', options, filtersLoading, onDateApply, @@ -86,7 +90,20 @@ export default function UsageFiltersBar({ [options?.usage_kinds], ) - const sourceOptions = callImportId ? evaluations : resources + const sectionSourceOptions = useMemo(() => { + if (callImportId) return [] + return (options?.product_sections ?? []) + .filter((s) => !CALL_IMPORT_PRODUCT_SECTIONS.has(s.id)) + .map((s) => ({ + id: `${USAGE_SECTION_SOURCE_PREFIX}${s.id}`, + label: s.label, + })) + }, [callImportId, options?.product_sections]) + + const sourceOptions = callImportId ? evaluations : [...sectionSourceOptions, ...resources] + const sourceSelectValue = + evaluationId || + (productSection ? `${USAGE_SECTION_SOURCE_PREFIX}${productSection}` : '') const activeChips = useMemo((): ActiveChip[] => { const chips: ActiveChip[] = [] @@ -124,10 +141,10 @@ export default function UsageFiltersBar({ } } const sourceLabel = - sourceOptions.find((e) => e.id === evaluationId)?.label || + sourceOptions.find((e) => e.id === sourceSelectValue)?.label || evaluations.find((e) => e.id === evaluationId)?.label || resources.find((r) => r.id === evaluationId)?.label - if (evaluationId && sourceLabel) { + if ((evaluationId || productSection) && sourceLabel) { chips.push({ key: 'evaluation', label: sourceLabel, @@ -158,6 +175,8 @@ export default function UsageFiltersBar({ tagId, tags, evaluationId, + productSection, + sourceSelectValue, sourceOptions, evaluations, resources, @@ -286,7 +305,7 @@ export default function UsageFiltersBar({ ? 'All evaluations for import' : 'Agents, voice sims, telephony, …' } - value={evaluationId} + value={sourceSelectValue} options={sourceOptions} onChange={onEvaluationChange} emptyMessage="No matching usage in this range" diff --git a/frontend/src/pages/usage/UsagePricing.tsx b/frontend/src/pages/usage/UsagePricing.tsx new file mode 100644 index 00000000..f976d116 --- /dev/null +++ b/frontend/src/pages/usage/UsagePricing.tsx @@ -0,0 +1,393 @@ +import { useMemo, useState, type ReactNode } from 'react' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { Card, CardBody, Spinner } from '@heroui/react' +import { Link } from 'react-router-dom' +import { ArrowLeft, DollarSign, Info, Plus, Trash2 } from 'lucide-react' +import { apiClient } from '../../lib/api' +import { useIsAdmin } from '../../hooks/useRole' +import { useToast } from '../../hooks/useToast' +import Button from '../../components/Button' +import SearchableSelect from './SearchableSelect' +import { usageTheme } from './usageTheme' + +type UsageKind = 'llm' | 'stt' | 'tts' + +type PricingRatesUsd = { + input_per_1m?: number | null + output_per_1m?: number | null + cache_read_per_1m?: number | null + cache_write_per_1m?: number | null + reasoning_per_1m?: number | null + audio_per_minute?: number | null + tts_per_1m_characters?: number | null +} + +type PricingOverride = { + id: string + model: string + usage_kind: string + effective_from: string + effective_to?: string | null + rates: PricingRatesUsd + recompute_enqueued?: boolean + recompute_job_id?: string | null +} + +type RateFieldKey = keyof PricingRatesUsd + +const EMPTY_RATES: PricingRatesUsd = { + input_per_1m: undefined, + output_per_1m: undefined, + cache_read_per_1m: undefined, + cache_write_per_1m: undefined, + reasoning_per_1m: undefined, + audio_per_minute: undefined, + tts_per_1m_characters: undefined, +} + +const RATE_FIELDS: Array<{ key: RateFieldKey; label: string; kinds: UsageKind[] }> = [ + { key: 'input_per_1m', label: 'Input / 1M tokens (USD)', kinds: ['llm'] }, + { key: 'output_per_1m', label: 'Output / 1M tokens (USD)', kinds: ['llm'] }, + { key: 'cache_read_per_1m', label: 'Cache read / 1M (USD)', kinds: ['llm'] }, + { key: 'cache_write_per_1m', label: 'Cache write / 1M (USD)', kinds: ['llm'] }, + { key: 'reasoning_per_1m', label: 'Reasoning / 1M (USD)', kinds: ['llm'] }, + { key: 'audio_per_minute', label: 'Audio / minute (USD)', kinds: ['llm', 'stt'] }, + { key: 'tts_per_1m_characters', label: 'TTS / 1M characters (USD)', kinds: ['tts'] }, +] + +const CONTROL_CLASS = `h-10 w-full rounded-lg border border-gray-200 bg-white px-3 text-sm text-gray-900 shadow-sm outline-none transition-colors hover:border-gray-300 ${usageTheme.focusRing}` + +function FormField({ + label, + children, + className = '', +}: { + label: string + children: ReactNode + className?: string +}) { + return ( +
+ {label} +
{children}
+
+ ) +} + +function formatUsd(value?: number | null): string { + if (value == null || Number.isNaN(value)) return '—' + return new Intl.NumberFormat(undefined, { + style: 'currency', + currency: 'USD', + minimumFractionDigits: 2, + maximumFractionDigits: 6, + }).format(value) +} + +function todayIso(): string { + return new Date().toISOString().slice(0, 10) +} + +export default function UsagePricing() { + const isAdmin = useIsAdmin() + const queryClient = useQueryClient() + const { showToast, ToastContainer } = useToast() + + const [model, setModel] = useState('') + const [usageKind, setUsageKind] = useState('llm') + const [effectiveFrom, setEffectiveFrom] = useState(todayIso()) + const [rates, setRates] = useState(EMPTY_RATES) + + const { data: filters } = useQuery({ + queryKey: ['org-usage', 'filters'], + queryFn: () => apiClient.getOrgUsageFilters(), + enabled: isAdmin, + }) + + const { data: overrides = [], isLoading } = useQuery({ + queryKey: ['usage-pricing', 'overrides'], + queryFn: () => apiClient.listUsagePricingOverrides(), + enabled: isAdmin, + }) + + const modelOptions = useMemo(() => { + const fromUsage = filters?.models || [] + const fromOverrides = overrides.map((row) => row.model) + return Array.from(new Set([...fromUsage, ...fromOverrides])) + .sort() + .map((name) => ({ id: name, label: name })) + }, [filters?.models, overrides]) + + const visibleRateFields = useMemo( + () => RATE_FIELDS.filter((field) => field.kinds.includes(usageKind)), + [usageKind] + ) + + const saveMutation = useMutation({ + mutationFn: () => { + const payloadRates = Object.fromEntries( + Object.entries(rates).filter((entry): entry is [string, number] => { + const value = entry[1] + return value != null && !Number.isNaN(value) + }) + ) + return apiClient.upsertUsagePricingOverride(model, { + usage_kind: usageKind, + effective_from: effectiveFrom, + rates: payloadRates, + recompute: true, + }) + }, + onSuccess: (data) => { + queryClient.invalidateQueries({ queryKey: ['usage-pricing'] }) + queryClient.invalidateQueries({ queryKey: ['org-usage'] }) + setRates(EMPTY_RATES) + showToast( + data.recompute_enqueued + ? 'Override saved; cost recompute started' + : 'Override saved (recompute already running)', + 'success' + ) + }, + onError: (error: any) => { + showToast(error.response?.data?.detail || 'Failed to save override', 'error') + }, + }) + + const deleteMutation = useMutation({ + mutationFn: (row: PricingOverride) => + apiClient.deleteUsagePricingOverride(row.model, { + usage_kind: row.usage_kind, + effective_from: row.effective_from, + }), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['usage-pricing'] }) + queryClient.invalidateQueries({ queryKey: ['org-usage'] }) + showToast('Override removed', 'success') + }, + onError: (error: any) => { + showToast(error.response?.data?.detail || 'Failed to delete override', 'error') + }, + }) + + if (!isAdmin) { + return ( +
+ Organization admin access is required to manage pricing overrides. +
+ ) + } + + return ( +
+ + +
+ + + Back to usage + +
+
+ +
+
+
+

Pricing overrides

+
+ +
+ Set custom USD rates for this organization. Empty fields inherit the platform + catalog. Saving updates costs for matching usage and starts a scoped recompute. +
+
+
+

+ Per-model rates for this org. Leave blank to use the default catalog. +

+
+
+
+ + + +
+

Add or update override

+

+ Saving triggers a scoped cost recompute for this model. +

+
+ +
+ + + + + + setEffectiveFrom(e.target.value)} + /> + +
+ +
+

+ Rate overrides (USD) +

+
+ {visibleRateFields.map(({ key, label }) => ( + + { + const value = e.target.value + setRates((prev) => ({ + ...prev, + [key]: value === '' ? undefined : Number(value), + })) + }} + /> + + ))} +
+
+ +
+ + +
+ + + + + +
+

Current overrides

+ {!isLoading ? ( + {overrides.length} total + ) : null} +
+ + {isLoading ? ( +
+ +
+ ) : overrides.length === 0 ? ( +

+ No overrides yet. Platform catalog rates apply to all usage. +

+ ) : ( +
+ + + + + + + + + + + + + + {overrides.map((row) => ( + + + + + + + + + + + ))} + +
ModelKindEffectiveInputOutputAudioTTS +
+ {row.model} + {row.usage_kind} + {row.effective_from} + {row.effective_to ? ` → ${row.effective_to}` : ''} + + {formatUsd(row.rates.input_per_1m)} + + {formatUsd(row.rates.output_per_1m)} + + {formatUsd(row.rates.audio_per_minute)} + + {formatUsd(row.rates.tts_per_1m_characters)} + + +
+
+ )} + + +
+ ) +} diff --git a/frontend/src/pages/usage/usageProductHints.ts b/frontend/src/pages/usage/usageProductHints.ts index 4a057b1c..db382f23 100644 --- a/frontend/src/pages/usage/usageProductHints.ts +++ b/frontend/src/pages/usage/usageProductHints.ts @@ -45,3 +45,6 @@ export const PRODUCT_SECTION_HINTS: Record = { export const CALL_IMPORT_HINT = 'Call import batch — CSV upload or manual audio recordings' + +/** Prefix for product-area rows in the Source filter (not a resource UUID). */ +export const USAGE_SECTION_SOURCE_PREFIX = 'section:' diff --git a/scripts/merge_pricing_into_models_json.py b/scripts/merge_pricing_into_models_json.py new file mode 100644 index 00000000..c31060dd --- /dev/null +++ b/scripts/merge_pricing_into_models_json.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Merge pricing_catalog.json micro-USD rates into models.json plan-format pricing blocks.""" + +from __future__ import annotations + +import json +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +MODELS_JSON = REPO_ROOT / "app" / "config" / "models.json" +CATALOG_JSON = REPO_ROOT / "app" / "config" / "pricing_catalog.json" +MANUAL_JSON = REPO_ROOT / "app" / "config" / "pricing_manual.json" +MICRO = 1_000_000 + + +def _usd_per_m(micro: int) -> float: + return round(micro / MICRO, 8) + + +def _micro_entry_to_plan_pricing(entry: dict) -> dict: + usage_kind = entry.get("usage_kind") or "llm" + source = entry.get("_price_source") or ( + "litellm_import" if entry.get("_litellm_key") else "catalog" + ) + pricing: dict = {"source": source} + if usage_kind: + pricing["usage_kind"] = usage_kind + + if entry.get("input_micro_usd_per_million"): + pricing["input_per_1m"] = _usd_per_m(entry["input_micro_usd_per_million"]) + if entry.get("output_micro_usd_per_million"): + pricing["output_per_1m"] = _usd_per_m(entry["output_micro_usd_per_million"]) + if entry.get("cache_read_micro_usd_per_million"): + pricing["cache_read_per_1m"] = _usd_per_m( + entry["cache_read_micro_usd_per_million"] + ) + if entry.get("cache_creation_micro_usd_per_million"): + pricing["cache_write_per_1m"] = _usd_per_m( + entry["cache_creation_micro_usd_per_million"] + ) + if entry.get("reasoning_micro_usd_per_million"): + pricing["reasoning_per_1m"] = _usd_per_m( + entry["reasoning_micro_usd_per_million"] + ) + if entry.get("audio_micro_usd_per_second"): + pricing["audio_per_minute"] = round( + entry["audio_micro_usd_per_second"] * 60 / MICRO, 8 + ) + if entry.get("tts_micro_usd_per_million_chars"): + pricing["tts_per_1m_characters"] = _usd_per_m( + entry["tts_micro_usd_per_million_chars"] + ) + return pricing + + +def _load_catalog() -> dict: + merged: dict = {} + if CATALOG_JSON.exists(): + payload = json.loads(CATALOG_JSON.read_text(encoding="utf-8")) + merged.update( + { + k: v + for k, v in payload.items() + if not k.startswith("_") and isinstance(v, dict) + } + ) + if MANUAL_JSON.exists(): + payload = json.loads(MANUAL_JSON.read_text(encoding="utf-8")) + for key, value in payload.items(): + if not key.startswith("_") and isinstance(value, dict): + merged[key] = value + return merged + + +def merge() -> tuple[int, int]: + models = json.loads(MODELS_JSON.read_text(encoding="utf-8")) + catalog = _load_catalog() + updated = 0 + skipped = 0 + for model_name, cfg in models.items(): + if model_name.startswith("_") or not isinstance(cfg, dict): + continue + entry = catalog.get(model_name) + if not entry: + skipped += 1 + continue + pricing = _micro_entry_to_plan_pricing(entry) + if len(pricing) <= 1: + skipped += 1 + continue + cfg["pricing"] = pricing + updated += 1 + MODELS_JSON.write_text( + json.dumps(models, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + return updated, skipped + + +def main() -> int: + updated, skipped = merge() + print(f"merged pricing into {updated} model(s); skipped {skipped}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/sync_pricing_catalog_from_litellm.py b/scripts/sync_pricing_catalog_from_litellm.py new file mode 100644 index 00000000..9d6375aa --- /dev/null +++ b/scripts/sync_pricing_catalog_from_litellm.py @@ -0,0 +1,428 @@ +#!/usr/bin/env python3 +"""Generate app/config/pricing_catalog.json from models.json + LiteLLM model_cost.""" + +from __future__ import annotations + +import argparse +import json +import importlib.util +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +REPO_ROOT = Path(__file__).resolve().parents[1] +MODELS_JSON = REPO_ROOT / "app" / "config" / "models.json" +MANUAL_JSON = REPO_ROOT / "app" / "config" / "pricing_manual.json" +OUTPUT_JSON = REPO_ROOT / "app" / "config" / "pricing_catalog.json" + +LITELLM_PROVIDER_PREFIX = { + "openai": "openai", + "anthropic": "anthropic", + "google": "gemini", + "azure": "azure", + "aws": "bedrock", + "deepseek": "deepseek", + "groq": "groq", + "xai": "xai", + "fireworks": "fireworks_ai", + "sarvam": "sarvam", + "deepgram": "deepgram", + "elevenlabs": "elevenlabs", + "cartesia": "cartesia", +} + +# Catalog key -> LiteLLM model_cost key (when auto-resolution fails). +EXPLICIT_LITELLM_KEYS: Dict[str, str] = { + "chat-latest": "gpt-5-chat-latest", + "azure-openai-gpt4": "gpt-4", + "aws-bedrock-claude": "anthropic.claude-sonnet-4-20250514-v1:0", + "aws-transcribe": "whisper-1", + "aws-polly": "aws_polly/neural", + "google-speech-v2": "gemini-2.5-flash", + "gemini-2.5-pro-stt": "gemini-2.5-flash", + "gemini-2.5-flash-stt": "gemini-2.5-flash", + "gemini-2.5-flash-lite-stt": "gemini-2.5-flash-lite", + "azure-speech-v1": "azure/speech/azure-stt", + "azure-tts-v1": "azure/speech/azure-tts", + "pulse-v4": "whisper-1", + "deepgram-flux": "deepgram/nova-3", + "deepgram-nova-3-general-preview-12-2025": "deepgram/nova-3-general", + "minimax-m2p5": "fireworks_ai/minimax-m2p7", + "qwen3p6-plus": "openrouter/qwen/qwen3.6-plus", + "grok-build-0.1": "xai/grok-3", + "grok-4.20-0309-non-reasoning": "xai/grok-4-fast-non-reasoning", + "grok-4.20-multi-agent-0309": "xai/grok-4", + # ElevenLabs: LiteLLM only prices a subset; map siblings to closest priced SKU. + "scribe_v2": "elevenlabs/scribe_v1", + "scribe_v2_realtime": "elevenlabs/scribe_v1", + "eleven_flash_v2_5": "elevenlabs/eleven_multilingual_v2", + "eleven_turbo_v2_5": "elevenlabs/eleven_multilingual_v2", + "eleven_ttv_v3": "elevenlabs/eleven_v3", + "eleven_multilingual_ttv_v2": "elevenlabs/eleven_multilingual_v2", + "eleven_english_sts_v2": "elevenlabs/eleven_multilingual_v2", + "eleven_multilingual_sts_v2": "elevenlabs/eleven_multilingual_v2", + "eleven_text_to_sound_v2": "elevenlabs/eleven_multilingual_v2", + "music_v1": "elevenlabs/eleven_multilingual_v2", +} + +MICRO_USD_PER_USD = 1_000_000 + + +def _azure_deployment_name(catalog_model: str) -> str: + if catalog_model == "azure-openai-gpt4": + return "gpt-4" + if catalog_model.startswith("azure-"): + return catalog_model[len("azure-") :] + return catalog_model + + +def _usage_kind(model_type: Optional[str]) -> str: + if model_type == "stt": + return "stt" + if model_type in {"tts", "sts", "sound_effects", "music"}: + return "tts" + return "llm" + + +def _fireworks_model(name: str) -> str: + if name.startswith("accounts/"): + return name + return f"accounts/fireworks/models/{name}" + + +def _per_million_micro_usd(cost_per_unit: float) -> int: + return int(round(float(cost_per_unit) * MICRO_USD_PER_USD * 1_000_000)) + + +def _per_second_micro_usd(cost_per_second: float) -> int: + return int(round(float(cost_per_second) * MICRO_USD_PER_USD)) + + +def _first_cost(info: Dict[str, Any], *keys: str) -> float: + for key in keys: + value = info.get(key) + if value is not None and float(value) > 0: + return float(value) + return 0.0 + + +def _litellm_candidates( + catalog_name: str, provider: str, model_type: str +) -> List[str]: + if catalog_name in EXPLICIT_LITELLM_KEYS: + return [EXPLICIT_LITELLM_KEYS[catalog_name]] + + prefix = LITELLM_PROVIDER_PREFIX.get(provider, provider) + candidates: List[str] = [] + + if provider == "azure" and model_type == "llm": + deployment = _azure_deployment_name(catalog_name) + candidates.extend( + [ + f"azure/{deployment}", + f"openai/{deployment}", + deployment, + ] + ) + elif provider == "azure" and model_type == "stt": + candidates.extend(["azure/speech/azure-stt", f"azure/{catalog_name}"]) + elif provider == "azure" and model_type == "tts": + candidates.extend(["azure/speech/azure-tts", f"azure/{catalog_name}"]) + elif provider == "deepgram": + stripped = ( + catalog_name[len("deepgram-") :] + if catalog_name.startswith("deepgram-") + else catalog_name + ) + candidates.extend([f"deepgram/{stripped}", stripped]) + elif provider == "fireworks" and model_type == "llm": + fw = _fireworks_model(catalog_name) + candidates.extend( + [ + f"fireworks_ai/{fw}", + f"fireworks_ai/{catalog_name}", + f"fireworks_ai/accounts/fireworks/models/{catalog_name}", + catalog_name, + ] + ) + elif provider == "elevenlabs": + candidates.extend([f"elevenlabs/{catalog_name}", catalog_name]) + elif provider == "google" and catalog_name.endswith("-stt"): + base = catalog_name[: -len("-stt")] + candidates.extend([base, f"gemini/{base}"]) + else: + candidates.extend([catalog_name, f"{prefix}/{catalog_name}"]) + + # De-dupe while preserving order. + seen = set() + ordered: List[str] = [] + for item in candidates: + if item not in seen: + seen.add(item) + ordered.append(item) + return ordered + + +def _resolve_litellm_key( + catalog_name: str, + provider: str, + model_type: str, + model_cost: Dict[str, Dict[str, Any]], +) -> Tuple[Optional[str], Optional[Dict[str, Any]]]: + for key in _litellm_candidates(catalog_name, provider, model_type): + info = model_cost.get(key) + if not info: + continue + if any("cost" in field and info.get(field) for field in info): + return key, info + + for key, info in model_cost.items(): + if key == "sample_spec": + continue + if key.endswith(f"/{catalog_name}") or key == catalog_name: + if any("cost" in field and info.get(field) for field in info): + return key, info + return None, None + + +def _convert_litellm_pricing( + info: Dict[str, Any], *, usage_kind: str +) -> Dict[str, int]: + entry: Dict[str, int] = { + "input_micro_usd_per_million": 0, + "output_micro_usd_per_million": 0, + "cache_read_micro_usd_per_million": 0, + "cache_creation_micro_usd_per_million": 0, + "reasoning_micro_usd_per_million": 0, + "audio_micro_usd_per_second": 0, + "tts_micro_usd_per_million_chars": 0, + } + + if usage_kind == "stt": + audio = _first_cost( + info, + "input_cost_per_second", + "output_cost_per_second", + ) + if not audio: + audio_token = _first_cost(info, "input_cost_per_audio_token") + if audio_token: + # LiteLLM audio-token STT models: ~25 audio tokens/sec (OpenAI convention). + audio = audio_token * 25.0 + entry["audio_micro_usd_per_second"] = _per_second_micro_usd(audio) + return entry + + if usage_kind == "tts": + per_char = _first_cost( + info, + "input_cost_per_character", + "output_cost_per_character", + ) + if per_char: + entry["tts_micro_usd_per_million_chars"] = _per_million_micro_usd(per_char) + return entry + per_token = _first_cost( + info, + "output_cost_per_token", + "input_cost_per_token", + ) + if per_token: + # Approximate chars/token ~= 4 when LiteLLM only exposes token pricing. + entry["tts_micro_usd_per_million_chars"] = _per_million_micro_usd( + per_token / 4.0 + ) + return entry + + entry["input_micro_usd_per_million"] = _per_million_micro_usd( + _first_cost(info, "input_cost_per_token", "input_cost_per_audio_token") + ) + entry["output_micro_usd_per_million"] = _per_million_micro_usd( + _first_cost(info, "output_cost_per_token") + ) + entry["cache_read_micro_usd_per_million"] = _per_million_micro_usd( + _first_cost(info, "cache_read_input_token_cost") + ) + entry["cache_creation_micro_usd_per_million"] = _per_million_micro_usd( + _first_cost(info, "cache_creation_input_token_cost") + ) + entry["reasoning_micro_usd_per_million"] = _per_million_micro_usd( + _first_cost(info, "output_cost_per_reasoning_token") + ) + return entry + + +def _load_model_cost(*, remote: bool) -> Dict[str, Dict[str, Any]]: + if remote: + from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map + + return get_model_cost_map( + url="https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" + ) + from litellm import model_cost + + return dict(model_cost) + + +def _load_manual_entries() -> Dict[str, Dict[str, Any]]: + if not MANUAL_JSON.exists(): + return {} + try: + payload = json.loads(MANUAL_JSON.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + if not isinstance(payload, dict): + return {} + return { + key: value + for key, value in payload.items() + if not key.startswith("_") and isinstance(value, dict) + } + + +def _apply_manual_entries( + catalog: Dict[str, Any], meta: Dict[str, Any], models: Dict[str, Any] +) -> None: + manual = _load_manual_entries() + if not manual: + return + meta.setdefault("manual", {}) + for catalog_name, entry in manual.items(): + if catalog_name in catalog: + continue + if catalog_name not in models or catalog_name.startswith("_"): + continue + model_type = str(models[catalog_name].get("model_type") or "llm") + usage_kind = entry.get("usage_kind") or _usage_kind(model_type) + pricing = { + k: int(v) + for k, v in entry.items() + if k != "usage_kind" and not k.startswith("_") and v is not None + } + if not any(pricing.values()): + continue + catalog[catalog_name] = { + "usage_kind": usage_kind, + **pricing, + "_price_source": entry.get("_price_source", "pricing_manual.json"), + "_litellm_proxy": True, + } + meta["manual"][catalog_name] = entry.get("_price_source", "pricing_manual.json") + meta["resolved"][catalog_name] = "manual" + meta["unresolved"] = [ + item for item in meta["unresolved"] if item.get("model") != catalog_name + ] + + +def build_catalog(*, remote: bool = True) -> Tuple[Dict[str, Any], Dict[str, Any]]: + models = json.loads(MODELS_JSON.read_text(encoding="utf-8")) + model_cost = _load_model_cost(remote=remote) + + catalog: Dict[str, Any] = {} + meta: Dict[str, Any] = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "source": "litellm_remote" if remote else "litellm_local", + "resolved": {}, + "unresolved": [], + } + + for catalog_name, cfg in sorted(models.items()): + if catalog_name.startswith("_") or not isinstance(cfg, dict): + continue + provider = str(cfg.get("provider") or "") + model_type = str(cfg.get("model_type") or "llm") + usage_kind = _usage_kind(model_type) + + litellm_key, info = _resolve_litellm_key( + catalog_name, provider, model_type, model_cost + ) + if not info: + meta["unresolved"].append( + { + "model": catalog_name, + "provider": provider, + "model_type": model_type, + } + ) + continue + + pricing = _convert_litellm_pricing(info, usage_kind=usage_kind) + if not any(pricing.values()): + meta["unresolved"].append( + { + "model": catalog_name, + "provider": provider, + "model_type": model_type, + "litellm_key": litellm_key, + "reason": "zero_cost", + } + ) + continue + + catalog[catalog_name] = { + "usage_kind": usage_kind, + **pricing, + "_litellm_key": litellm_key, + "_litellm_proxy": catalog_name not in {litellm_key, litellm_key.split("/")[-1]}, + } + meta["resolved"][catalog_name] = litellm_key + + _apply_manual_entries(catalog, meta, models) + catalog["_metadata"] = meta + return catalog, meta + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--local", + action="store_true", + help="Use bundled LiteLLM model_cost instead of fetching remote JSON", + ) + parser.add_argument( + "--stdout", + action="store_true", + help="Print JSON to stdout instead of writing pricing_catalog.json", + ) + parser.add_argument( + "--write-models", + action="store_true", + help="Also merge plan-format pricing blocks into app/config/models.json", + ) + args = parser.parse_args() + + catalog, meta = build_catalog(remote=not args.local) + payload = json.dumps(catalog, indent=2, sort_keys=True) + "\n" + if args.stdout: + sys.stdout.write(payload) + else: + OUTPUT_JSON.write_text(payload, encoding="utf-8") + + if args.write_models: + merge_path = REPO_ROOT / "scripts" / "merge_pricing_into_models_json.py" + spec = importlib.util.spec_from_file_location( + "merge_pricing_into_models_json", merge_path + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + updated, skipped = module.merge() + print( + f"models.json pricing: {updated} updated, {skipped} skipped", + file=sys.stderr, + ) + + resolved = len(meta["resolved"]) + unresolved = len(meta["unresolved"]) + print( + f"pricing catalog: {resolved} resolved, {unresolved} unresolved -> {OUTPUT_JSON if not args.stdout else 'stdout'}", + file=sys.stderr, + ) + if unresolved: + for item in meta["unresolved"]: + print(f" unresolved: {item['model']} ({item.get('reason', 'no_litellm_match')})", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_services/test_usage/test_llm_usage.py b/tests/test_services/test_usage/test_llm_usage.py index 4abdf609..28c06104 100644 --- a/tests/test_services/test_usage/test_llm_usage.py +++ b/tests/test_services/test_usage/test_llm_usage.py @@ -494,6 +494,10 @@ def test_concurrent_flush_does_not_double_count(fake_redis, org_ctx, monkeypatch db_b = MagicMock() db_b.execute.return_value = MagicMock(rowcount=1) monkeypatch.setattr(usage_mod, "_flush_pending_buffer", lambda _db, _org: 0) + monkeypatch.setattr( + "app.services.usage.pricing.apply_cost_to_bucket", + lambda *args, **kwargs: False, + ) first = usage_mod.flush_usage_to_catalog(db_a, org_id) second = usage_mod.flush_usage_to_catalog(db_b, org_id) @@ -600,6 +604,10 @@ def test_upsert_bucket_sql_uses_valid_empty_jsonb_literal(): def test_upsert_bucket_matches_legacy_resource_context_key(monkeypatch): """Per-row context must merge into an existing evaluation-level bucket.""" + monkeypatch.setattr( + "app.services.usage.pricing.apply_cost_to_bucket", + lambda *args, **kwargs: False, + ) org_id = uuid4() evaluation_id = uuid4() bucket = { @@ -629,6 +637,30 @@ def test_upsert_bucket_matches_legacy_resource_context_key(monkeypatch): assert "context->>'resource_type'" in legacy_sql +def test_upsert_bucket_applies_cost_after_update(monkeypatch): + org_id = uuid4() + bucket = { + "workspace_id": uuid4(), + "product_section": "chat", + "model": "gpt-test", + "context": {}, + "usage_date": date(2026, 8, 12), + "usage_kind": "llm", + } + deltas = {"prompt_tokens": 10, "completion_tokens": 5} + db = MagicMock() + db.execute.return_value = MagicMock(rowcount=1) + called: list[bool] = [] + monkeypatch.setattr( + "app.services.usage.pricing.apply_cost_to_bucket", + lambda *args, **kwargs: called.append(True) or True, + ) + + usage_mod._upsert_bucket(db, org_id, bucket, deltas) + + assert called + + def test_orphan_recovery_runs_at_most_once_per_interval(fake_redis, monkeypatch): org_id = uuid4() claim_key = f"usage:flushing:{org_id}:{uuid4()}" diff --git a/tests/test_services/test_usage/test_org_usage_costs.py b/tests/test_services/test_usage/test_org_usage_costs.py new file mode 100644 index 00000000..63c791c5 --- /dev/null +++ b/tests/test_services/test_usage/test_org_usage_costs.py @@ -0,0 +1,76 @@ +"""Tests for usage cost API response shaping.""" + +from types import SimpleNamespace + +from app.api.v1.routes.org_usage import ( + _attach_costs, + _breakdown_metrics_from_tuple, + _usage_totals_from_row, +) + + +def test_usage_totals_from_row_includes_nested_costs(): + row = SimpleNamespace( + prompt_tokens=100, + completion_tokens=50, + cache_read_tokens=0, + cache_creation_tokens=0, + reasoning_tokens=0, + audio_seconds=0, + tts_characters=0, + call_count=2, + input_cost_micro_usd=1_000_000, + output_cost_micro_usd=500_000, + cache_read_cost_micro_usd=0, + cache_creation_cost_micro_usd=0, + reasoning_cost_micro_usd=0, + audio_cost_micro_usd=0, + tts_cost_micro_usd=0, + total_cost_micro_usd=1_500_000, + has_unpriced_usage=True, + ) + totals = _usage_totals_from_row(row) + assert totals["total_tokens"] == 150 + assert totals["costs"]["total_cost_usd"] == 1.5 + assert totals["costs"]["has_unpriced_usage"] is True + + +def test_breakdown_metrics_from_tuple_includes_costs(): + metrics = ( + 10, + 5, + 0, + 0, + 0, + 0, + 0, + 1, + 100, + 50, + 0, + 0, + 0, + 0, + 0, + 150, + True, + ) + row = _breakdown_metrics_from_tuple(metrics) + assert row["costs"]["total_cost_usd"] == 0.00015 + assert row["costs"]["has_unpriced_usage"] is True + + +def test_attach_costs_uses_cache_write_field_name(): + payload = _attach_costs( + { + "input_cost_micro_usd": 0, + "output_cost_micro_usd": 0, + "cache_read_cost_micro_usd": 0, + "cache_creation_cost_micro_usd": 2_000_000, + "reasoning_cost_micro_usd": 0, + "audio_cost_micro_usd": 0, + "tts_cost_micro_usd": 0, + "total_cost_micro_usd": 2_000_000, + } + ) + assert payload["costs"]["cache_write_cost_usd"] == 2.0 diff --git a/tests/test_services/test_usage/test_pricing.py b/tests/test_services/test_usage/test_pricing.py new file mode 100644 index 00000000..6a526916 --- /dev/null +++ b/tests/test_services/test_usage/test_pricing.py @@ -0,0 +1,211 @@ +"""Tests for usage pricing computation and rate resolution.""" + +from __future__ import annotations + +from datetime import date +from unittest.mock import MagicMock +from uuid import uuid4 + +from app.services.usage.pricing import ( + CostBreakdown, + RateCard, + UsageMetrics, + _catalog_lookup_models, + _normalize_pricing_block, + _normalize_rate_source, + compute_cost, + RATE_SOURCE_CATALOG, +) + + +def test_compute_cost_llm_token_breakdown(): + rate = RateCard( + source=RATE_SOURCE_CATALOG, + rate_id=uuid4(), + input_micro_usd_per_million=1_000_000, + output_micro_usd_per_million=2_000_000, + cache_read_micro_usd_per_million=500_000, + cache_creation_micro_usd_per_million=1_250_000, + ) + costs = compute_cost( + UsageMetrics( + prompt_tokens=1_000_000, + completion_tokens=500_000, + cache_read_tokens=200_000, + cache_creation_tokens=100_000, + ), + rate, + ) + assert costs.input_cost_micro_usd == 1_000_000 + assert costs.output_cost_micro_usd == 1_000_000 + assert costs.cache_read_cost_micro_usd == 100_000 + assert costs.cache_creation_cost_micro_usd == 125_000 + assert costs.total_cost_micro_usd == 2_225_000 + assert costs.pricing_rate_source == RATE_SOURCE_CATALOG + + +def test_compute_cost_stt_audio_seconds(): + rate = RateCard( + source=RATE_SOURCE_CATALOG, + rate_id=uuid4(), + audio_micro_usd_per_second=100, + ) + costs = compute_cost(UsageMetrics(audio_seconds=90), rate) + assert costs.audio_cost_micro_usd == 9_000 + assert costs.total_cost_micro_usd == 9_000 + + +def test_compute_cost_tts_characters(): + rate = RateCard( + source=RATE_SOURCE_CATALOG, + rate_id=uuid4(), + tts_micro_usd_per_million_chars=15_000_000, + ) + costs = compute_cost(UsageMetrics(tts_characters=2_000_000), rate) + assert costs.tts_cost_micro_usd == 30_000_000 + assert costs.total_cost_micro_usd == 30_000_000 + + +def test_compute_cost_without_rate_is_zero(): + costs = compute_cost(UsageMetrics(prompt_tokens=10_000), None) + assert costs == CostBreakdown() + + +def test_normalize_rate_source_truncates_long_values(): + long_source = "x" * 300 + assert len(_normalize_rate_source(long_source)) == 255 + + +def test_normalize_pricing_block_plan_format(): + normalized = _normalize_pricing_block( + { + "input_per_1m": 2.5, + "output_per_1m": 10.0, + "cache_read_per_1m": 1.25, + "cache_write_per_1m": 0.0, + "source": "litellm_import", + }, + model_type="llm", + ) + assert normalized["usage_kind"] == "llm" + assert normalized["input_micro_usd_per_million"] == 2_500_000 + assert normalized["output_micro_usd_per_million"] == 10_000_000 + assert normalized["cache_read_micro_usd_per_million"] == 1_250_000 + assert normalized["source"] == "litellm_import" + + +def test_normalize_pricing_block_stt_audio_per_minute(): + normalized = _normalize_pricing_block( + {"audio_per_minute": 0.36, "usage_kind": "stt"}, + model_type="stt", + ) + assert normalized["usage_kind"] == "stt" + assert normalized["audio_micro_usd_per_second"] == 6_000 + + +def test_catalog_lookup_models_includes_azure_alias(): + assert _catalog_lookup_models("gpt-4o", "llm") == ("gpt-4o", "azure-gpt-4o") + assert _catalog_lookup_models("azure-gpt-4o", "llm") == ( + "azure-gpt-4o", + "gpt-4o", + ) + + +def test_compute_cost_reasoning_tokens(): + rate = RateCard( + source=RATE_SOURCE_CATALOG, + rate_id=uuid4(), + reasoning_micro_usd_per_million=3_000_000, + ) + costs = compute_cost(UsageMetrics(reasoning_tokens=1_000_000), rate) + assert costs.reasoning_cost_micro_usd == 3_000_000 + assert costs.total_cost_micro_usd == 3_000_000 + + +def test_cost_fields_from_deltas_voice_agent_call_audio(): + from app.services.usage.pricing import ( + RateCard, + _pricing_entries_from_models_json, + cost_fields_from_deltas, + ) + + entries = _pricing_entries_from_models_json() + assert "voice-agent-call" in entries + assert "unknown" in entries + assert entries["voice-agent-call"]["audio_micro_usd_per_second"] > 0 + + rate = RateCard( + source="catalog", + rate_id=uuid4(), + audio_micro_usd_per_second=833, # ~$0.05/min + ) + fields = cost_fields_from_deltas( + {"audio_seconds": 60, "call_count": 1}, + organization_id=uuid4(), + model="voice-agent-call", + usage_kind="llm", + usage_date=date(2026, 8, 11), + db=MagicMock(), + resolver=MagicMock(resolve_rate=MagicMock(return_value=rate)), + ) + assert fields["audio_cost_micro_usd"] == 60 * 833 + assert fields["total_cost_micro_usd"] == fields["audio_cost_micro_usd"] + assert fields["pricing_rate_source"] == "catalog" + + +def test_resolve_rate_ignores_stale_negative_cache(monkeypatch): + from unittest.mock import MagicMock + + import app.services.usage.pricing as pricing_mod + import app.services.usage.pricing_cache as pricing_cache_mod + from app.services.usage.pricing import RATE_SOURCE_CATALOG, PricingResolver + from tests.test_services.test_usage.test_pricing_cache import _FakeRedis + + pricing_mod._RATES_TABLE_CACHE = "model_pricing_rates" + + org_id = uuid4() + rate_id = uuid4() + usage_day = date(2026, 8, 11) + + fake_redis = _FakeRedis() + monkeypatch.setattr(pricing_cache_mod, "_client", lambda: fake_redis) + + redis_key = pricing_cache_mod.pricing_cache_key( + organization_id=org_id, + model="gpt-oss-120b", + usage_kind="llm", + usage_date=usage_day, + ) + pricing_cache_mod.set_cached_rate_payload(redis_key, None) + + row = { + "id": rate_id, + "input_micro_usd_per_million": 1_000_000, + "output_micro_usd_per_million": 2_000_000, + "cache_read_micro_usd_per_million": 0, + "cache_creation_micro_usd_per_million": 0, + "reasoning_micro_usd_per_million": 0, + "audio_micro_usd_per_second": 0, + "tts_micro_usd_per_million_chars": 0, + } + + override_result = MagicMock() + override_result.mappings.return_value.first.return_value = None + catalog_result = MagicMock() + catalog_result.mappings.return_value.first.return_value = row + + db = MagicMock() + db.execute.side_effect = [override_result, catalog_result] + + resolver = PricingResolver(db) + card = resolver.resolve_rate( + organization_id=org_id, + model="gpt-oss-120b", + usage_kind="llm", + usage_date=usage_day, + ) + + assert card is not None + assert card.rate_id == rate_id + assert card.source == RATE_SOURCE_CATALOG + assert fake_redis.store[redis_key] != "__null__" diff --git a/tests/test_services/test_usage/test_pricing_cache.py b/tests/test_services/test_usage/test_pricing_cache.py new file mode 100644 index 00000000..e4276240 --- /dev/null +++ b/tests/test_services/test_usage/test_pricing_cache.py @@ -0,0 +1,91 @@ +"""Tests for Redis-backed usage pricing cache.""" + +from __future__ import annotations + +from datetime import date +from uuid import uuid4 + +import app.services.usage.pricing_cache as pricing_cache_mod + + +class _FakeRedis: + def __init__(self): + self.store: dict[str, str] = {} + + def get(self, key: str): + return self.store.get(key) + + def setex(self, key: str, _ttl: int, value: str): + self.store[key] = value + + def scan(self, cursor: int, match: str, count: int): + keys = [key for key in self.store if key.startswith(match.rstrip("*"))] + return 0, keys + + def delete(self, *keys: str): + for key in keys: + self.store.pop(key, None) + + +def test_pricing_cache_roundtrip(monkeypatch): + fake = _FakeRedis() + monkeypatch.setattr(pricing_cache_mod, "_client", lambda: fake) + + org_id = uuid4() + key = pricing_cache_mod.pricing_cache_key( + organization_id=org_id, + model="gpt-4o", + usage_kind="llm", + usage_date=date(2026, 8, 13), + ) + payload = { + "source": "catalog", + "rate_id": str(uuid4()), + "input_micro_usd_per_million": 2_500_000, + "output_micro_usd_per_million": 10_000_000, + "cache_read_micro_usd_per_million": 0, + "cache_creation_micro_usd_per_million": 0, + "reasoning_micro_usd_per_million": 0, + "audio_micro_usd_per_second": 0, + "tts_micro_usd_per_million_chars": 0, + } + pricing_cache_mod.set_cached_rate_payload(key, payload) + cached = pricing_cache_mod.get_cached_rate_payload(key) + assert cached == payload + + +def test_pricing_cache_null_marker(monkeypatch): + fake = _FakeRedis() + monkeypatch.setattr(pricing_cache_mod, "_client", lambda: fake) + + org_id = uuid4() + key = pricing_cache_mod.pricing_cache_key( + organization_id=org_id, + model="unknown-model", + usage_kind="llm", + usage_date=date(2026, 8, 13), + ) + pricing_cache_mod.set_cached_rate_payload(key, None) + assert pricing_cache_mod.get_cached_rate_payload(key) == {} + + +def test_null_cache_uses_shorter_ttl(monkeypatch): + fake = _FakeRedis() + recorded: list[tuple[int, str]] = [] + + def _setex(key: str, ttl: int, value: str): + fake.store[key] = value + recorded.append((ttl, value)) + + fake.setex = _setex # type: ignore[method-assign] + monkeypatch.setattr(pricing_cache_mod, "_client", lambda: fake) + + key = "usage:pricing:test" + pricing_cache_mod.set_cached_rate_payload(key, None) + pricing_cache_mod.set_cached_rate_payload( + key + ":hit", + {"source": "catalog", "rate_id": str(uuid4())}, + ) + assert recorded[0][0] == pricing_cache_mod.PRICING_NULL_CACHE_TTL_SEC + assert recorded[0][1] == "__null__" + assert recorded[1][0] == pricing_cache_mod.PRICING_CACHE_TTL_SEC diff --git a/tests/test_services/test_usage/test_pricing_ops.py b/tests/test_services/test_usage/test_pricing_ops.py new file mode 100644 index 00000000..e5a3c51b --- /dev/null +++ b/tests/test_services/test_usage/test_pricing_ops.py @@ -0,0 +1,10 @@ +"""Tests for usage pricing ops helpers.""" + +from __future__ import annotations + +from app.services.usage.pricing_ops import models_missing_pricing_blocks + + +def test_models_missing_pricing_blocks_is_sorted_list(): + missing = models_missing_pricing_blocks() + assert missing == sorted(missing) diff --git a/tests/test_services/test_usage/test_pricing_overrides.py b/tests/test_services/test_usage/test_pricing_overrides.py new file mode 100644 index 00000000..3e18d191 --- /dev/null +++ b/tests/test_services/test_usage/test_pricing_overrides.py @@ -0,0 +1,72 @@ +"""Tests for org pricing override merge and service helpers.""" + +from __future__ import annotations + +from uuid import uuid4 + +from app.services.usage.pricing import ( + RATE_SOURCE_CATALOG, + RATE_SOURCE_OVERRIDE, + RateCard, + _merge_override_with_catalog, +) +from app.services.usage.pricing_overrides import _usd_payload_to_micro_columns + + +def test_merge_override_inherits_null_fields_from_catalog(): + catalog = RateCard( + source=RATE_SOURCE_CATALOG, + rate_id=uuid4(), + input_micro_usd_per_million=1_000_000, + output_micro_usd_per_million=2_000_000, + cache_read_micro_usd_per_million=500_000, + ) + override_id = uuid4() + merged = _merge_override_with_catalog( + { + "id": override_id, + "input_micro_usd_per_million": 3_000_000, + "output_micro_usd_per_million": None, + "cache_read_micro_usd_per_million": None, + "cache_creation_micro_usd_per_million": None, + "reasoning_micro_usd_per_million": None, + "audio_micro_usd_per_second": None, + "tts_micro_usd_per_million_chars": None, + }, + catalog, + ) + assert merged.source == RATE_SOURCE_OVERRIDE + assert merged.rate_id == override_id + assert merged.input_micro_usd_per_million == 3_000_000 + assert merged.output_micro_usd_per_million == 2_000_000 + assert merged.cache_read_micro_usd_per_million == 500_000 + + +def test_merge_override_without_catalog_uses_zero_defaults(): + override_id = uuid4() + merged = _merge_override_with_catalog( + { + "id": override_id, + "input_micro_usd_per_million": 100, + "output_micro_usd_per_million": None, + "cache_read_micro_usd_per_million": None, + "cache_creation_micro_usd_per_million": None, + "reasoning_micro_usd_per_million": None, + "audio_micro_usd_per_second": None, + "tts_micro_usd_per_million_chars": None, + }, + None, + ) + assert merged.input_micro_usd_per_million == 100 + assert merged.output_micro_usd_per_million == 0 + + +def test_usd_payload_to_micro_columns(): + columns = _usd_payload_to_micro_columns( + { + "input_per_1m": 1.5, + "audio_per_minute": 0.006, + } + ) + assert columns["input_micro_usd_per_million"] == 1_500_000 + assert columns["audio_micro_usd_per_second"] == 100 diff --git a/tests/test_services/test_usage/test_usage_costs.py b/tests/test_services/test_usage/test_usage_costs.py new file mode 100644 index 00000000..2e6d028c --- /dev/null +++ b/tests/test_services/test_usage/test_usage_costs.py @@ -0,0 +1,33 @@ +"""Tests for usage cost presentation helpers.""" + +from app.services.usage.usage_costs import costs_from_micro, micro_to_usd + + +def test_micro_to_usd(): + assert micro_to_usd(1_500_000) == 1.5 + assert micro_to_usd(0) == 0 + + +def test_costs_from_micro_maps_cache_write_and_unpriced_flag(): + costs = costs_from_micro( + input_cost_micro_usd=1_000_000, + output_cost_micro_usd=500_000, + cache_creation_cost_micro_usd=250_000, + total_cost_micro_usd=1_750_000, + has_unpriced_usage=True, + ) + assert costs["input_cost_usd"] == 1.0 + assert costs["output_cost_usd"] == 0.5 + assert costs["cache_write_cost_usd"] == 0.25 + assert costs["total_cost_usd"] == 1.75 + assert costs["currency"] == "USD" + assert costs["has_unpriced_usage"] is True + + +def test_costs_from_micro_sums_total_when_not_provided(): + costs = costs_from_micro( + input_cost_micro_usd=100, + output_cost_micro_usd=200, + audio_cost_micro_usd=300, + ) + assert costs["total_cost_usd"] == micro_to_usd(600) diff --git a/tests/test_workers/test_usage_queue_routing.py b/tests/test_workers/test_usage_queue_routing.py new file mode 100644 index 00000000..8ccdb6f3 --- /dev/null +++ b/tests/test_workers/test_usage_queue_routing.py @@ -0,0 +1,18 @@ +"""Tests for usage pricing Celery queue routing.""" + +from app.workers.config import USAGE_WORKER_QUEUE, celery_app + + +def test_flush_usage_counters_routes_to_usage_queue(): + routes = celery_app.conf.task_routes + assert routes["flush_usage_counters"]["queue"] == USAGE_WORKER_QUEUE + + +def test_recompute_usage_costs_routes_to_usage_queue(): + routes = celery_app.conf.task_routes + assert routes["recompute_usage_costs"]["queue"] == USAGE_WORKER_QUEUE + + +def test_usage_beat_flush_targets_usage_queue(): + schedule = celery_app.conf.beat_schedule["flush-llm-usage-counters"] + assert schedule["options"]["queue"] == USAGE_WORKER_QUEUE From 7d59a5babe804d695f21d861301c95fffb45b9f0 Mon Sep 17 00:00:00 2001 From: M Sami Date: Fri, 14 Aug 2026 18:45:37 +0530 Subject: [PATCH 21/32] feat: add usage flush configuration and enterprise entitlement checks for usage features --- README.md | 24 +- app/api/v1/routes/org_usage.py | 92 +++++-- app/api/v1/routes/settings.py | 3 + app/api/v1/routes/usage_pricing.py | 22 +- app/core/usage_entitlement.py | 55 ++++ app/dependencies.py | 24 ++ app/services/usage/access.py | 83 ++++++ app/services/usage/llm_usage.py | 256 ++++++++++++------ app/services/usage/retention.py | 41 +++ app/workers/config.py | 7 + app/workers/tasks/__init__.py | 2 + app/workers/tasks/prune_oss_usage_history.py | 22 ++ env.example | 10 + frontend/src/App.tsx | 27 +- frontend/src/lib/api.ts | 11 + .../pages/enterprise/EnterpriseUpgrade.tsx | 8 +- frontend/src/pages/usage/Usage.tsx | 82 +++++- .../src/pages/usage/UsageDateRangePicker.tsx | 34 ++- frontend/src/pages/usage/UsageFiltersBar.tsx | 9 +- frontend/src/store/licenseStore.ts | 16 +- tests/test_core/test_usage_entitlement.py | 49 ++++ .../test_usage/test_llm_usage.py | 57 +++- .../test_usage/test_usage_access.py | 67 +++++ .../test_usage/test_usage_retention.py | 68 +++++ 24 files changed, 930 insertions(+), 139 deletions(-) create mode 100644 app/core/usage_entitlement.py create mode 100644 app/services/usage/access.py create mode 100644 app/services/usage/retention.py create mode 100644 app/workers/tasks/prune_oss_usage_history.py create mode 100644 tests/test_core/test_usage_entitlement.py create mode 100644 tests/test_services/test_usage/test_usage_access.py create mode 100644 tests/test_services/test_usage/test_usage_retention.py diff --git a/README.md b/README.md index 009ca348..c5ad0a0b 100644 --- a/README.md +++ b/README.md @@ -373,7 +373,7 @@ celery -A app.workers.celery_app worker --loglevel=info **Note:** Workers are required for background tasks (transcription, evaluation, usage cost flush, etc.). If you use `eai start-all`, they start automatically. Only use `eai worker` if you need to run a worker separately (e.g. `eai worker --queues usage` for the usage queue only). ### Usage Pricing Ops -Manage model pricing rates and backfill stored usage costs on `llm_usage_daily` rollups. +Manage model pricing rates and backfill stored usage costs on `llm_usage_daily` rollups. Requires `worker-usage` + `beat` (or `eai start-all`). ```bash # Upsert model_pricing_rates from app/config/models.json @@ -382,12 +382,14 @@ eai usage seed-rates --config config.yml # Compare models.json pricing vs Postgres eai usage diff-rates --config config.yml -# Backfill / recompute costs (sync, runs in this process) +# Backfill costs in-process (all orgs; use after migrate or catalog change) eai usage recompute --config config.yml --sync -# Enqueue Celery recompute on the usage queue (requires --organization-id) +# Async recompute via usage queue (requires --organization-id) eai usage recompute --config config.yml --organization-id +# Optional scopes: --model, --usage-kind, --start-date, --end-date + # Optional: fetch LiteLLM prices into pricing_catalog.json eai usage sync-litellm --local eai usage sync-litellm --local --write-models @@ -400,7 +402,17 @@ eai usage seed-rates --config config.yml eai usage recompute --config config.yml --sync ``` -Requires `worker-usage` (or `eai start-all`) for async recompute; Beat + `worker-usage` keep Redis usage counters flushed into Postgres on a schedule (`USAGE_FLUSH_BEAT_SECONDS`, default 120). +**Flush / Usage UI tuning** — set in `.env` (see `env.example`): + +| Variable | Default | Purpose | +|----------|---------|---------| +| `USAGE_FLUSH_BUCKET_BATCH_SIZE` | `500` | Buckets per DB transaction | +| `USAGE_FLUSH_MAX_BATCHES_PER_RUN` | `30` | Batches per flush tick (≤ **15,000** buckets/run) | +| `USAGE_FLUSH_BEAT_SECONDS` | `120` | Beat interval (~2 min lag vs Redis) | +| `USAGE_FLUSH_LOCK_TTL_SECONDS` | `300` | Per-org flush lock TTL | +| `USAGE_API_FLUSH_COOLDOWN_SECONDS` | `60` | Min gap between Usage page catalog syncs | + +Usage UI calls `POST /api/v1/organizations/usage/catalog/sync` once per visit; summary/breakdown/filters are read-only. If Redis backlog grows, lower `USAGE_FLUSH_BEAT_SECONDS` or raise `USAGE_FLUSH_MAX_BATCHES_PER_RUN`. ### Generate Config File ```bash @@ -515,6 +527,10 @@ POSTGRES_PASSWORD=password POSTGRES_DB=efficientai SECRET_KEY=your-secret-key-here +# Usage cost flush (see README "Usage Pricing Ops"; full list in env.example) +# USAGE_FLUSH_BEAT_SECONDS=120 +# USAGE_FLUSH_MAX_BATCHES_PER_RUN=30 + # Optional: GCS blob storage (also set storage.blob_provider: gcs in config.yml) BLOB_STORAGE_PROVIDER=gcs GCS_BUCKET_NAME=your-gcs-bucket diff --git a/app/api/v1/routes/org_usage.py b/app/api/v1/routes/org_usage.py index 2d6854f6..95688b3c 100644 --- a/app/api/v1/routes/org_usage.py +++ b/app/api/v1/routes/org_usage.py @@ -3,6 +3,7 @@ from __future__ import annotations from datetime import date, datetime, timezone +from app.services.usage.access import UsageAccessPolicy from app.services.usage.dates import usage_date_filter_bounds, usage_local_today from typing import List, Literal, Optional from uuid import UUID @@ -122,10 +123,21 @@ class UsageTotals(BaseModel): costs: UsageCosts = Field(default_factory=UsageCosts) +class UsageCatalogSyncResponse(BaseModel): + flushed_buckets: int = 0 + + +class UsagePolicyMeta(BaseModel): + extended_history: bool + max_history_days: Optional[int] = None + range_clamped: bool = False + + class UsageSummaryResponse(BaseModel): start: date end: date totals: UsageTotals + usage_policy: UsagePolicyMeta last_updated_at: Optional[datetime] = None @@ -168,6 +180,7 @@ class UsageBreakdownResponse(BaseModel): rows: List[UsageBreakdownRow] total_count: int truncated_at_limit: bool = False + usage_policy: UsagePolicyMeta last_updated_at: Optional[datetime] = None @@ -198,6 +211,14 @@ def _parse_usage_range( return display_start, display_end, filter_start, filter_end +def _usage_policy_meta(access) -> UsagePolicyMeta: + return UsagePolicyMeta( + extended_history=access.policy.extended_history, + max_history_days=access.policy.max_history_days, + range_clamped=access.range_clamped, + ) + + def _evaluation_id_expr(): return func.coalesce( LLMUsageDaily.context["evaluation_id"].astext, @@ -418,6 +439,7 @@ def _apply_filters( organization_id: UUID, start: date, end: date, + enforced_floor: Optional[date] = None, workspace_id: Optional[UUID], product_section: Optional[str], model: Optional[str], @@ -430,9 +452,12 @@ def _apply_filters( tag_id: Optional[UUID] = None, db: Optional[Session] = None, ): + effective_start = start + if enforced_floor is not None and effective_start < enforced_floor: + effective_start = enforced_floor query = query.filter( LLMUsageDaily.organization_id == organization_id, - LLMUsageDaily.usage_date >= start, + LLMUsageDaily.usage_date >= effective_start, LLMUsageDaily.usage_date <= end, ) if workspace_id is not None: @@ -506,6 +531,7 @@ def _filtered_query( organization_id: UUID, start: date, end: date, + enforced_floor: Optional[date] = None, workspace_id: Optional[UUID] = None, product_section: Optional[str] = None, model: Optional[str] = None, @@ -522,6 +548,7 @@ def _filtered_query( organization_id=organization_id, start=start, end=end, + enforced_floor=enforced_floor, workspace_id=workspace_id, product_section=product_section, model=model, @@ -1010,6 +1037,7 @@ def _summary_aggregate_query( organization_id: UUID, start: date, end: date, + enforced_floor: Optional[date] = None, workspace_id: Optional[UUID], product_section: Optional[str], model: Optional[str], @@ -1045,6 +1073,7 @@ def _summary_aggregate_query( organization_id=organization_id, start=start, end=end, + enforced_floor=enforced_floor, workspace_id=workspace_id, product_section=product_section, model=model, @@ -1059,6 +1088,16 @@ def _summary_aggregate_query( ) +@router.post("/catalog/sync", response_model=UsageCatalogSyncResponse) +def sync_usage_catalog( + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +): + """Drain Redis usage counters into Postgres (rate-limited per org via Redis).""" + flushed = flush_usage_to_catalog(db, organization_id) + return UsageCatalogSyncResponse(flushed_buckets=flushed) + + @router.get("/summary", response_model=UsageSummaryResponse) def get_usage_summary( start: Optional[date] = Query(None), @@ -1080,19 +1119,18 @@ def get_usage_summary( organization_id: UUID = Depends(get_organization_id), db: Session = Depends(get_db), ): - display_start, display_end, filter_start, filter_end = _parse_usage_range( - start, end, tz - ) + access = UsageAccessPolicy.resolve(organization_id, start, end, tz) + display_start = access.display_start + display_end = access.display_end if display_end < display_start: raise HTTPException(status_code=400, detail="end must be >= start") - flush_usage_to_catalog(db, organization_id) - row = _summary_aggregate_query( db, organization_id=organization_id, - start=filter_start, - end=filter_end, + start=access.filter_start, + end=access.filter_end, + enforced_floor=access.enforced_filter_floor, workspace_id=workspace_id, product_section=product_section, model=model, @@ -1109,6 +1147,7 @@ def get_usage_summary( start=display_start, end=display_end, totals=UsageTotals(**totals), + usage_policy=_usage_policy_meta(access), last_updated_at=_last_updated(db, organization_id), ) @@ -1137,14 +1176,12 @@ def get_usage_breakdown( organization_id: UUID = Depends(get_organization_id), db: Session = Depends(get_db), ): - display_start, display_end, filter_start, filter_end = _parse_usage_range( - start, end, tz - ) + access = UsageAccessPolicy.resolve(organization_id, start, end, tz) + display_start = access.display_start + display_end = access.display_end if display_end < display_start: raise HTTPException(status_code=400, detail="end must be >= start") - flush_usage_to_catalog(db, organization_id) - dim = { "workspace": LLMUsageDaily.workspace_id, "product_section": LLMUsageDaily.product_section, @@ -1186,8 +1223,9 @@ def get_usage_breakdown( query = _apply_filters( db.query(*select_cols, *aggregates), organization_id=organization_id, - start=filter_start, - end=filter_end, + start=access.filter_start, + end=access.filter_end, + enforced_floor=access.enforced_filter_floor, workspace_id=workspace_id, product_section=product_section, model=model, @@ -1227,8 +1265,9 @@ def get_usage_breakdown( label_query = _filtered_query( db, organization_id=organization_id, - start=filter_start, - end=filter_end, + start=access.filter_start, + end=access.filter_end, + enforced_floor=access.enforced_filter_floor, workspace_id=workspace_id, product_section=product_section, model=model, @@ -1243,8 +1282,9 @@ def get_usage_breakdown( label_query = _filtered_query( db, organization_id=organization_id, - start=filter_start, - end=filter_end, + start=access.filter_start, + end=access.filter_end, + enforced_floor=access.enforced_filter_floor, workspace_id=workspace_id, product_section=product_section, model=model, @@ -1353,6 +1393,7 @@ def get_usage_breakdown( rows=rows, total_count=len(rows), truncated_at_limit=len(rows) >= limit, + usage_policy=_usage_policy_meta(access), last_updated_at=_last_updated(db, organization_id), ) @@ -1378,9 +1419,10 @@ def get_usage_filters( organization_id: UUID = Depends(get_organization_id), db: Session = Depends(get_db), ): - _, _, filter_start, filter_end = _parse_usage_range(start, end, tz) - - flush_usage_to_catalog(db, organization_id) + access = UsageAccessPolicy.resolve(organization_id, start, end, tz) + filter_start = access.filter_start + filter_end = access.filter_end + enforced_floor = access.enforced_filter_floor scoped_resource_id = resource_id or evaluation_id @@ -1389,6 +1431,7 @@ def get_usage_filters( organization_id=organization_id, start=filter_start, end=filter_end, + enforced_floor=enforced_floor, dataset=dataset, tag_id=tag_id, ) @@ -1397,6 +1440,7 @@ def get_usage_filters( organization_id=organization_id, start=filter_start, end=filter_end, + enforced_floor=enforced_floor, workspace_id=workspace_id, dataset=dataset, tag_id=tag_id, @@ -1406,6 +1450,7 @@ def get_usage_filters( organization_id=organization_id, start=filter_start, end=filter_end, + enforced_floor=enforced_floor, workspace_id=workspace_id, product_section=product_section, resource_id=scoped_resource_id, @@ -1419,6 +1464,7 @@ def get_usage_filters( organization_id=organization_id, start=filter_start, end=filter_end, + enforced_floor=enforced_floor, workspace_id=workspace_id, product_section=product_section, resource_id=scoped_resource_id, @@ -1433,6 +1479,7 @@ def get_usage_filters( organization_id=organization_id, start=filter_start, end=filter_end, + enforced_floor=enforced_floor, workspace_id=workspace_id, product_section=product_section, dataset=dataset, @@ -1443,6 +1490,7 @@ def get_usage_filters( organization_id=organization_id, start=filter_start, end=filter_end, + enforced_floor=enforced_floor, workspace_id=workspace_id, product_section=product_section, model=model, diff --git a/app/api/v1/routes/settings.py b/app/api/v1/routes/settings.py index 3c49c0cc..83225380 100644 --- a/app/api/v1/routes/settings.py +++ b/app/api/v1/routes/settings.py @@ -32,6 +32,7 @@ is_feature_enabled, ENTERPRISE_FEATURES, ) +from app.core.usage_entitlement import get_usage_policy REPORT_LOGO_CONTENT_TYPES = { @@ -85,12 +86,14 @@ def license_info(organization_id: UUID = Depends(get_organization_id)): data = get_license_info() all_licensed = data.get("features", []) if isinstance(data.get("features"), list) else [] enabled_for_org = [f for f in all_licensed if is_feature_enabled(f, organization_id)] + usage_policy = get_usage_policy(organization_id) return { "is_enterprise": bool(enabled_for_org), "enabled_features": enabled_for_org, "all_enterprise_features": ENTERPRISE_FEATURES, "feature_catalog": get_feature_catalog(), "organization": data.get("org_id"), + "usage_policy": usage_policy.as_dict(), } diff --git a/app/api/v1/routes/usage_pricing.py b/app/api/v1/routes/usage_pricing.py index 0840a29a..0716eef6 100644 --- a/app/api/v1/routes/usage_pricing.py +++ b/app/api/v1/routes/usage_pricing.py @@ -12,7 +12,7 @@ from app.core.auth.rbac import require_admin from app.database import get_db -from app.dependencies import get_organization_id +from app.dependencies import get_organization_id, require_enterprise_entitlement from app.services.usage.pricing_jobs import ( create_recompute_job, enqueue_recompute_job, @@ -27,10 +27,12 @@ upsert_override, ) +from app.services.usage.access import UsageAccessPolicy + router = APIRouter( prefix="/organizations/usage/pricing", tags=["Usage"], - dependencies=[Depends(require_admin)], + dependencies=[Depends(require_admin), Depends(require_enterprise_entitlement())], ) @@ -218,13 +220,25 @@ def trigger_usage_cost_recompute( ): raise HTTPException(status_code=400, detail="end_date must be >= start_date") + clamped_start = body.start_date + clamped_end = body.end_date + if body.start_date is not None or body.end_date is not None: + access = UsageAccessPolicy.resolve( + organization_id, + body.start_date, + body.end_date, + None, + ) + clamped_start = access.display_start if body.start_date is not None else None + clamped_end = access.display_end if body.end_date is not None else None + job = create_recompute_job( db, organization_id=organization_id, model=body.model, usage_kind=body.usage_kind, - start_date=body.start_date, - end_date=body.end_date, + start_date=clamped_start, + end_date=clamped_end, ) enqueue_recompute_job(db, job) db.refresh(job) diff --git a/app/core/usage_entitlement.py b/app/core/usage_entitlement.py new file mode 100644 index 00000000..cbedbf7d --- /dev/null +++ b/app/core/usage_entitlement.py @@ -0,0 +1,55 @@ +"""Enterprise entitlement for usage history (any catalog feature, not per-feature gates).""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional +from uuid import UUID + +from app.core.license import get_enabled_features, get_license_info + +OSS_USAGE_HISTORY_DAYS = 7 + + +@dataclass(frozen=True) +class UsagePolicySnapshot: + extended_history: bool + max_history_days: Optional[int] = None + + def as_dict(self) -> dict: + return { + "extended_history": self.extended_history, + "max_history_days": self.max_history_days, + } + + +def has_enterprise_entitlement(organization_id: Optional[UUID] = None) -> bool: + """Valid license with at least one catalog feature; org-scoped licenses must match.""" + if not get_enabled_features(): + return False + + info = get_license_info() + licensed_org = info.get("org_id") + if licensed_org is None: + return True + + if organization_id is None: + return False + + return str(organization_id) == str(licensed_org) + + +def deployment_has_entitlement() -> bool: + """Deployment-wide entitlement (license without org_id scoping).""" + if not get_enabled_features(): + return False + return get_license_info().get("org_id") is None + + +def get_usage_policy(organization_id: UUID) -> UsagePolicySnapshot: + if has_enterprise_entitlement(organization_id): + return UsagePolicySnapshot(extended_history=True, max_history_days=None) + return UsagePolicySnapshot( + extended_history=False, + max_history_days=OSS_USAGE_HISTORY_DAYS, + ) diff --git a/app/dependencies.py b/app/dependencies.py index c3597d75..5ac8f1ca 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -21,6 +21,7 @@ from app.core.auth import Principal, get_principal # noqa: F401 - re-exported from app.core.auth.rbac import get_org_role from app.core.license import is_feature_enabled +from app.core.usage_entitlement import has_enterprise_entitlement from app.database import get_db from app.models.database import RoleEnum, Workspace, WorkspaceMember from app.core.auth.capabilities import capability_denied_message @@ -291,3 +292,26 @@ def _check( ) return _check + + +def require_enterprise_entitlement(): + """ + FastAPI dependency: valid enterprise license with any catalog feature. + Distinct from require_enterprise_feature (per-feature product gates). + """ + + def _check(organization_id: UUID = Depends(get_organization_id)): + if not has_enterprise_entitlement(organization_id): + raise HTTPException( + status_code=403, + detail={ + "error": "enterprise_license_required", + "message": ( + "This capability requires a valid EfficientAI Enterprise license. " + "Set EFFICIENTAI_LICENSE in your environment with any enterprise " + "feature enabled. Contact sales@efficientai.com for a license key." + ), + }, + ) + + return _check diff --git a/app/services/usage/access.py b/app/services/usage/access.py new file mode 100644 index 00000000..9591bf2d --- /dev/null +++ b/app/services/usage/access.py @@ -0,0 +1,83 @@ +"""Usage read access policy — date window clamping and SQL floor for OSS tier.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import date, timedelta +from typing import Optional +from uuid import UUID + +from loguru import logger + +from app.core.usage_entitlement import ( + OSS_USAGE_HISTORY_DAYS, + UsagePolicySnapshot, + get_usage_policy, + has_enterprise_entitlement, +) +from app.services.usage.dates import usage_date_filter_bounds, usage_local_today + + +@dataclass(frozen=True) +class UsageAccessResult: + display_start: date + display_end: date + filter_start: date + filter_end: date + enforced_filter_floor: Optional[date] + policy: UsagePolicySnapshot + range_clamped: bool + + +def oss_usage_min_local_date(tz: Optional[str]) -> date: + """Earliest inclusive local calendar day allowed for OSS tier.""" + today = usage_local_today(tz) + return today - timedelta(days=OSS_USAGE_HISTORY_DAYS - 1) + + +class UsageAccessPolicy: + @staticmethod + def resolve( + organization_id: UUID, + start: Optional[date], + end: Optional[date], + tz: Optional[str], + ) -> UsageAccessResult: + today = usage_local_today(tz) + display_start = start or today + display_end = end or today + policy = get_usage_policy(organization_id) + range_clamped = False + enforced_floor: Optional[date] = None + + if not has_enterprise_entitlement(organization_id): + oss_min = oss_usage_min_local_date(tz) + if display_start < oss_min: + display_start = oss_min + range_clamped = True + enforced_floor = usage_date_filter_bounds(oss_min, oss_min, tz)[0] + + filter_start, filter_end = usage_date_filter_bounds( + display_start, display_end, tz + ) + + if enforced_floor is not None and filter_start < enforced_floor: + filter_start = enforced_floor + + if range_clamped: + logger.info( + "usage_history_clamped org_id={} effective_start={} effective_end={}", + organization_id, + display_start, + display_end, + ) + + return UsageAccessResult( + display_start=display_start, + display_end=display_end, + filter_start=filter_start, + filter_end=filter_end, + enforced_filter_floor=enforced_floor, + policy=policy, + range_clamped=range_clamped, + ) diff --git a/app/services/usage/llm_usage.py b/app/services/usage/llm_usage.py index 0fdb48f1..51ffc6bc 100644 --- a/app/services/usage/llm_usage.py +++ b/app/services/usage/llm_usage.py @@ -2,10 +2,11 @@ from __future__ import annotations +import json import math +import os import time import uuid -import json from datetime import date, datetime, timezone from typing import Any, Dict, Iterable, List, Optional, Tuple from uuid import UUID @@ -34,7 +35,10 @@ _NONE = "__none__" _PENDING_TTL_SECONDS = 14 * 24 * 60 * 60 -_FLUSH_LOCK_TTL_SECONDS = 45 +_DEFAULT_FLUSH_BUCKET_BATCH_SIZE = 500 +_DEFAULT_FLUSH_MAX_BATCHES_PER_RUN = 30 +_DEFAULT_FLUSH_LOCK_TTL_SECONDS = 300 +_DEFAULT_API_FLUSH_COOLDOWN_SECONDS = 60 _FLUSH_LOCK_WAIT_SECONDS = 3.0 USAGE_KIND_LLM = "llm" USAGE_KIND_STT = "stt" @@ -59,6 +63,63 @@ """ +def _env_int(name: str, default: int, *, minimum: int = 1) -> int: + raw = os.environ.get(name) + if raw is None: + return default + try: + return max(minimum, int(raw)) + except (TypeError, ValueError): + logger.warning("invalid {}={!r}; using default {}", name, raw, default) + return default + + +def flush_bucket_batch_size() -> int: + """Max rollup buckets persisted per DB transaction during Redis flush.""" + return _env_int( + "USAGE_FLUSH_BUCKET_BATCH_SIZE", + _DEFAULT_FLUSH_BUCKET_BATCH_SIZE, + ) + + +def flush_max_batches_per_run() -> int: + """Max Redis flush batches per org per flush_usage_to_catalog invocation.""" + return _env_int( + "USAGE_FLUSH_MAX_BATCHES_PER_RUN", + _DEFAULT_FLUSH_MAX_BATCHES_PER_RUN, + ) + + +def _flush_lock_ttl_seconds() -> int: + return _env_int( + "USAGE_FLUSH_LOCK_TTL_SECONDS", + _DEFAULT_FLUSH_LOCK_TTL_SECONDS, + ) + + +def api_flush_cooldown_seconds() -> int: + """Min seconds between API-triggered catalog syncs per org (0 = no cooldown).""" + return _env_int( + "USAGE_API_FLUSH_COOLDOWN_SECONDS", + _DEFAULT_API_FLUSH_COOLDOWN_SECONDS, + minimum=0, + ) + + +def _split_buckets_for_flush( + buckets: Dict[str, Dict[str, int]], + batch_size: int, +) -> Tuple[Dict[str, Dict[str, int]], Dict[str, Dict[str, int]]]: + if batch_size <= 0 or len(buckets) <= batch_size: + return buckets, {} + prefixes = sorted(buckets.keys())[:batch_size] + batch = {prefix: buckets[prefix] for prefix in prefixes} + remainder = { + prefix: metrics for prefix, metrics in buckets.items() if prefix not in batch + } + return batch, remainder + + def _client() -> redis.Redis: global _redis if _redis is None: @@ -683,12 +744,13 @@ def _acquire_flush_lock(organization_id: UUID) -> bool: try: client = _client() lock_key = _flush_lock_key(organization_id) - if client.set(lock_key, "1", nx=True, ex=_FLUSH_LOCK_TTL_SECONDS): + lock_ttl = _flush_lock_ttl_seconds() + if client.set(lock_key, "1", nx=True, ex=lock_ttl): return True deadline = time.monotonic() + _FLUSH_LOCK_WAIT_SECONDS while time.monotonic() < deadline: time.sleep(0.05) - if client.set(lock_key, "1", nx=True, ex=_FLUSH_LOCK_TTL_SECONDS): + if client.set(lock_key, "1", nx=True, ex=lock_ttl): return True if client.get(lock_key) is None: continue @@ -872,6 +934,104 @@ def _upsert_bucket( logger.warning("usage cost apply failed: {}", exc) +def _upsert_claimed_buckets( + db: Session, + organization_id: UUID, + buckets: Dict[str, Dict[str, int]], + *, + pricing_resolver: Any, +) -> Tuple[int, Dict[str, Dict[str, int]]]: + """Persist claimed Redis buckets; return flushed count and unparseable buckets.""" + flushed = 0 + skipped: Dict[str, Dict[str, int]] = {} + for prefix, deltas in buckets.items(): + parsed = _parse_bucket_prefix(prefix) + if not parsed: + skipped[prefix] = deltas + logger.warning( + "llm usage skipped unparseable bucket prefix for org {}", + organization_id, + ) + continue + _upsert_bucket( + db, + organization_id, + parsed, + deltas, + pricing_resolver=pricing_resolver, + ) + flushed += 1 + return flushed, skipped + + +def _flush_redis_pending_to_catalog( + db: Session, + organization_id: UUID, + *, + pricing_resolver: Any, +) -> int: + """Drain Redis pending hash into llm_usage_daily in bounded batches.""" + batch_size = flush_bucket_batch_size() + max_batches = flush_max_batches_per_run() + flushed = 0 + + for _ in range(max_batches): + if not _has_pending_usage(organization_id): + break + + claim_key, claimed_buckets = _claim_pending(organization_id) + if not claim_key or not claimed_buckets: + break + + batch, remainder = _split_buckets_for_flush(claimed_buckets, batch_size) + skipped: Dict[str, Dict[str, int]] = {} + try: + batch_flushed, skipped = _upsert_claimed_buckets( + db, + organization_id, + batch, + pricing_resolver=pricing_resolver, + ) + if claim_key: + _record_claim_committed_pg(db, claim_key, organization_id) + db.commit() + flushed += batch_flushed + if remainder: + _restore_buckets_to_pending(organization_id, remainder) + if skipped: + _restore_buckets_to_pending(organization_id, skipped) + if claim_key: + _mark_claim_committed(claim_key, fast=True) + _ack_claim(claim_key, organization_id) + except Exception as exc: + db.rollback() + if _is_missing_organization_fk(exc): + logger.warning( + "llm usage flush dropped for unknown organization {}: {}", + organization_id, + exc, + ) + if claim_key: + try: + _record_claim_committed_pg(db, claim_key, organization_id) + db.commit() + except Exception: + db.rollback() + _discard_committed_claim(claim_key, organization_id) + return flushed + logger.warning("llm usage catalog flush failed, restoring redis: {}", exc) + restore_buckets = dict(claimed_buckets) + _restore_buckets_to_pending(organization_id, restore_buckets) + if claim_key: + try: + _client().delete(claim_key) + except redis.RedisError: + pass + return flushed + + return flushed + + def _is_unique_violation(exc: BaseException) -> bool: text_blob = " ".join( str(part) @@ -913,10 +1073,13 @@ def _flush_pending_buffer( FROM usage_pending_buffer WHERE organization_id = CAST(:organization_id AS uuid) ORDER BY created_at ASC - LIMIT 2000 + LIMIT :batch_limit """ ), - {"organization_id": str(organization_id)}, + { + "organization_id": str(organization_id), + "batch_limit": flush_bucket_batch_size(), + }, ).mappings().all() except Exception as exc: db.rollback() @@ -997,15 +1160,15 @@ def _flush_pending_buffer( return 0 -_CATALOG_FLUSH_COOLDOWN_SEC = 20 - - def _catalog_flush_recently(organization_id: UUID) -> bool: """Skip flush if another request flushed this org within the cooldown window.""" + cooldown = api_flush_cooldown_seconds() + if cooldown <= 0: + return False try: client = _client() key = f"usage:catalog_flush:{organization_id}" - return not client.set(key, "1", nx=True, ex=_CATALOG_FLUSH_COOLDOWN_SEC) + return not client.set(key, "1", nx=True, ex=cooldown) except redis.RedisError: return False @@ -1022,79 +1185,18 @@ def flush_usage_to_catalog(db: Session, organization_id: UUID, *, force: bool = pricing_resolver = PricingResolver(db) if not skip_redis_flush: redis_locked = _acquire_flush_lock(organization_id) - claim_key = None - buckets: Dict[str, Dict[str, int]] = {} try: if redis_locked: - claim_key, buckets = _claim_pending(organization_id) - if claim_key and buckets: - skipped: Dict[str, Dict[str, int]] = {} - try: - for prefix, deltas in buckets.items(): - parsed = _parse_bucket_prefix(prefix) - if not parsed: - skipped[prefix] = deltas - logger.warning( - "llm usage skipped unparseable bucket prefix for org {}", - organization_id, - ) - continue - _upsert_bucket( - db, - organization_id, - parsed, - deltas, - pricing_resolver=pricing_resolver, - ) - flushed += 1 - if claim_key: - _record_claim_committed_pg(db, claim_key, organization_id) - db.commit() - except Exception as exc: - db.rollback() - if _is_missing_organization_fk(exc): - logger.warning( - "llm usage flush dropped for unknown organization {}: {}", - organization_id, - exc, - ) - if claim_key: - try: - _record_claim_committed_pg( - db, claim_key, organization_id - ) - db.commit() - except Exception: - db.rollback() - _discard_committed_claim(claim_key, organization_id) - claim_key = None - buckets = {} - else: - logger.warning( - "llm usage catalog flush failed, restoring redis: {}", exc - ) - restore_buckets = dict(buckets) - if skipped: - restore_buckets.update(skipped) - _restore_buckets_to_pending(organization_id, restore_buckets) - if claim_key: - try: - _client().delete(claim_key) - except redis.RedisError: - pass - claim_key = None - return _flush_pending_buffer(db, organization_id) - if skipped: - _restore_buckets_to_pending(organization_id, skipped) - if claim_key: - _mark_claim_committed(claim_key, fast=True) - _ack_claim(claim_key, organization_id) - claim_key = None + flushed += _flush_redis_pending_to_catalog( + db, + organization_id, + pricing_resolver=pricing_resolver, + ) finally: if redis_locked: _release_flush_lock(organization_id) - flushed += _flush_pending_buffer(db, organization_id) + flushed += _flush_pending_buffer(db, organization_id, pricing_resolver=pricing_resolver) return flushed diff --git a/app/services/usage/retention.py b/app/services/usage/retention.py new file mode 100644 index 00000000..763cbbcc --- /dev/null +++ b/app/services/usage/retention.py @@ -0,0 +1,41 @@ +"""OSS usage rollup retention — per-org flush beyond history window.""" + +from __future__ import annotations + +from datetime import date, timedelta + +from sqlalchemy.orm import Session + +from loguru import logger + +from app.core.license import get_enabled_features, get_license_info +from app.core.usage_entitlement import OSS_USAGE_HISTORY_DAYS +from app.models.database import LLMUsageDaily + + +def oss_usage_cutoff_date() -> date: + return date.today() - timedelta(days=OSS_USAGE_HISTORY_DAYS - 1) + + +def prune_oss_usage_history(db: Session) -> dict: + """ + Delete rollup rows older than OSS window for non-entitled orgs. + Deployment-wide license → no deletes. Org-scoped license → skip licensed org. + """ + if get_enabled_features() and get_license_info().get("org_id") is None: + return {"deleted": 0} + + cutoff = oss_usage_cutoff_date() + licensed_org = get_license_info().get("org_id") + + query = db.query(LLMUsageDaily).filter(LLMUsageDaily.usage_date < cutoff) + if get_enabled_features() and licensed_org is not None: + query = query.filter(LLMUsageDaily.organization_id != licensed_org) + + deleted = query.delete(synchronize_session=False) + db.commit() + + if deleted: + logger.info("pruned_oss_usage_history deleted_rows={} cutoff={}", deleted, cutoff) + + return {"deleted": deleted, "cutoff": cutoff.isoformat()} diff --git a/app/workers/config.py b/app/workers/config.py index e06336d0..b43306e7 100644 --- a/app/workers/config.py +++ b/app/workers/config.py @@ -140,14 +140,21 @@ "map_agent_flowchart_prompt_sections": {"queue": "celery"}, "flush_usage_counters": {"queue": USAGE_WORKER_QUEUE}, "recompute_usage_costs": {"queue": USAGE_WORKER_QUEUE}, + "prune_oss_usage_history": {"queue": USAGE_WORKER_QUEUE}, } # Periodic flush of Redis usage counters into catalog rollups (usage queue). _flush_interval = float(os.environ.get("USAGE_FLUSH_BEAT_SECONDS", "120")) +_prune_interval = float(os.environ.get("USAGE_PRUNE_BEAT_SECONDS", "86400")) celery_app.conf.beat_schedule = { "flush-llm-usage-counters": { "task": "flush_usage_counters", "schedule": _flush_interval, "options": {"queue": USAGE_WORKER_QUEUE}, }, + "prune-oss-usage-history": { + "task": "prune_oss_usage_history", + "schedule": _prune_interval, + "options": {"queue": USAGE_WORKER_QUEUE}, + }, } diff --git a/app/workers/tasks/__init__.py b/app/workers/tasks/__init__.py index 499d9a5d..1f237e29 100644 --- a/app/workers/tasks/__init__.py +++ b/app/workers/tasks/__init__.py @@ -24,6 +24,7 @@ from . import call_import_bulk_ops from . import flush_usage_counters from . import recompute_usage_costs +from . import prune_oss_usage_history from app.workers.concurrency import eval_dispatch from app.workers.concurrency import fair_dispatch from app.workers.concurrency import fair_diarization_dispatch @@ -119,3 +120,4 @@ ) flush_usage_counters_task = flush_usage_counters.flush_usage_counters_task recompute_usage_costs_task = recompute_usage_costs.recompute_usage_costs_task +prune_oss_usage_history_task = prune_oss_usage_history.prune_oss_usage_history_task diff --git a/app/workers/tasks/prune_oss_usage_history.py b/app/workers/tasks/prune_oss_usage_history.py new file mode 100644 index 00000000..a91b186c --- /dev/null +++ b/app/workers/tasks/prune_oss_usage_history.py @@ -0,0 +1,22 @@ +"""Celery task: prune OSS usage rollup rows beyond history window.""" + +from __future__ import annotations + +from loguru import logger + +from app.database import SessionLocal +from app.workers.config import celery_app + + +@celery_app.task(name="prune_oss_usage_history") +def prune_oss_usage_history_task() -> dict: + from app.services.usage.retention import prune_oss_usage_history + + db = SessionLocal() + try: + result = prune_oss_usage_history(db) + finally: + db.close() + if result.get("deleted"): + logger.info("prune_oss_usage_history_task {}", result) + return result diff --git a/env.example b/env.example index cd34c25a..7e87c790 100644 --- a/env.example +++ b/env.example @@ -69,3 +69,13 @@ AUTH_LOCAL_ALLOW_SIGNUP=true # Enterprise license JWT (RS256). Obtain from the EfficientAI team. # EFFICIENTAI_LICENSE=eyJhbGciOi... +# ----------------------------------------------------------------------------- +# Usage cost flush (worker-usage + beat; optional overrides for api service too) +# See README "Usage Pricing Ops". Copy to .env for Docker Compose / local dev. +# ----------------------------------------------------------------------------- +USAGE_FLUSH_BUCKET_BATCH_SIZE=500 +USAGE_FLUSH_MAX_BATCHES_PER_RUN=30 +USAGE_FLUSH_BEAT_SECONDS=120 +USAGE_FLUSH_LOCK_TTL_SECONDS=300 +USAGE_API_FLUSH_COOLDOWN_SECONDS=60 + diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 287bdd2d..7a078b20 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -132,6 +132,24 @@ function EnterpriseGate({ feature, children }: { feature: string; children: Reac return <>{children} } +function EnterpriseLicenseGate({ children }: { children: React.ReactNode }) { + const { hasExtendedUsageHistory, isLoaded } = useLicenseStore() + + if (!isLoaded) { + return ( +
+
+
+ ) + } + + if (!hasExtendedUsageHistory()) { + return + } + + return <>{children} +} + function App() { return ( @@ -187,7 +205,14 @@ function App() { } /> } /> } /> - } /> + + + + } + /> } diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 6dfe8744..fc6ec8ec 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -100,12 +100,18 @@ export interface VoicePlaygroundBlindTestPair { y: VoicePlaygroundBlindTestAudioRef } +export interface UsagePolicy { + extended_history: boolean + max_history_days: number | null +} + export interface LicenseInfoResponse { is_enterprise: boolean enabled_features: string[] all_enterprise_features: string[] feature_catalog?: EnterpriseFeatureCatalog organization?: string + usage_policy?: UsagePolicy } export interface ReportBranding { @@ -2991,6 +2997,11 @@ class ApiClient { return response.data } + async syncOrgUsageCatalog(): Promise<{ flushed_buckets: number }> { + const response = await this.client.post('/api/v1/organizations/usage/catalog/sync') + return response.data + } + async getOrgUsageSummary(params: { start?: string end?: string diff --git a/frontend/src/pages/enterprise/EnterpriseUpgrade.tsx b/frontend/src/pages/enterprise/EnterpriseUpgrade.tsx index 8c990df7..46c5fc2e 100644 --- a/frontend/src/pages/enterprise/EnterpriseUpgrade.tsx +++ b/frontend/src/pages/enterprise/EnterpriseUpgrade.tsx @@ -3,9 +3,9 @@ import { useLicenseStore } from '../../store/licenseStore' const FALLBACK_TITLE = 'Enterprise Feature' -export default function EnterpriseUpgrade({ feature }: { feature: string }) { +export default function EnterpriseUpgrade({ feature }: { feature?: string }) { const getFeatureMeta = useLicenseStore((state) => state.getFeatureMeta) - const title = getFeatureMeta(feature)?.title ?? FALLBACK_TITLE + const title = feature ? (getFeatureMeta(feature)?.title ?? FALLBACK_TITLE) : 'Extended Usage History' return (
@@ -21,7 +21,9 @@ export default function EnterpriseUpgrade({ feature }: { feature: string }) {

- This feature is available with an EfficientAI Enterprise license. + {feature + ? 'This feature is available with an EfficientAI Enterprise license.' + : 'Extended usage history and pricing tools require a valid EfficientAI Enterprise license with any enabled feature.'}

diff --git a/frontend/src/pages/usage/Usage.tsx b/frontend/src/pages/usage/Usage.tsx index 1b049984..a87cf73c 100644 --- a/frontend/src/pages/usage/Usage.tsx +++ b/frontend/src/pages/usage/Usage.tsx @@ -8,8 +8,9 @@ import { useIsAdmin } from '../../hooks/useRole' import UsageFiltersBar from './UsageFiltersBar' import UsageDrillPath from './UsageDrillPath' import UsageCostBreakdownModal from './UsageCostBreakdownModal' -import { defaultUsageDateRange } from './UsageDateRangePicker' +import { defaultUsageDateRange, isRangeWithinMaxDays, rangeForDays } from './UsageDateRangePicker' import { getUsageTimezone } from './usageTimezone' +import { useLicenseStore } from '../../store/licenseStore' import { CALL_IMPORT_BATCH_HEADLINE, CALL_IMPORT_HINT, @@ -561,8 +562,35 @@ export default function Usage() { const usageKind = (searchParams.get('usage_kind') as Kind) || '' const productSection = searchParams.get('product_section') || '' const isAdmin = useIsAdmin() + const { usagePolicy, isLoaded: licenseLoaded, fetchLicense } = useLicenseStore() + const showOssUsageNotice = licenseLoaded && !usagePolicy.extended_history + const maxHistoryDays = !licenseLoaded + ? null + : usagePolicy.extended_history + ? null + : usagePolicy.max_history_days ?? 7 const [costBreakdownOpen, setCostBreakdownOpen] = useState(false) + useEffect(() => { + if (!licenseLoaded) { + void fetchLicense() + } + }, [licenseLoaded, fetchLicense]) + + useEffect(() => { + if (!licenseLoaded || usagePolicy.extended_history) return + const maxDays = usagePolicy.max_history_days ?? 7 + if (!isRangeWithinMaxDays(start, end, maxDays)) { + const r = rangeForDays(maxDays) + setSearchParams((prev) => { + const next = new URLSearchParams(prev) + next.set('start', r.start) + next.set('end', r.end) + return next + }) + } + }, [licenseLoaded, usagePolicy.extended_history, usagePolicy.max_history_days, start, end, setSearchParams]) + const showWorkspaceComposite = Boolean(workspaceId) && !callImportId && @@ -609,13 +637,25 @@ export default function Usage() { staleTime: 60 * 1000, } + const catalogSync = useQuery({ + queryKey: ['org-usage', 'catalog-sync'], + queryFn: () => apiClient.syncOrgUsageCatalog(), + staleTime: 60 * 1000, + refetchOnWindowFocus: false, + }) + + const usageReadsReady = catalogSync.isSuccess || catalogSync.isError + const { data: summary, isLoading: summaryLoading, isFetching: summaryFetching } = useQuery({ queryKey: ['org-usage', 'summary', dataParams], queryFn: () => apiClient.getOrgUsageSummary(dataParams), + enabled: usageReadsReady, ...usageQueryDefaults, placeholderData: keepPreviousData, }) + const usageStatsLoading = catalogSync.isLoading || summaryLoading + const { data: breakdown, isLoading: breakdownLoading, @@ -628,7 +668,7 @@ export default function Usage() { group_by: groupBy, limit: 100, }), - enabled: !showWorkspaceComposite, + enabled: usageReadsReady && !showWorkspaceComposite, ...usageQueryDefaults, placeholderData: (previousData, previousQuery) => { if (!previousQuery || previousQuery.queryKey[2] !== groupBy) return undefined @@ -648,7 +688,7 @@ export default function Usage() { group_by: 'call_import', limit: 100, }), - enabled: showWorkspaceComposite, + enabled: usageReadsReady && showWorkspaceComposite, ...usageQueryDefaults, }) @@ -664,13 +704,14 @@ export default function Usage() { group_by: 'resource', limit: 100, }), - enabled: showWorkspaceComposite, + enabled: usageReadsReady && showWorkspaceComposite, ...usageQueryDefaults, }) const { data: filterOptions, isFetching: filtersLoading } = useQuery({ queryKey: ['org-usage', 'filters', scopeParams], queryFn: () => apiClient.getOrgUsageFilters(scopeParams), + enabled: usageReadsReady, staleTime: 60 * 1000, placeholderData: (previousData, previousQuery) => { if (!previousQuery) return undefined @@ -1077,7 +1118,7 @@ export default function Usage() { Cost breakdown ) : null} - {isAdmin ? ( + {isAdmin && licenseLoaded && usagePolicy.extended_history ? (
+ {showOssUsageNotice ? ( +
+ Showing the last {maxHistoryDays} days of usage history. Set{' '} + + EFFICIENTAI_LICENSE + {' '} + on your server with any enterprise feature to unlock extended history. +
+ ) : null} +
- - - - + + + +
@@ -1110,31 +1161,31 @@ export default function Usage() { ) : null} {showTts ? ( - + ) : null} {(totals?.cache_read_tokens || 0) > 0 ? ( - + ) : null} {(totals?.cache_creation_tokens || 0) > 0 ? ( ) : null} {(totals?.reasoning_tokens || 0) > 0 ? ( - + ) : null}
@@ -1196,6 +1247,7 @@ export default function Usage() { onUsageKindChange={(k) => setParams({ usage_kind: k || null })} onModelChange={(v) => setParams({ model: v || null })} onClearAll={handleClearAll} + maxHistoryDays={maxHistoryDays} /> diff --git a/frontend/src/pages/usage/UsageDateRangePicker.tsx b/frontend/src/pages/usage/UsageDateRangePicker.tsx index 3ffeaeec..cd5718ab 100644 --- a/frontend/src/pages/usage/UsageDateRangePicker.tsx +++ b/frontend/src/pages/usage/UsageDateRangePicker.tsx @@ -9,6 +9,7 @@ type UsageDateRangePickerProps = { start: string end: string onApply: (start: string, end: string) => void + maxHistoryDays?: number | null } function toDateInput(d: Date): string { @@ -23,13 +24,18 @@ function formatDisplay(start: string, end: string): string { return `${start} → ${end}` } -function rangeForDays(days: number): { start: string; end: string } { +export function rangeForDays(days: number): { start: string; end: string } { const end = new Date() const start = new Date() start.setDate(start.getDate() - (days - 1)) return { start: toDateInput(start), end: toDateInput(end) } } +export function isRangeWithinMaxDays(start: string, end: string, maxDays: number): boolean { + const r = rangeForDays(maxDays) + return start >= r.start && end <= r.end +} + const QUICK_RANGES = [ { label: '1d', days: 1 }, { label: '7d', days: 7 }, @@ -44,6 +50,7 @@ export default function UsageDateRangePicker({ start, end, onApply, + maxHistoryDays = null, }: UsageDateRangePickerProps) { const [open, setOpen] = useState(false) const [mode, setMode] = useState('relative') @@ -52,13 +59,25 @@ export default function UsageDateRangePicker({ const [relDays, setRelDays] = useState(1) const rootRef = useRef(null) + const quickRanges = useMemo(() => { + if (!maxHistoryDays) return QUICK_RANGES + return QUICK_RANGES.filter((q) => q.days <= maxHistoryDays) + }, [maxHistoryDays]) + + const minStartDate = useMemo(() => { + if (!maxHistoryDays) return null + return rangeForDays(maxHistoryDays).start + }, [maxHistoryDays]) + + const maxRelDays = maxHistoryDays ?? 365 + const activeQuick = useMemo(() => { - for (const q of QUICK_RANGES) { + for (const q of quickRanges) { const r = rangeForDays(q.days) if (r.start === start && r.end === end) return q.label } return null - }, [start, end]) + }, [start, end, quickRanges]) useEffect(() => { if (open) { @@ -96,7 +115,7 @@ export default function UsageDateRangePicker({ return (
- {QUICK_RANGES.map((q) => ( + {quickRanges.map((q) => ( + ))} +
{showCostBreakdown ? (
) diff --git a/frontend/src/pages/usage/UsageCostBreakdownModal.tsx b/frontend/src/pages/usage/UsageCostBreakdownModal.tsx index 9277de53..645729ba 100644 --- a/frontend/src/pages/usage/UsageCostBreakdownModal.tsx +++ b/frontend/src/pages/usage/UsageCostBreakdownModal.tsx @@ -17,9 +17,10 @@ type Props = { onClose: () => void costs?: UsageCosts | null scopeLabel?: string + formatCost?: (usd?: number | null) => string } -function formatCostUsd(usd?: number | null): string { +function defaultFormatCostUsd(usd?: number | null): string { const amount = Number(usd || 0) if (!amount) return '$0.00' return new Intl.NumberFormat(undefined, { @@ -45,6 +46,7 @@ export default function UsageCostBreakdownModal({ onClose, costs, scopeLabel, + formatCost = defaultFormatCostUsd, }: Props) { if (!isOpen) return null @@ -93,7 +95,7 @@ export default function UsageCostBreakdownModal({
Estimated total - {formatCostUsd(costs?.total_cost_usd)} + {formatCost(costs?.total_cost_usd)}
{rows.length > 0 ? ( @@ -105,7 +107,7 @@ export default function UsageCostBreakdownModal({ > {item.label} - {formatCostUsd(costs?.[item.key] as number)} + {formatCost(costs?.[item.key] as number)} ))} diff --git a/frontend/src/pages/usage/UsagePricing.tsx b/frontend/src/pages/usage/UsagePricing.tsx index f976d116..92d159f3 100644 --- a/frontend/src/pages/usage/UsagePricing.tsx +++ b/frontend/src/pages/usage/UsagePricing.tsx @@ -9,6 +9,12 @@ import { useToast } from '../../hooks/useToast' import Button from '../../components/Button' import SearchableSelect from './SearchableSelect' import { usageTheme } from './usageTheme' +import { + formatUsageCostUsd, + getUsageDisplayCurrency, + setUsageDisplayCurrency, + type UsageDisplayCurrency, +} from '../../lib/usageCurrency' type UsageKind = 'llm' | 'stt' | 'tts' @@ -74,15 +80,6 @@ function FormField({ ) } -function formatUsd(value?: number | null): string { - if (value == null || Number.isNaN(value)) return '—' - return new Intl.NumberFormat(undefined, { - style: 'currency', - currency: 'USD', - minimumFractionDigits: 2, - maximumFractionDigits: 6, - }).format(value) -} function todayIso(): string { return new Date().toISOString().slice(0, 10) @@ -93,6 +90,18 @@ export default function UsagePricing() { const queryClient = useQueryClient() const { showToast, ToastContainer } = useToast() + const [displayCurrency, setDisplayCurrency] = useState(() => + getUsageDisplayCurrency(), + ) + const { data: fxRate } = useQuery({ + queryKey: ['org-usage', 'fx-rate'], + queryFn: () => apiClient.getOrgUsageFxRate(), + staleTime: 60 * 60 * 1000, + }) + const inrRate = fxRate?.rate ?? 83 + const formatDisplayRate = (value?: number | null) => + formatUsageCostUsd(value, displayCurrency, inrRate) + const [model, setModel] = useState('') const [usageKind, setUsageKind] = useState('llm') const [effectiveFrom, setEffectiveFrom] = useState(todayIso()) @@ -197,6 +206,29 @@ export default function UsagePricing() {

Pricing overrides

+
+ {(['USD', 'INR'] as const).map((currency) => ( + + ))} +
- ))} -
- {showCostBreakdown ? ( - - ) : null} - {isAdmin && licenseLoaded && usagePolicy.extended_history ? ( - - Pricing overrides - - ) : null} -
- {summary?.last_updated_at ? ( -

- Updated {new Date(summary.last_updated_at).toLocaleString()} -

- ) : null} -
-
- - {showOssUsageNotice ? ( -
- Showing the last {maxHistoryDays} days of usage history. Set{' '} - - EFFICIENTAI_LICENSE - {' '} - on your server with any enterprise feature to unlock extended history. -
- ) : null} - -
-
- - - - -
- -
-
- {showAudio ? ( - - ) : null} - {showTts ? ( - - ) : null} - {(totals?.cache_read_tokens || 0) > 0 ? ( - - ) : null} - {(totals?.cache_creation_tokens || 0) > 0 ? ( - - ) : null} - {(totals?.reasoning_tokens || 0) > 0 ? ( - - ) : null} -
-
- -
-
-
- - setParams({ start: s, end: e })} - onWorkspaceChange={handleWorkspaceChange} - onCallImportChange={handleCallImportChange} - onDatasetChange={(v) => - setParams({ - dataset: v || null, - call_import_id: null, - resource_id: null, - }) - } - onTagChange={(v) => - setParams({ - tag_id: v || null, - call_import_id: null, - resource_id: null, - }) - } - onEvaluationChange={(id) => { - if (id.startsWith(USAGE_SECTION_SOURCE_PREFIX)) { - const section = id.slice(USAGE_SECTION_SOURCE_PREFIX.length) - setParams({ - product_section: section || null, - resource_id: null, - call_import_id: null, - model: null, - usage_kind: null, - }) - return - } - const resource = filterOptions?.resources?.find( - (r) => idKey(r.id) === idKey(id), - ) - setParams({ - resource_id: id || null, - product_section: id ? resource?.product_section || null : null, - model: null, - usage_kind: null, - }) - }} - onUsageKindChange={(k) => setParams({ usage_kind: k || null })} - onModelChange={(v) => setParams({ model: v || null })} - onClearAll={handleClearAll} - maxHistoryDays={maxHistoryDays} - /> - - -
- - {showTruncation ? ( -

- Showing the first 100 rows for this level. Narrow the date range or drill - further for complete detail. -

- ) : null} -
- - {tableLoading || breakdownStale ? ( -
- -
- ) : rows.length === 0 ? ( -
- No usage in this period for the current scope. Try 7d or 30d, or go back up a - level. -
- ) : ( -
- {tableFetching ? ( -
- -
- ) : null} - - - - - - - - - - - - - - - - {rows.map((row, idx) => { - const drillable = isRowDrillable(row) - const compositeHeadline = - 'rowKind' in row ? compositeRowHeadline(row) : null - const rowTitle = tableRowLabel(groupBy, row, filterOptions) - const showRowTitle = - 'rowKind' in row - ? row.rowKind === 'workspace_resource' || - (compositeHeadline && - rowTitle.toLowerCase() !== compositeHeadline.toLowerCase()) - : true - return ( - drillable && handleRowDrill(row)} - > - - - - - - - - - - - ) - })} - -
- {drillColumnLabel(groupBy, showWorkspaceComposite)} - LLM callsInput tokensOutput tokensTotal tokensEst. costSTT audioTTS charsCache read
- - - {'rowKind' in row ? ( - - {compositeHeadline} - - ) : null} - {showRowTitle ? ( - - {rowTitle} - - ) : !('rowKind' in row) ? ( - {rowTitle} - ) : null} - {'hint' in row && row.hint ? ( - - {row.hint} - - ) : null} - - {drillable ? ( - - ) : null} - - - {formatNumber(row.call_count)} - - {formatNumber(row.prompt_tokens)} - - {formatNumber(row.completion_tokens)} - - {formatNumber(row.total_tokens)} - - {formatCostUsd(rowCostUsd(row))} - - {row.audio_seconds ? formatAudio(row.audio_seconds) : '—'} - - {row.tts_characters ? formatNumber(row.tts_characters) : '—'} - - {formatNumber(row.cache_read_tokens)} -
-
- )} - - - - setCostBreakdownOpen(false)} - costs={totals?.costs} - scopeLabel={scopeSubtitle} - formatCost={formatCostUsd} - /> -
- ) -} - -const statCardClass = 'border border-gray-200 ring-1 ring-[#fde047]/25 shadow-sm' - -function StatCard({ - label, - value, - valueLabel, - loading, - emphasize = false, - className = '', -}: { - label: string - value?: number - valueLabel?: string - loading?: boolean - emphasize?: boolean - className?: string -}) { - return ( - - -

{label}

-

- {loading ? '—' : valueLabel ?? formatNumber(value || 0)} -

- - - ) -} +import { useEffect, useMemo, useState } from 'react' +import { keepPreviousData, useQuery } from '@tanstack/react-query' +import { Link, useSearchParams } from 'react-router-dom' +import { Card, CardBody, Spinner } from '@heroui/react' +import { Activity, ChevronRight, CircleDollarSign } from 'lucide-react' +import { apiClient } from '../../lib/api' +import { useIsAdmin } from '../../hooks/useRole' +import UsageFiltersBar from './UsageFiltersBar' +import UsageDrillPath from './UsageDrillPath' +import UsageCostBreakdownModal from './UsageCostBreakdownModal' +import { defaultUsageDateRange, isRangeWithinMaxDays, rangeForDays } from './UsageDateRangePicker' +import { getUsageTimezone } from './usageTimezone' +import { + formatUsageCostUsd, + getUsageDisplayCurrency, + setUsageDisplayCurrency, + type UsageDisplayCurrency, +} from '../../lib/usageCurrency' +import { useLicenseStore } from '../../store/licenseStore' +import { + CALL_IMPORT_BATCH_HEADLINE, + CALL_IMPORT_HINT, + CALL_IMPORT_PRODUCT_SECTIONS, + PRODUCT_SECTION_HEADLINES, + PRODUCT_SECTION_HINTS, + USAGE_SECTION_SOURCE_PREFIX, +} from './usageProductHints' + +type DrillGroupBy = + | 'workspace' + | 'call_import' + | 'resource' + | 'model' + | 'usage_kind' + | 'product_section' +type Kind = '' | 'llm' | 'stt' | 'tts' + +type FilterOptions = { + workspaces: Array<{ id: string; name: string }> + call_imports: Array<{ id: string; label: string }> + evaluations: Array<{ id: string; label: string }> + resources?: Array<{ id: string; label: string; type?: string; product_section?: string }> + models: string[] + usage_kinds: Array<{ id: string; label: string }> + product_sections?: Array<{ id: string; label: string }> + datasets?: string[] + tags?: Array<{ id: string; label: string }> +} + +type UsageCosts = { + input_cost_usd: number + output_cost_usd: number + cache_read_cost_usd: number + cache_write_cost_usd: number + reasoning_cost_usd: number + audio_cost_usd: number + tts_cost_usd: number + total_cost_usd: number + currency: string + has_unpriced_usage: boolean +} + +type BreakdownRow = { + workspace_id?: string | null + workspace_name?: string | null + call_import_id?: string | null + call_import_label?: string | null + resource_id?: string | null + resource_type?: string | null + resource_label?: string | null + model?: string | null + usage_kind?: string | null + product_section?: string | null + product_section_label?: string | null + prompt_tokens: number + completion_tokens: number + total_tokens: number + cache_read_tokens: number + cache_creation_tokens: number + reasoning_tokens: number + audio_seconds: number + tts_characters: number + call_count: number + input_cost_micro_usd?: number + output_cost_micro_usd?: number + cache_read_cost_micro_usd?: number + cache_creation_cost_micro_usd?: number + reasoning_cost_micro_usd?: number + audio_cost_micro_usd?: number + tts_cost_micro_usd?: number + total_cost_micro_usd?: number + costs?: UsageCosts +} + +type WorkspaceSourceRow = BreakdownRow & { + rowKind: 'call_import' | 'workspace_resource' + hint: string +} + +const NON_COMPOSITE_RESOURCE_TYPES = new Set([ + 'call_import', + 'call_import_evaluation', +]) + +type TableRow = BreakdownRow | WorkspaceSourceRow + +const EMPTY_METRICS = { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + cache_read_tokens: 0, + cache_creation_tokens: 0, + reasoning_tokens: 0, + audio_seconds: 0, + tts_characters: 0, + call_count: 0, + input_cost_micro_usd: 0, + output_cost_micro_usd: 0, + cache_read_cost_micro_usd: 0, + cache_creation_cost_micro_usd: 0, + reasoning_cost_micro_usd: 0, + audio_cost_micro_usd: 0, + tts_cost_micro_usd: 0, + total_cost_micro_usd: 0, + costs: { + input_cost_usd: 0, + output_cost_usd: 0, + cache_read_cost_usd: 0, + cache_write_cost_usd: 0, + reasoning_cost_usd: 0, + audio_cost_usd: 0, + tts_cost_usd: 0, + total_cost_usd: 0, + currency: 'USD', + has_unpriced_usage: false, + }, +} + +function compositeRowHeadline(row: WorkspaceSourceRow): string { + if (row.rowKind === 'call_import') return CALL_IMPORT_BATCH_HEADLINE + const section = row.product_section || '' + return ( + PRODUCT_SECTION_HEADLINES[section] || + row.product_section_label || + section || + 'Other' + ) +} + +function usableResourceLabel(label: string | null | undefined): string | undefined { + if (!label || label === 'Unscoped') return undefined + return label +} + +function compositeRowTitle(row: WorkspaceSourceRow, options?: FilterOptions): string { + if (row.rowKind === 'call_import') { + return row.call_import_label || 'Call import batch' + } + const fromFilters = row.resource_id + ? options?.resources?.find((r) => idKey(r.id) === idKey(row.resource_id))?.label + : undefined + return ( + usableResourceLabel(row.resource_label) || + fromFilters || + row.product_section_label || + 'Unscoped usage' + ) +} + +function sortRows(rows: BreakdownRow[]): BreakdownRow[] { + return [...rows].sort((a, b) => b.total_tokens - a.total_tokens) +} + +function formatNumber(value: number): string { + return new Intl.NumberFormat().format(value || 0) +} + +function rowCostUsd(row: Pick): number { + if (row.costs?.total_cost_usd != null) return row.costs.total_cost_usd + return Number(row.total_cost_micro_usd || 0) / 1_000_000 +} + +function formatAudio(seconds: number): string { + const total = Math.max(0, Math.floor(seconds || 0)) + if (total < 60) return `${total}s` + const mins = Math.floor(total / 60) + const secs = total % 60 + if (mins < 60) return secs ? `${mins}m ${secs}s` : `${mins}m` + const hours = Math.floor(mins / 60) + const remMins = mins % 60 + return remMins ? `${hours}h ${remMins}m` : `${hours}h` +} + +function rowHasUsageForKind(row: BreakdownRow, kind: Kind): boolean { + if (!kind) return true + if (kind === 'llm') { + return (row.total_tokens ?? 0) > 0 || (row.call_count ?? 0) > 0 + } + if (kind === 'stt') { + return (row.audio_seconds ?? 0) > 0 || (row.call_count ?? 0) > 0 + } + if (kind === 'tts') { + return (row.tts_characters ?? 0) > 0 || (row.call_count ?? 0) > 0 + } + return true +} + +function filterRowsForUsageKind(rows: BreakdownRow[], kind: Kind): BreakdownRow[] { + if (!kind) return rows + return rows.filter((row) => rowHasUsageForKind(row, kind)) +} + +function rowHasAnyUsage(row: BreakdownRow): boolean { + return ( + (row.total_tokens ?? 0) > 0 || + (row.call_count ?? 0) > 0 || + (row.audio_seconds ?? 0) > 0 || + (row.tts_characters ?? 0) > 0 + ) +} + +function isUsageScopeActive( + workspaceId: string, + callImportId: string, + evaluationId: string, + dataset: string, + tagId: string, + usageKind: Kind, + model: string, + productSection: string, +): boolean { + return Boolean( + workspaceId || + callImportId || + evaluationId || + dataset || + tagId || + usageKind || + model || + productSection, + ) +} + +/** Drop zero rows when any filter is active; kind filter uses kind-specific metrics. */ +function filterTableRows( + rows: BreakdownRow[], + usageKind: Kind, + scopeActive: boolean, +): BreakdownRow[] { + if (usageKind) return filterRowsForUsageKind(rows, usageKind) + if (scopeActive) return rows.filter(rowHasAnyUsage) + return rows +} + +function drillGroupBy( + workspaceId: string, + callImportId: string, + evaluationId: string, + model: string, + productSection: string, +): DrillGroupBy { + if (productSection && model) return 'usage_kind' + if (productSection) return 'model' + if (evaluationId && model) return 'usage_kind' + if (evaluationId) return 'model' + if (callImportId) return 'resource' + if (workspaceId) return 'call_import' + return 'workspace' +} + +function enrichCallImportRows( + rawRows: BreakdownRow[], + options?: FilterOptions, +): BreakdownRow[] { + const labelById = new Map( + (options?.call_imports ?? []).map((c) => [idKey(c.id), c.label]), + ) + return sortRows( + rawRows + .filter((r) => r.call_import_id) + .map((row) => { + const key = idKey(row.call_import_id) + return { + ...row, + call_import_label: + labelById.get(key) || row.call_import_label || 'Call import', + } + }), + ) +} + +function buildWorkspaceCompositeRows( + callImportRaw: BreakdownRow[], + resourceRaw: BreakdownRow[], + options?: FilterOptions, + padMissingRows = true, +): WorkspaceSourceRow[] { + const resourceLabelById = new Map( + (options?.resources ?? []).map((r) => [idKey(r.id), r.label]), + ) + const importRows = enrichCallImportRows(callImportRaw, options) + const rows: WorkspaceSourceRow[] = [] + const shownImportIds = new Set() + + for (const row of importRows) { + if (!row.call_import_id) continue + shownImportIds.add(idKey(row.call_import_id)) + rows.push({ + ...row, + rowKind: 'call_import', + hint: CALL_IMPORT_HINT, + }) + } + + if (padMissingRows) { + for (const ci of options?.call_imports ?? []) { + const key = idKey(ci.id) + if (shownImportIds.has(key)) continue + shownImportIds.add(key) + rows.push({ + call_import_id: ci.id, + call_import_label: ci.label, + ...EMPTY_METRICS, + rowKind: 'call_import', + hint: CALL_IMPORT_HINT, + }) + } + } + + for (const row of resourceRaw) { + const section = row.product_section + if (!section || CALL_IMPORT_PRODUCT_SECTIONS.has(section)) continue + if (NON_COMPOSITE_RESOURCE_TYPES.has(row.resource_type || '')) continue + if ( + row.total_tokens === 0 && + row.call_count === 0 && + !row.audio_seconds && + !row.tts_characters + ) { + continue + } + if (!rowHasAnyUsage(row)) { + continue + } + rows.push({ + ...row, + rowKind: 'workspace_resource', + product_section: section, + product_section_label: row.product_section_label || section, + resource_label: + resourceLabelById.get(idKey(row.resource_id)) || + usableResourceLabel(row.resource_label), + hint: PRODUCT_SECTION_HINTS[section] || 'Product usage', + }) + } + + return sortRows(rows) as WorkspaceSourceRow[] +} + +function drillColumnLabel(groupBy: DrillGroupBy, composite = false): string { + if (composite) return 'Source' + if (groupBy === 'workspace') return 'Workspace' + if (groupBy === 'call_import') return 'Call import' + if (groupBy === 'resource') return 'Evaluation run' + if (groupBy === 'product_section') return 'Product area' + if (groupBy === 'model') return 'Model' + return 'Kind' +} + +function rowLabel(groupBy: DrillGroupBy, row: BreakdownRow, options?: FilterOptions): string { + if (groupBy === 'workspace') return row.workspace_name || 'Unknown' + if (groupBy === 'call_import') return row.call_import_label || 'Call import' + if (groupBy === 'resource') { + const fromFilters = row.resource_id + ? options?.resources?.find((r) => idKey(r.id) === idKey(row.resource_id))?.label + : undefined + return ( + usableResourceLabel(row.resource_label) || fromFilters || 'Unscoped' + ) + } + if (groupBy === 'model') return row.model || '—' + if (groupBy === 'product_section') + return row.product_section_label || row.product_section || '—' + if (row.usage_kind === 'stt') return 'STT' + if (row.usage_kind === 'llm') return 'LLM' + if (row.usage_kind === 'tts') return 'TTS' + return row.usage_kind || '—' +} + +function tableRowLabel( + groupBy: DrillGroupBy, + row: TableRow, + options?: FilterOptions, +): string { + if ('rowKind' in row) { + if (row.rowKind === 'call_import') return row.call_import_label || 'Call import batch' + return compositeRowTitle(row, options) + } + return rowLabel(groupBy, row, options) +} + +function idInOptions( + id: string, + options: Array<{ id: string }> | undefined, +): boolean { + if (!id || !options?.length) return false + const key = idKey(id) + return options.some((o) => idKey(o.id) === key) +} + +function idKey(id: string | null | undefined): string { + return id ? String(id).toLowerCase() : '' +} + +function mergeDrillRows( + groupBy: DrillGroupBy, + rawRows: BreakdownRow[], + options?: FilterOptions, + padMissingRows = true, +): BreakdownRow[] { + if (groupBy === 'workspace' && options?.workspaces?.length && padMissingRows) { + const byId = new Map( + rawRows + .filter((r) => r.workspace_id) + .map((r) => [idKey(r.workspace_id), r]), + ) + const merged: BreakdownRow[] = options.workspaces.map((ws) => ({ + workspace_id: ws.id, + workspace_name: ws.name, + ...(byId.get(idKey(ws.id)) ?? EMPTY_METRICS), + })) + for (const row of rawRows) { + if (!row.workspace_id) { + merged.push({ + ...row, + workspace_name: row.workspace_name || 'No workspace', + }) + } + } + return sortRows(merged) + } + + if (groupBy === 'call_import') { + if (rawRows.length === 0) return [] + + const labelById = new Map( + (options?.call_imports ?? []).map((c) => [idKey(c.id), c.label]), + ) + const shown = new Set() + const merged: BreakdownRow[] = [] + + for (const row of rawRows) { + if (row.call_import_id) { + const key = idKey(row.call_import_id) + shown.add(key) + merged.push({ + ...row, + call_import_label: + labelById.get(key) || row.call_import_label || 'Call import', + }) + } + } + + if (padMissingRows) { + for (const ci of options?.call_imports ?? []) { + const key = idKey(ci.id) + if (!shown.has(key)) { + merged.push({ + call_import_id: ci.id, + call_import_label: ci.label, + ...EMPTY_METRICS, + }) + } + } + } + + return sortRows(merged) + } + + if (groupBy === 'resource') { + if (rawRows.length === 0) return [] + + const evalLabelById = new Map( + (options?.evaluations ?? []).map((e) => [idKey(e.id), e.label]), + ) + const resourceLabelById = new Map( + (options?.resources ?? []).map((r) => [idKey(r.id), r.label]), + ) + const shown = new Set() + const merged: BreakdownRow[] = [] + + for (const row of rawRows) { + if (row.resource_id) { + const key = idKey(row.resource_id) + shown.add(key) + merged.push({ + ...row, + resource_label: + resourceLabelById.get(key) || + evalLabelById.get(key) || + usableResourceLabel(row.resource_label) || + 'Evaluation', + }) + } + } + + if (padMissingRows) { + for (const ev of options?.evaluations ?? []) { + const key = idKey(ev.id) + if (!shown.has(key)) { + merged.push({ + resource_id: ev.id, + resource_label: ev.label, + ...EMPTY_METRICS, + }) + } + } + } + + return sortRows(merged) + } + + if (groupBy === 'model' && options?.models?.length && padMissingRows) { + if (rawRows.length === 0) return [] + const byName = new Map( + rawRows.filter((r) => r.model).map((r) => [r.model!, r]), + ) + const merged = options.models.map((name) => ({ + model: name, + ...(byName.get(name) ?? EMPTY_METRICS), + })) + return sortRows(merged) + } + + if (groupBy === 'usage_kind') { + return sortRows( + rawRows.filter((row) => rowHasUsageForKind(row, row.usage_kind as Kind)), + ) + } + + return sortRows(rawRows) +} + +export default function Usage() { + const [searchParams, setSearchParams] = useSearchParams() + const defaultRange = useMemo(() => defaultUsageDateRange(), []) + const usageTimezone = useMemo(() => getUsageTimezone(), []) + + const start = searchParams.get('start') || defaultRange.start + const end = searchParams.get('end') || defaultRange.end + const workspaceId = searchParams.get('workspace_id') || '' + const callImportId = searchParams.get('call_import_id') || '' + const dataset = searchParams.get('dataset') || '' + const tagId = searchParams.get('tag_id') || '' + const evaluationId = searchParams.get('resource_id') || '' + const model = searchParams.get('model') || '' + const usageKind = (searchParams.get('usage_kind') as Kind) || '' + const productSection = searchParams.get('product_section') || '' + const isAdmin = useIsAdmin() + const { usagePolicy, isLoaded: licenseLoaded, fetchLicense } = useLicenseStore() + const showOssUsageNotice = licenseLoaded && !usagePolicy.extended_history + const maxHistoryDays = !licenseLoaded + ? null + : usagePolicy.extended_history + ? null + : usagePolicy.max_history_days ?? 7 + const [costBreakdownOpen, setCostBreakdownOpen] = useState(false) + const [displayCurrency, setDisplayCurrency] = useState(() => + getUsageDisplayCurrency(), + ) + + const { data: fxRate } = useQuery({ + queryKey: ['org-usage', 'fx-rate'], + queryFn: () => apiClient.getOrgUsageFxRate(), + staleTime: 60 * 60 * 1000, + }) + const inrRate = fxRate?.rate ?? 83 + const formatCostUsd = (usd?: number | null) => + formatUsageCostUsd(usd, displayCurrency, inrRate) + + useEffect(() => { + if (!licenseLoaded) { + void fetchLicense() + } + }, [licenseLoaded, fetchLicense]) + + useEffect(() => { + if (!licenseLoaded || usagePolicy.extended_history) return + const maxDays = usagePolicy.max_history_days ?? 7 + if (!isRangeWithinMaxDays(start, end, maxDays)) { + const r = rangeForDays(maxDays) + setSearchParams((prev) => { + const next = new URLSearchParams(prev) + next.set('start', r.start) + next.set('end', r.end) + return next + }) + } + }, [licenseLoaded, usagePolicy.extended_history, usagePolicy.max_history_days, start, end, setSearchParams]) + + const showWorkspaceComposite = + Boolean(workspaceId) && + !callImportId && + !evaluationId && + !productSection + + const groupBy = drillGroupBy( + workspaceId, + callImportId, + evaluationId, + model, + productSection, + ) + + const setParams = (updates: Record) => { + const next = new URLSearchParams(searchParams) + for (const [key, value] of Object.entries(updates)) { + if (!value) next.delete(key) + else next.set(key, value) + } + setSearchParams(next) + } + + const scopeParams = { + start, + end, + tz: usageTimezone, + workspace_id: workspaceId || undefined, + call_import_id: callImportId || undefined, + dataset: dataset || undefined, + tag_id: tagId || undefined, + product_section: productSection || undefined, + usage_kind: usageKind || undefined, + model: model || undefined, + resource_id: evaluationId || undefined, + } + + const dataParams = { + ...scopeParams, + evaluation_id: callImportId ? evaluationId || undefined : undefined, + } + + const usageQueryDefaults = { + staleTime: 0, + refetchOnMount: 'always' as const, + refetchOnWindowFocus: true, + } + + const { + data: summary, + isLoading: summaryLoading, + isFetching: summaryFetching, + isError: summaryError, + error: summaryQueryError, + } = useQuery({ + queryKey: ['org-usage', 'summary', dataParams], + queryFn: () => apiClient.getOrgUsageSummary(dataParams), + ...usageQueryDefaults, + placeholderData: keepPreviousData, + }) + + const usageStatsLoading = summaryLoading + + const { + data: breakdown, + isLoading: breakdownLoading, + isFetching: breakdownFetching, + } = useQuery({ + queryKey: ['org-usage', 'breakdown', groupBy, dataParams], + queryFn: () => + apiClient.getOrgUsageBreakdown({ + ...dataParams, + group_by: groupBy, + limit: 100, + }), + enabled: !showWorkspaceComposite, + ...usageQueryDefaults, + placeholderData: (previousData, previousQuery) => { + if (!previousQuery || previousQuery.queryKey[2] !== groupBy) return undefined + return previousData + }, + }) + + const { + data: importBreakdown, + isLoading: importBreakdownLoading, + isFetching: importBreakdownFetching, + } = useQuery({ + queryKey: ['org-usage', 'breakdown', 'call_import', dataParams], + queryFn: () => + apiClient.getOrgUsageBreakdown({ + ...dataParams, + group_by: 'call_import', + limit: 100, + }), + enabled: showWorkspaceComposite, + ...usageQueryDefaults, + }) + + const { + data: resourceBreakdown, + isLoading: resourceBreakdownLoading, + isFetching: resourceBreakdownFetching, + } = useQuery({ + queryKey: ['org-usage', 'breakdown', 'resource', dataParams], + queryFn: () => + apiClient.getOrgUsageBreakdown({ + ...dataParams, + group_by: 'resource', + limit: 100, + }), + enabled: showWorkspaceComposite, + ...usageQueryDefaults, + }) + + const { data: filterOptions, isFetching: filtersLoading } = useQuery({ + queryKey: ['org-usage', 'filters', scopeParams], + queryFn: () => apiClient.getOrgUsageFilters(scopeParams), + ...usageQueryDefaults, + placeholderData: (previousData, previousQuery) => { + if (!previousQuery) return undefined + const prevScope = previousQuery.queryKey[2] as typeof scopeParams + if (JSON.stringify(prevScope) !== JSON.stringify(scopeParams)) return undefined + return previousData + }, + }) + + useEffect(() => { + if (!filterOptions) return + const updates: Record = {} + if ( + workspaceId && + !idInOptions(workspaceId, filterOptions.workspaces) + ) { + updates.workspace_id = null + updates.call_import_id = null + updates.resource_id = null + updates.product_section = null + } + if ( + callImportId && + !idInOptions(callImportId, filterOptions.call_imports) + ) { + updates.call_import_id = null + updates.resource_id = null + updates.product_section = null + } + if ( + evaluationId && + !idInOptions(evaluationId, filterOptions.evaluations) && + !idInOptions(evaluationId, filterOptions.resources) + ) { + updates.resource_id = null + updates.product_section = null + } + if (model && !filterOptions.models?.includes(model)) { + updates.model = null + } + if ( + usageKind && + !filterOptions.usage_kinds?.some((k) => k.id === usageKind) + ) { + updates.usage_kind = null + } + if ( + productSection && + !filterOptions.product_sections?.some((s) => s.id === productSection) + ) { + updates.product_section = null + } + if (dataset && !filterOptions.datasets?.includes(dataset)) { + updates.dataset = null + } + if (tagId && !idInOptions(tagId, filterOptions.tags)) { + updates.tag_id = null + } + if (Object.keys(updates).length > 0) setParams(updates) + }, [ + filterOptions, + workspaceId, + callImportId, + dataset, + tagId, + evaluationId, + model, + usageKind, + productSection, + ]) + + const breakdownMatchesLevel = breakdown?.group_by === groupBy + const rawRows = useMemo((): BreakdownRow[] => { + if (!breakdown || !breakdownMatchesLevel) return [] + return breakdown.rows as BreakdownRow[] + }, [breakdown, breakdownMatchesLevel]) + + const mergeOptions = filterOptions + const scopeActive = isUsageScopeActive( + workspaceId, + callImportId, + evaluationId, + dataset, + tagId, + usageKind, + model, + productSection, + ) + const padMissingRows = !scopeActive + const filteredRawRows = useMemo( + () => filterTableRows(rawRows, usageKind, scopeActive), + [rawRows, usageKind, scopeActive], + ) + const rows: TableRow[] = useMemo(() => { + if (showWorkspaceComposite) { + const importReady = importBreakdown?.group_by === 'call_import' + const resourceReady = resourceBreakdown?.group_by === 'resource' + if (!importReady && !resourceReady) return [] + return buildWorkspaceCompositeRows( + importReady + ? filterTableRows(importBreakdown.rows as BreakdownRow[], usageKind, scopeActive) + : [], + resourceReady + ? filterTableRows(resourceBreakdown.rows as BreakdownRow[], usageKind, scopeActive) + : [], + mergeOptions, + padMissingRows, + ) + } + return mergeDrillRows(groupBy, filteredRawRows, mergeOptions, padMissingRows) + }, [ + showWorkspaceComposite, + importBreakdown, + resourceBreakdown, + groupBy, + filteredRawRows, + mergeOptions, + padMissingRows, + usageKind, + scopeActive, + ]) + + const breakdownStale = + !showWorkspaceComposite && breakdownFetching && !breakdownMatchesLevel + const tableLoading = showWorkspaceComposite + ? (importBreakdownLoading || resourceBreakdownLoading) && + !importBreakdown && + !resourceBreakdown + : breakdownLoading && !breakdown + const tableFetching = showWorkspaceComposite + ? importBreakdownFetching || resourceBreakdownFetching + : breakdownFetching + const totals = summary?.totals + const estimatedTotalCost = + totals?.costs?.total_cost_usd ?? (totals?.total_cost_micro_usd || 0) / 1_000_000 + const showCostBreakdown = + estimatedTotalCost > 0 || Boolean(totals?.costs?.has_unpriced_usage) + const showAudio = Boolean(totals?.audio_seconds) + const showTts = + Boolean(totals?.tts_characters) || + rows.some((r) => Boolean(r.tts_characters)) + + const showTruncation = + (!showWorkspaceComposite && + breakdownMatchesLevel && + Boolean(breakdown?.truncated_at_limit)) || + (showWorkspaceComposite && + Boolean( + importBreakdown?.truncated_at_limit || resourceBreakdown?.truncated_at_limit, + )) + + const workspaceLabel = + filterOptions?.workspaces?.find((w) => idKey(w.id) === idKey(workspaceId))?.name + const callImportLabel = + filterOptions?.call_imports?.find((c) => idKey(c.id) === idKey(callImportId))?.label + const evaluationLabel = + filterOptions?.resources?.find((r) => idKey(r.id) === idKey(evaluationId))?.label || + filterOptions?.evaluations?.find((e) => idKey(e.id) === idKey(evaluationId))?.label + + const productSectionLabel = + filterOptions?.product_sections?.find((s) => s.id === productSection)?.label + + const scopeSubtitle = model + ? model + : evaluationId + ? evaluationLabel || 'Evaluation' + : callImportId + ? callImportLabel || 'Call import' + : productSection + ? productSectionLabel || 'Product area' + : workspaceId + ? workspaceLabel || 'Workspace' + : 'Organization' + + const levelHint = (() => { + if (showWorkspaceComposite) { + return 'Call import batches and other product usage — click a row to drill down' + } + if (groupBy === 'workspace') return 'Click a workspace to drill down' + if (groupBy === 'call_import') return 'Click a call import to see evaluation runs' + if (groupBy === 'product_section') return 'Click a product area to see models used' + if (groupBy === 'resource') return 'Click an evaluation to see models used' + if (groupBy === 'model') return 'Click a model to see usage by kind' + return 'Token totals by LLM / STT / TTS' + })() + + const drillCrumbs = [ + { + label: 'Organization', + onClick: + workspaceId || + callImportId || + evaluationId || + model || + productSection + ? () => + setParams({ + workspace_id: null, + call_import_id: null, + resource_id: null, + model: null, + usage_kind: null, + product_section: null, + }) + : undefined, + }, + ...(workspaceId + ? [ + { + label: workspaceLabel || 'Workspace', + onClick: + callImportId || evaluationId || model || productSection + ? () => + setParams({ + call_import_id: null, + resource_id: null, + model: null, + usage_kind: null, + product_section: null, + }) + : undefined, + }, + ] + : []), + ...(productSection + ? [ + { + label: productSectionLabel || 'Product area', + onClick: + model + ? () => setParams({ model: null, usage_kind: null }) + : undefined, + }, + ] + : []), + ...(callImportId + ? [ + { + label: callImportLabel || 'Call import', + onClick: + evaluationId || model + ? () => + setParams({ + resource_id: null, + model: null, + usage_kind: null, + product_section: null, + }) + : undefined, + }, + ] + : []), + ...(evaluationId + ? [ + { + label: evaluationLabel || 'Evaluation', + onClick: model + ? () => setParams({ model: null, usage_kind: null }) + : undefined, + }, + ] + : []), + ...(model ? [{ label: model }] : []), + ] + + const handleWorkspaceChange = (id: string) => { + setParams({ + workspace_id: id || null, + call_import_id: null, + resource_id: null, + model: null, + usage_kind: null, + product_section: null, + }) + } + + const handleCallImportChange = (id: string) => { + setParams({ + call_import_id: id || null, + resource_id: null, + model: null, + usage_kind: null, + product_section: null, + }) + } + + const handleClearAll = () => { + setParams({ + workspace_id: null, + call_import_id: null, + dataset: null, + tag_id: null, + resource_id: null, + usage_kind: null, + model: null, + product_section: null, + }) + } + + const handleRowDrill = (row: TableRow) => { + if ('rowKind' in row) { + if (row.rowKind === 'call_import' && row.call_import_id) { + setParams({ + call_import_id: row.call_import_id, + resource_id: null, + model: null, + usage_kind: null, + product_section: null, + }) + return + } + if (row.rowKind === 'workspace_resource' && row.product_section) { + setParams({ + product_section: row.product_section, + call_import_id: null, + resource_id: row.resource_id || null, + model: null, + usage_kind: null, + }) + return + } + } + if (groupBy === 'workspace' && row.workspace_id) { + setParams({ + workspace_id: row.workspace_id, + call_import_id: null, + resource_id: null, + model: null, + usage_kind: null, + product_section: null, + }) + return + } + if (groupBy === 'call_import' && row.call_import_id) { + setParams({ + call_import_id: row.call_import_id, + resource_id: null, + model: null, + usage_kind: null, + product_section: null, + }) + return + } + if (groupBy === 'product_section' && row.product_section) { + setParams({ + product_section: row.product_section, + call_import_id: null, + resource_id: null, + model: null, + usage_kind: null, + }) + return + } + if (groupBy === 'resource' && row.resource_id) { + setParams({ + resource_id: row.resource_id, + model: null, + usage_kind: null, + product_section: null, + }) + return + } + if (groupBy === 'model' && row.model) { + setParams({ model: row.model, usage_kind: null }) + } + } + + const isRowDrillable = (row: TableRow): boolean => { + if ('rowKind' in row) { + if (row.rowKind === 'call_import') return Boolean(row.call_import_id) + return Boolean(row.product_section) + } + if (groupBy === 'workspace') return Boolean(row.workspace_id) + if (groupBy === 'call_import') return Boolean(row.call_import_id) + if (groupBy === 'product_section') return Boolean(row.product_section) + if (groupBy === 'resource') return Boolean(row.resource_id) + if (groupBy === 'model') return Boolean(row.model) + return false + } + + return ( +
+
+
+

+ + Usage +

+

+ Cards show usage for {scopeSubtitle}. + Drill down: workspaces → call imports or product areas → evaluations / models. +

+
+
+
+
+ {(['USD', 'INR'] as const).map((currency) => ( + + ))} +
+ {showCostBreakdown ? ( + + ) : null} + {isAdmin && licenseLoaded && usagePolicy.extended_history ? ( + + Pricing overrides + + ) : null} +
+ {summary?.last_updated_at ? ( +

+ Updated {new Date(summary.last_updated_at).toLocaleString()} +

+ ) : null} +
+
+ + {summaryError ? ( +
+ Could not load usage data + {summaryQueryError instanceof Error && summaryQueryError.message + ? `: ${summaryQueryError.message}` + : '.'}{' '} + Try refreshing the page. +
+ ) : null} + + {showOssUsageNotice ? ( +
+ Showing the last {maxHistoryDays} days of usage history. Set{' '} + + EFFICIENTAI_LICENSE + {' '} + on your server with any enterprise feature to unlock extended history. +
+ ) : null} + +
+
+ + + + +
+ +
+
+ {showAudio ? ( + + ) : null} + {showTts ? ( + + ) : null} + {(totals?.cache_read_tokens || 0) > 0 ? ( + + ) : null} + {(totals?.cache_creation_tokens || 0) > 0 ? ( + + ) : null} + {(totals?.reasoning_tokens || 0) > 0 ? ( + + ) : null} +
+
+ +
+
+
+ + setParams({ start: s, end: e })} + onWorkspaceChange={handleWorkspaceChange} + onCallImportChange={handleCallImportChange} + onDatasetChange={(v) => + setParams({ + dataset: v || null, + call_import_id: null, + resource_id: null, + }) + } + onTagChange={(v) => + setParams({ + tag_id: v || null, + call_import_id: null, + resource_id: null, + }) + } + onEvaluationChange={(id) => { + if (id.startsWith(USAGE_SECTION_SOURCE_PREFIX)) { + const section = id.slice(USAGE_SECTION_SOURCE_PREFIX.length) + setParams({ + product_section: section || null, + resource_id: null, + call_import_id: null, + model: null, + usage_kind: null, + }) + return + } + const resource = filterOptions?.resources?.find( + (r) => idKey(r.id) === idKey(id), + ) + setParams({ + resource_id: id || null, + product_section: id ? resource?.product_section || null : null, + model: null, + usage_kind: null, + }) + }} + onUsageKindChange={(k) => setParams({ usage_kind: k || null })} + onModelChange={(v) => setParams({ model: v || null })} + onClearAll={handleClearAll} + maxHistoryDays={maxHistoryDays} + /> + + +
+ + {showTruncation ? ( +

+ Showing the first 100 rows for this level. Narrow the date range or drill + further for complete detail. +

+ ) : null} +
+ + {tableLoading || breakdownStale ? ( +
+ +
+ ) : rows.length === 0 ? ( +
+ No usage in this period for the current scope. Try 7d or 30d, or go back up a + level. +
+ ) : ( +
+ {tableFetching ? ( +
+ +
+ ) : null} + + + + + + + + + + + + + + + + {rows.map((row, idx) => { + const drillable = isRowDrillable(row) + const compositeHeadline = + 'rowKind' in row ? compositeRowHeadline(row) : null + const rowTitle = tableRowLabel(groupBy, row, filterOptions) + const showRowTitle = + 'rowKind' in row + ? row.rowKind === 'workspace_resource' || + (compositeHeadline && + rowTitle.toLowerCase() !== compositeHeadline.toLowerCase()) + : true + return ( + drillable && handleRowDrill(row)} + > + + + + + + + + + + + ) + })} + +
+ {drillColumnLabel(groupBy, showWorkspaceComposite)} + LLM callsInput tokensOutput tokensTotal tokensEst. costSTT audioTTS charsCache read
+ + + {'rowKind' in row ? ( + + {compositeHeadline} + + ) : null} + {showRowTitle ? ( + + {rowTitle} + + ) : !('rowKind' in row) ? ( + {rowTitle} + ) : null} + {'hint' in row && row.hint ? ( + + {row.hint} + + ) : null} + + {drillable ? ( + + ) : null} + + + {formatNumber(row.call_count)} + + {formatNumber(row.prompt_tokens)} + + {formatNumber(row.completion_tokens)} + + {formatNumber(row.total_tokens)} + + {formatCostUsd(rowCostUsd(row))} + + {row.audio_seconds ? formatAudio(row.audio_seconds) : '—'} + + {row.tts_characters ? formatNumber(row.tts_characters) : '—'} + + {formatNumber(row.cache_read_tokens)} +
+
+ )} + + + + setCostBreakdownOpen(false)} + costs={totals?.costs} + scopeLabel={scopeSubtitle} + formatCost={formatCostUsd} + /> +
+ ) +} + +const statCardClass = 'border border-gray-200 ring-1 ring-[#fde047]/25 shadow-sm' + +function StatCard({ + label, + value, + valueLabel, + loading, + emphasize = false, + className = '', +}: { + label: string + value?: number + valueLabel?: string + loading?: boolean + emphasize?: boolean + className?: string +}) { + return ( + + +

{label}

+

+ {loading ? '—' : valueLabel ?? formatNumber(value || 0)} +

+ + + ) +} diff --git a/frontend/src/pages/usage/UsageDateRangePicker.tsx b/frontend/src/pages/usage/UsageDateRangePicker.tsx index cd5718ab..4c6ca19f 100644 --- a/frontend/src/pages/usage/UsageDateRangePicker.tsx +++ b/frontend/src/pages/usage/UsageDateRangePicker.tsx @@ -1,274 +1,273 @@ -import { useEffect, useMemo, useRef, useState } from 'react' -import { Calendar } from 'lucide-react' -import { usageTheme } from './usageTheme' -import { getUsageTimezone, formatUsageTimezoneLabel } from './usageTimezone' - -type Mode = 'relative' | 'absolute' - -type UsageDateRangePickerProps = { - start: string - end: string - onApply: (start: string, end: string) => void - maxHistoryDays?: number | null -} - -function toDateInput(d: Date): string { - const y = d.getFullYear() - const m = String(d.getMonth() + 1).padStart(2, '0') - const day = String(d.getDate()).padStart(2, '0') - return `${y}-${m}-${day}` -} - -function formatDisplay(start: string, end: string): string { - if (start === end) return start - return `${start} → ${end}` -} - -export function rangeForDays(days: number): { start: string; end: string } { - const end = new Date() - const start = new Date() - start.setDate(start.getDate() - (days - 1)) - return { start: toDateInput(start), end: toDateInput(end) } -} - -export function isRangeWithinMaxDays(start: string, end: string, maxDays: number): boolean { - const r = rangeForDays(maxDays) - return start >= r.start && end <= r.end -} - -const QUICK_RANGES = [ - { label: '1d', days: 1 }, - { label: '7d', days: 7 }, - { label: '30d', days: 30 }, - { label: '90d', days: 90 }, -] as const - -const RELATIVE_DAYS = [1, 2, 3, 4, 5, 6] -const RELATIVE_WEEKS = [1, 2, 3, 4] - -export default function UsageDateRangePicker({ - start, - end, - onApply, - maxHistoryDays = null, -}: UsageDateRangePickerProps) { - const [open, setOpen] = useState(false) - const [mode, setMode] = useState('relative') - const [draftStart, setDraftStart] = useState(start) - const [draftEnd, setDraftEnd] = useState(end) - const [relDays, setRelDays] = useState(1) - const rootRef = useRef(null) - - const quickRanges = useMemo(() => { - if (!maxHistoryDays) return QUICK_RANGES - return QUICK_RANGES.filter((q) => q.days <= maxHistoryDays) - }, [maxHistoryDays]) - - const minStartDate = useMemo(() => { - if (!maxHistoryDays) return null - return rangeForDays(maxHistoryDays).start - }, [maxHistoryDays]) - - const maxRelDays = maxHistoryDays ?? 365 - - const activeQuick = useMemo(() => { - for (const q of quickRanges) { - const r = rangeForDays(q.days) - if (r.start === start && r.end === end) return q.label - } - return null - }, [start, end, quickRanges]) - - useEffect(() => { - if (open) { - setDraftStart(start) - setDraftEnd(end) - } - }, [open, start, end]) - - useEffect(() => { - const onDoc = (e: MouseEvent) => { - if (!rootRef.current?.contains(e.target as Node)) setOpen(false) - } - document.addEventListener('mousedown', onDoc) - return () => document.removeEventListener('mousedown', onDoc) - }, []) - - const applyRelativeDays = (days: number) => { - const r = rangeForDays(days) - onApply(r.start, r.end) - setOpen(false) - } - - const applyAbsolute = () => { - if (!draftStart || !draftEnd) return - if (draftEnd < draftStart) return - onApply(draftStart, draftEnd) - setOpen(false) - } - - const pillClass = (active: boolean) => - `rounded-md px-2.5 py-1 text-xs font-medium transition-colors border ${ - active ? usageTheme.pillActive : usageTheme.pillInactive - }` - - return ( -
-
- {quickRanges.map((q) => ( - - ))} - -
- - - {formatDisplay(start, end)} - - {' '} - · {formatUsageTimezoneLabel(getUsageTimezone())} - - - - {open ? ( -
-
- - -
- - {mode === 'relative' ? ( -
-

- Dates use your local timezone ({formatUsageTimezoneLabel(getUsageTimezone())}). - Usage is stored by UTC day; filters include activity that happened on each - selected local day. -

-
-

Days

-
- {RELATIVE_DAYS.map((d) => ( - - ))} -
-
-
-

Weeks

-
- {RELATIVE_WEEKS.map((w) => ( - - ))} -
-
-
- - -
-
- ) : ( -
-
- - -
-
- - -
-
- )} -
- ) : null} -
- ) -} - -export function defaultUsageDateRange(): { start: string; end: string } { - const today = toDateInput(new Date()) - return { start: today, end: today } -} +import { useEffect, useMemo, useRef, useState } from 'react' +import { Calendar } from 'lucide-react' +import { usageTheme } from './usageTheme' +import { getUsageTimezone, formatUsageTimezoneLabel } from './usageTimezone' + +type Mode = 'relative' | 'absolute' + +type UsageDateRangePickerProps = { + start: string + end: string + onApply: (start: string, end: string) => void + maxHistoryDays?: number | null +} + +function toDateInput(d: Date): string { + const y = d.getFullYear() + const m = String(d.getMonth() + 1).padStart(2, '0') + const day = String(d.getDate()).padStart(2, '0') + return `${y}-${m}-${day}` +} + +function formatDisplay(start: string, end: string): string { + if (start === end) return start + return `${start} → ${end}` +} + +export function rangeForDays(days: number): { start: string; end: string } { + const end = new Date() + const start = new Date() + start.setDate(start.getDate() - (days - 1)) + return { start: toDateInput(start), end: toDateInput(end) } +} + +export function isRangeWithinMaxDays(start: string, end: string, maxDays: number): boolean { + const r = rangeForDays(maxDays) + return start >= r.start && end <= r.end +} + +const QUICK_RANGES = [ + { label: '1d', days: 1 }, + { label: '7d', days: 7 }, + { label: '30d', days: 30 }, + { label: '90d', days: 90 }, +] as const + +const RELATIVE_DAYS = [1, 2, 3, 4, 5, 6] +const RELATIVE_WEEKS = [1, 2, 3, 4] + +export default function UsageDateRangePicker({ + start, + end, + onApply, + maxHistoryDays = null, +}: UsageDateRangePickerProps) { + const [open, setOpen] = useState(false) + const [mode, setMode] = useState('relative') + const [draftStart, setDraftStart] = useState(start) + const [draftEnd, setDraftEnd] = useState(end) + const [relDays, setRelDays] = useState(1) + const rootRef = useRef(null) + + const quickRanges = useMemo(() => { + if (!maxHistoryDays) return QUICK_RANGES + return QUICK_RANGES.filter((q) => q.days <= maxHistoryDays) + }, [maxHistoryDays]) + + const minStartDate = useMemo(() => { + if (!maxHistoryDays) return null + return rangeForDays(maxHistoryDays).start + }, [maxHistoryDays]) + + const maxRelDays = maxHistoryDays ?? 365 + + const activeQuick = useMemo(() => { + for (const q of quickRanges) { + const r = rangeForDays(q.days) + if (r.start === start && r.end === end) return q.label + } + return null + }, [start, end, quickRanges]) + + useEffect(() => { + if (open) { + setDraftStart(start) + setDraftEnd(end) + } + }, [open, start, end]) + + useEffect(() => { + const onDoc = (e: MouseEvent) => { + if (!rootRef.current?.contains(e.target as Node)) setOpen(false) + } + document.addEventListener('mousedown', onDoc) + return () => document.removeEventListener('mousedown', onDoc) + }, []) + + const applyRelativeDays = (days: number) => { + const r = rangeForDays(days) + onApply(r.start, r.end) + setOpen(false) + } + + const applyAbsolute = () => { + if (!draftStart || !draftEnd) return + if (draftEnd < draftStart) return + onApply(draftStart, draftEnd) + setOpen(false) + } + + const pillClass = (active: boolean) => + `rounded-md px-2.5 py-1 text-xs font-medium transition-colors border ${ + active ? usageTheme.pillActive : usageTheme.pillInactive + }` + + return ( +
+
+ {quickRanges.map((q) => ( + + ))} + +
+ + + {formatDisplay(start, end)} + + {' '} + · {formatUsageTimezoneLabel(getUsageTimezone())} + + + + {open ? ( +
+
+ + +
+ + {mode === 'relative' ? ( +
+

+ Dates use your local timezone ({formatUsageTimezoneLabel(getUsageTimezone())}). + Usage is stored by UTC day; filters include activity that happened on each + selected local day. +

+
+

Days

+
+ {RELATIVE_DAYS.map((d) => ( + + ))} +
+
+
+

Weeks

+
+ {RELATIVE_WEEKS.map((w) => ( + + ))} +
+
+
+ + +
+
+ ) : ( +
+
+ + +
+
+ + +
+
+ )} +
+ ) : null} +
+ ) +} + +export function defaultUsageDateRange(): { start: string; end: string } { + return rangeForDays(7) +} diff --git a/tests/test_api/test_org_usage_routes.py b/tests/test_api/test_org_usage_routes.py new file mode 100644 index 00000000..f83e0af5 --- /dev/null +++ b/tests/test_api/test_org_usage_routes.py @@ -0,0 +1,12 @@ +"""Ensure org usage API routes are mounted on the v1 router.""" + +from app.api.v1.api import api_router + + +def test_org_usage_routes_are_registered(): + paths = {route.path for route in api_router.routes if hasattr(route, "path")} + assert "/organizations/usage/summary" in paths + assert "/organizations/usage/breakdown" in paths + assert "/organizations/usage/filters" in paths + assert "/organizations/usage/fx-rate" in paths + assert "/organizations/usage/pricing/overrides" in paths diff --git a/tests/test_services/test_usage/test_call_import_context.py b/tests/test_services/test_usage/test_call_import_context.py index 6d798f65..0466be72 100644 --- a/tests/test_services/test_usage/test_call_import_context.py +++ b/tests/test_services/test_usage/test_call_import_context.py @@ -9,33 +9,26 @@ call_import_row_usage_context, enrich_usage_context_workspace, ) +from app.services.usage.llm_usage import _bucket_prefix -def test_evaluation_row_gets_distinct_bucket_context(): +def test_evaluation_rows_share_bucket_context(): org_id = uuid4() ws_id = uuid4() eval_id = uuid4() import_id = uuid4() - row_a = uuid4() - row_b = uuid4() - source_a = uuid4() - source_b = uuid4() ctx_a = call_import_evaluation_usage_context( organization_id=org_id, workspace_id=ws_id, evaluation_id=eval_id, call_import_id=import_id, - evaluation_row_id=row_a, - call_import_row_id=source_a, ) ctx_b = call_import_evaluation_usage_context( organization_id=org_id, workspace_id=ws_id, evaluation_id=eval_id, call_import_id=import_id, - evaluation_row_id=row_b, - call_import_row_id=source_b, ) bucket_a = build_bucket_context( @@ -48,45 +41,75 @@ def test_evaluation_row_gets_distinct_bucket_context(): resource_type=ctx_b.resource_type, extra=ctx_b.extra, ) - assert bucket_a != bucket_b - assert bucket_a["evaluation_row_id"] == str(row_a) + assert bucket_a == bucket_b + assert bucket_a["evaluation_id"] == str(eval_id) assert bucket_a["call_import_id"] == str(import_id) + assert "evaluation_row_id" not in bucket_a + assert "call_import_row_id" not in bucket_a -def test_row_only_transcribe_context(): +def test_evaluation_rows_share_redis_bucket_prefix(): + from datetime import date + + org_id = uuid4() + ws_id = uuid4() + eval_id = uuid4() + import_id = uuid4() + model = "gpt-test" + usage_date = date(2026, 8, 15) + + def prefix_for(_row_id: object) -> str: + ctx = call_import_evaluation_usage_context( + organization_id=org_id, + workspace_id=ws_id, + evaluation_id=eval_id, + call_import_id=import_id, + ) + context = build_bucket_context( + resource_id=ctx.resource_id, + resource_type=ctx.resource_type, + extra=ctx.extra, + ) + return _bucket_prefix( + workspace_id=ws_id, + product_section=ctx.product_section.value, + model=model, + context=context, + usage_date=usage_date, + usage_kind="llm", + ) + + assert prefix_for(uuid4()) == prefix_for(uuid4()) + + +def test_row_only_transcribe_context_rolls_up_to_call_import(): org_id = uuid4() import_id = uuid4() - row_id = uuid4() ctx = call_import_row_usage_context( organization_id=org_id, workspace_id=None, call_import_id=import_id, - call_import_row_id=row_id, ) - assert ctx.extra is not None - assert ctx.extra["call_import_row_id"] == str(row_id) + assert ctx.extra == {"call_import_id": str(import_id)} assert ctx.resource_type == "call_import" + assert "call_import_row_id" not in (ctx.extra or {}) def test_call_import_ids_from_evaluation_context(): org_id = uuid4() eval_id = uuid4() import_id = uuid4() - row_id = uuid4() - source_id = uuid4() ctx = call_import_evaluation_usage_context( organization_id=org_id, workspace_id=uuid4(), evaluation_id=eval_id, call_import_id=import_id, - evaluation_row_id=row_id, - call_import_row_id=source_id, ) ids = call_import_ids_from_usage_context(ctx) assert ids["call_import_id"] == import_id assert ids["evaluation_id"] == eval_id - assert ids["evaluation_row_id"] == row_id - assert ids["call_import_row_id"] == source_id + assert ids["evaluation_row_id"] is None + assert ids["call_import_row_id"] is None def test_enrich_usage_context_workspace_noop_when_set(): diff --git a/tests/test_services/test_usage/test_usage_read_cache.py b/tests/test_services/test_usage/test_usage_read_cache.py index dc7884fd..20d1bd61 100644 --- a/tests/test_services/test_usage/test_usage_read_cache.py +++ b/tests/test_services/test_usage/test_usage_read_cache.py @@ -1,69 +1,91 @@ -"""Tests for usage read cache.""" - -from __future__ import annotations - -from datetime import date -from uuid import uuid4 - -import pytest - -from app.core.usage_entitlement import UsagePolicySnapshot -from app.services.usage import read_cache as cache_mod -from app.services.usage.access import UsageAccessResult -from app.services.usage.read_cache import ( - cache_key_for, - get_cached_response, - invalidate_org_usage_read_cache, - set_cached_response, -) -from tests.test_services.test_usage.test_llm_usage import _FakeRedis - - -@pytest.fixture -def fake_redis(monkeypatch): - client = _FakeRedis() - cache_mod._redis = client - monkeypatch.setattr(cache_mod.redis, "from_url", lambda *_args, **_kwargs: client) - yield client - cache_mod._redis = None - - -def _access() -> UsageAccessResult: - policy = UsagePolicySnapshot(extended_history=False, max_history_days=7) - return UsageAccessResult( - display_start=date(2026, 8, 1), - display_end=date(2026, 8, 7), - filter_start=date(2026, 8, 1), - filter_end=date(2026, 8, 8), - enforced_filter_floor=date(2026, 8, 1), - policy=policy, - range_clamped=False, - ) - - -def test_usage_read_cache_round_trip(fake_redis, monkeypatch): - org_id = uuid4() - access = _access() - key = cache_key_for(access, workspace_id=None) - payload = {"start": "2026-08-01", "end": "2026-08-07", "totals": {}} - - set_cached_response(org_id, "summary", key, payload) - cached = get_cached_response(org_id, "summary", key) - - assert cached == payload - - -def test_usage_read_cache_invalidate_org(fake_redis, monkeypatch): - org_id = uuid4() - other_org = uuid4() - access = _access() - key = cache_key_for(access) - - set_cached_response(org_id, "summary", key, {"x": 1}) - set_cached_response(other_org, "summary", key, {"x": 2}) - - deleted = invalidate_org_usage_read_cache(org_id) - - assert deleted >= 1 - assert get_cached_response(org_id, "summary", key) is None - assert get_cached_response(other_org, "summary", key) is not None +"""Tests for usage read cache.""" + +from __future__ import annotations + +from datetime import date +from uuid import uuid4 + +import pytest + +from app.core.usage_entitlement import UsagePolicySnapshot +from app.services.usage import read_cache as cache_mod +from app.services.usage.access import UsageAccessResult +from app.services.usage.read_cache import ( + cache_key_for, + get_cached_response, + invalidate_org_usage_read_cache, + set_cached_response, +) +from tests.test_services.test_usage.test_llm_usage import _FakeRedis + + +@pytest.fixture +def fake_redis(monkeypatch): + client = _FakeRedis() + cache_mod._redis = client + monkeypatch.setattr(cache_mod.redis, "from_url", lambda *_args, **_kwargs: client) + yield client + cache_mod._redis = None + + +def _access() -> UsageAccessResult: + policy = UsagePolicySnapshot(extended_history=False, max_history_days=7) + return UsageAccessResult( + display_start=date(2026, 8, 1), + display_end=date(2026, 8, 7), + filter_start=date(2026, 8, 1), + filter_end=date(2026, 8, 8), + enforced_filter_floor=date(2026, 8, 1), + policy=policy, + range_clamped=False, + ) + + +def test_usage_read_cache_skips_empty_summary(fake_redis, monkeypatch): + org_id = uuid4() + access = _access() + key = cache_key_for(access) + empty = { + "start": "2026-08-15", + "end": "2026-08-15", + "totals": {"prompt_tokens": 0, "completion_tokens": 0, "call_count": 0}, + } + set_cached_response(org_id, "summary", key, empty) + assert get_cached_response(org_id, "summary", key) is None + + +def test_usage_read_cache_round_trip(fake_redis, monkeypatch): + org_id = uuid4() + access = _access() + key = cache_key_for(access, workspace_id=None) + payload = { + "start": "2026-08-01", + "end": "2026-08-07", + "totals": {"prompt_tokens": 10, "completion_tokens": 5, "call_count": 1}, + } + + set_cached_response(org_id, "summary", key, payload) + cached = get_cached_response(org_id, "summary", key) + + assert cached == payload + + +def test_usage_read_cache_invalidate_org(fake_redis, monkeypatch): + org_id = uuid4() + other_org = uuid4() + access = _access() + key = cache_key_for(access) + payload = { + "start": "2026-08-01", + "end": "2026-08-07", + "totals": {"prompt_tokens": 10, "completion_tokens": 5, "call_count": 1}, + } + + set_cached_response(org_id, "summary", key, payload) + set_cached_response(other_org, "summary", key, payload) + + deleted = invalidate_org_usage_read_cache(org_id) + + assert deleted >= 1 + assert get_cached_response(org_id, "summary", key) is None + assert get_cached_response(other_org, "summary", key) is not None From a48dc64f1fa7e3e62a66fdc2becb8b073760eecf Mon Sep 17 00:00:00 2001 From: M Sami Date: Sat, 15 Aug 2026 16:16:23 +0530 Subject: [PATCH 25/32] feat: add enabled models support for AI providers and new endpoint for available models --- app/api/v1/routes/aiproviders.py | 1 + app/api/v1/routes/usage_pricing.py | 16 + .../076_ai_provider_enabled_models.py | 38 ++ app/models/database.py | 2 + app/models/schemas.py | 39 ++ app/services/usage/enabled_models.py | 130 +++++ app/services/usage/fx_rates.py | 77 ++- app/services/usage/pricing_overrides.py | 24 +- frontend/src/App.tsx | 32 +- frontend/src/lib/api.ts | 31 ++ frontend/src/lib/llmModelOptions.ts | 9 + frontend/src/lib/usageCurrency.ts | 30 +- .../AIProviderEnabledModelsStep.tsx | 213 ++++++++ .../src/pages/configurations/Integrations.tsx | 128 ++++- .../pages/usage/ProviderCredentialSelect.tsx | 172 +++++++ frontend/src/pages/usage/SearchableSelect.tsx | 144 ++++-- frontend/src/pages/usage/Usage.tsx | 112 ++--- .../pages/usage/UsageCostBreakdownModal.tsx | 11 +- frontend/src/pages/usage/UsagePage.tsx | 72 +++ frontend/src/pages/usage/UsagePricing.tsx | 462 +++++++++++++----- .../src/pages/usage/pricingModelOptions.ts | 72 +++ .../src/pages/usage/useAnchoredDropdown.ts | 37 ++ frontend/src/types/api.ts | 3 + .../test_usage/test_enabled_models.py | 66 +++ .../test_services/test_usage/test_fx_rates.py | 69 +++ 25 files changed, 1667 insertions(+), 323 deletions(-) create mode 100644 app/migrations/076_ai_provider_enabled_models.py create mode 100644 app/services/usage/enabled_models.py create mode 100644 frontend/src/pages/configurations/AIProviderEnabledModelsStep.tsx create mode 100644 frontend/src/pages/usage/ProviderCredentialSelect.tsx create mode 100644 frontend/src/pages/usage/UsagePage.tsx create mode 100644 frontend/src/pages/usage/pricingModelOptions.ts create mode 100644 frontend/src/pages/usage/useAnchoredDropdown.ts create mode 100644 tests/test_services/test_usage/test_enabled_models.py create mode 100644 tests/test_services/test_usage/test_fx_rates.py diff --git a/app/api/v1/routes/aiproviders.py b/app/api/v1/routes/aiproviders.py index b7f8b9e6..7f8f8e01 100644 --- a/app/api/v1/routes/aiproviders.py +++ b/app/api/v1/routes/aiproviders.py @@ -198,6 +198,7 @@ async def create_aiprovider( else None ), gateway_extra_headers=aiprovider.gateway_extra_headers, + enabled_models=aiprovider.enabled_models, ) db.add(db_aiprovider) db.flush() diff --git a/app/api/v1/routes/usage_pricing.py b/app/api/v1/routes/usage_pricing.py index 0716eef6..c48deaeb 100644 --- a/app/api/v1/routes/usage_pricing.py +++ b/app/api/v1/routes/usage_pricing.py @@ -112,6 +112,22 @@ class UsageRecomputeJobResponse(BaseModel): completed_at: Optional[datetime] = None +class AvailableModelsResponse(BaseModel): + models: List[str] + + +@router.get("/available-models", response_model=AvailableModelsResponse) +def list_pricing_available_models( + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db), +): + from app.services.usage.enabled_models import org_pricing_eligible_models + + return AvailableModelsResponse( + models=org_pricing_eligible_models(db, organization_id), + ) + + @router.get("", response_model=List[EffectivePricingResponse]) def list_effective_usage_pricing( usage_kind: Optional[str] = Query(None), diff --git a/app/migrations/076_ai_provider_enabled_models.py b/app/migrations/076_ai_provider_enabled_models.py new file mode 100644 index 00000000..c1714f28 --- /dev/null +++ b/app/migrations/076_ai_provider_enabled_models.py @@ -0,0 +1,38 @@ +"""Migration: per-credential enabled model allowlist for integrations.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add enabled_models JSONB to aiproviders" + + +def _column_exists(db: Session, table: str, column: str) -> bool: + return ( + db.execute( + text( + """ + SELECT 1 FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + """ + ), + {"table_name": table, "column_name": column}, + ).first() + is not None + ) + + +def upgrade(db: Session) -> None: + if not _column_exists(db, "aiproviders", "enabled_models"): + db.execute( + text( + """ + ALTER TABLE aiproviders + ADD COLUMN enabled_models JSONB NULL + """ + ) + ) + + +def downgrade(db: Session) -> None: + if _column_exists(db, "aiproviders", "enabled_models"): + db.execute(text("ALTER TABLE aiproviders DROP COLUMN enabled_models")) diff --git a/app/models/database.py b/app/models/database.py index 3e886445..92350139 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -702,6 +702,8 @@ class AIProvider(Base): gateway_auth_secret = Column(String, nullable=True) # Arbitrary HTTP headers sent with gateway-routed LiteLLM calls gateway_extra_headers = Column(JSON, nullable=True) + # Non-empty list restricts model pickers; null/empty = all catalog models for provider. + enabled_models = Column(JSON, nullable=True) created_at = Column(DateTime(timezone=True), server_default=func.now()) updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated diff --git a/app/models/schemas.py b/app/models/schemas.py index 1e6f0e7f..25241e77 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -970,6 +970,13 @@ class AIProviderCreate(BaseModel): None, description="Arbitrary HTTP headers sent with gateway-routed LiteLLM calls.", ) + enabled_models: Optional[List[str]] = Field( + None, + description=( + "Allowlisted model names for this credential. " + "Null or empty means all catalog models for the provider." + ), + ) is_default: Optional[bool] = Field( None, description=( @@ -1053,6 +1060,21 @@ def validate_gateway_auth_secret(cls, v: Optional[str]) -> Optional[str]: def validate_gateway_extra_headers(cls, v: Optional[Dict[str, Any]]) -> Optional[Dict[str, str]]: return _validate_gateway_extra_headers(v) + @field_validator("enabled_models") + @classmethod + def validate_enabled_models_create(cls, v: Optional[List[str]]) -> Optional[List[str]]: + if v is None: + return None + seen: set[str] = set() + out: list[str] = [] + for item in v: + name = str(item).strip() + if not name or name in seen: + continue + seen.add(name) + out.append(name) + return out or None + class AIProviderUpdate(BaseModel): """Schema for updating an AI Provider.""" @@ -1069,6 +1091,7 @@ class AIProviderUpdate(BaseModel): gateway_auth_secret: Optional[str] = None clear_gateway_auth_secret: bool = False gateway_extra_headers: Optional[Dict[str, str]] = None + enabled_models: Optional[List[str]] = None @field_validator("gateway_model") @classmethod @@ -1127,6 +1150,21 @@ def validate_gateway_extra_headers_update( ) -> Optional[Dict[str, str]]: return _validate_gateway_extra_headers(v) + @field_validator("enabled_models") + @classmethod + def validate_enabled_models_update(cls, v: Optional[List[str]]) -> Optional[List[str]]: + if v is None: + return None + seen: set[str] = set() + out: list[str] = [] + for item in v: + name = str(item).strip() + if not name or name in seen: + continue + seen.add(name) + out.append(name) + return out or None + class AIProviderResponse(BaseModel): """Schema for AI Provider response.""" @@ -1145,6 +1183,7 @@ class AIProviderResponse(BaseModel): gateway_auth_secret_env: Optional[str] = None has_gateway_auth_secret: bool = False gateway_extra_headers: Optional[Dict[str, str]] = None + enabled_models: Optional[List[str]] = None gateway_managed: bool = False effective_routing: Literal["inherit", "direct", "gateway", "bifrost", "litellm_proxy"] = "inherit" effective_gateway_interface: Literal["litellm_shim", "native_openai"] = "litellm_shim" diff --git a/app/services/usage/enabled_models.py b/app/services/usage/enabled_models.py new file mode 100644 index 00000000..87d9652e --- /dev/null +++ b/app/services/usage/enabled_models.py @@ -0,0 +1,130 @@ +"""Org- and credential-level enabled model allowlists.""" + +from __future__ import annotations + +from typing import Iterable, List, Optional, Set +from uuid import UUID + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.models.database import AIProvider, ModelProvider +from app.services.ai.model_config_service import ModelConfigService + + +def normalize_enabled_models(raw: Optional[Iterable[str]]) -> Optional[List[str]]: + if raw is None: + return None + seen: set[str] = set() + normalized: list[str] = [] + for item in raw: + if item is None: + continue + name = str(item).strip() + if not name or name in seen: + continue + seen.add(name) + normalized.append(name) + return normalized or None + + +def catalog_models_for_provider(provider: str) -> List[str]: + service = ModelConfigService() + try: + provider_enum = ModelProvider(provider.lower()) + except ValueError: + return [] + options = service.get_model_options_by_provider(provider_enum) + models: list[str] = [] + for key in ("llm", "stt", "tts", "s2s"): + models.extend(options.get(key) or []) + return sorted({m for m in models if m}) + + +def effective_enabled_models_for_credential(credential: AIProvider) -> Optional[List[str]]: + """Return explicit allowlist, or None meaning unrestricted (full provider catalog).""" + return normalize_enabled_models(credential.enabled_models) + + +def filter_models_by_credential( + credential: Optional[AIProvider], + catalog_models: List[str], +) -> List[str]: + if credential is None: + return catalog_models + allowlist = effective_enabled_models_for_credential(credential) + if not allowlist: + return catalog_models + allowed = set(allowlist) + filtered = [m for m in catalog_models if m in allowed] + gateway = (credential.gateway_model or "").strip() + if gateway and gateway not in filtered: + filtered = [gateway, *filtered] + return filtered + + +def _usage_models_for_org(db: Session, organization_id: UUID) -> Set[str]: + rows = db.execute( + text( + """ + SELECT DISTINCT model + FROM llm_usage_daily + WHERE organization_id = CAST(:organization_id AS uuid) + AND model IS NOT NULL + AND model <> '' + """ + ), + {"organization_id": str(organization_id)}, + ).scalars().all() + return {str(row).strip() for row in rows if row} + + +def _override_models_for_org(db: Session, organization_id: UUID) -> Set[str]: + rows = db.execute( + text( + """ + SELECT DISTINCT model + FROM org_model_pricing_overrides + WHERE organization_id = CAST(:organization_id AS uuid) + """ + ), + {"organization_id": str(organization_id)}, + ).scalars().all() + return {str(row).strip() for row in rows if row} + + +def org_pricing_eligible_models(db: Session, organization_id: UUID) -> List[str]: + """Models org admins may set pricing overrides for.""" + models: set[str] = set() + providers = ( + db.query(AIProvider) + .filter( + AIProvider.organization_id == organization_id, + AIProvider.is_active.is_(True), + ) + .all() + ) + any_explicit_allowlist = False + for provider in providers: + gateway = (provider.gateway_model or "").strip() + if gateway: + models.add(gateway) + allowlist = effective_enabled_models_for_credential(provider) + if allowlist: + any_explicit_allowlist = True + models.update(allowlist) + else: + models.update(catalog_models_for_provider(provider.provider)) + + if not any_explicit_allowlist and not models: + for provider in providers: + models.update(catalog_models_for_provider(provider.provider)) + + models.update(_usage_models_for_org(db, organization_id)) + models.update(_override_models_for_org(db, organization_id)) + return sorted(models) + + +def org_union_enabled_models(db: Session, organization_id: UUID) -> List[str]: + """All models enabled on any active integration credential.""" + return org_pricing_eligible_models(db, organization_id) diff --git a/app/services/usage/fx_rates.py b/app/services/usage/fx_rates.py index a99cc68b..927026b1 100644 --- a/app/services/usage/fx_rates.py +++ b/app/services/usage/fx_rates.py @@ -14,7 +14,8 @@ _REDIS_KEY = "usage:fx:USD_INR" _FRANKFURTER_URL = "https://api.frankfurter.dev/v2/rate/USD/INR" -_DEFAULT_RATE = 83.0 +_DEFAULT_RATE = 95.0 +_CACHE_TTL_SECONDS = 25 * 3600 _redis: redis.Redis | None = None @@ -41,40 +42,84 @@ def _read_cached() -> Optional[dict[str, Any]]: if not raw: return None payload = json.loads(raw) + if payload.get("source") != "frankfurter": + return None if not isinstance(payload.get("rate"), (int, float)): return None return payload - except (redis.RedisError, json.JSONDecodeError): + except (redis.RedisError, json.JSONDecodeError) as exc: + logger.warning("USD/INR FX cache read failed: {}", exc) return None +def _write_cached(payload: dict[str, Any]) -> None: + try: + _client().set(_REDIS_KEY, json.dumps(payload), ex=_CACHE_TTL_SECONDS) + except redis.RedisError as exc: + logger.warning("USD/INR FX cache write failed: {}", exc) + + +def _parse_frankfurter_payload(data: Any) -> tuple[float, datetime]: + if not isinstance(data, dict): + raise ValueError(f"Frankfurter response is not an object: {type(data).__name__}") + + base = data.get("base") + quote = data.get("quote") + if base != "USD" or quote != "INR": + raise ValueError(f"Unexpected Frankfurter pair: {base}/{quote}") + + rate_raw = data.get("rate") + if not isinstance(rate_raw, (int, float)): + raise ValueError(f"Frankfurter rate missing or invalid: {rate_raw!r}") + + rate = float(rate_raw) + if rate <= 0: + raise ValueError(f"Frankfurter rate must be positive: {rate}") + + date_raw = data.get("date") + if not isinstance(date_raw, str) or not date_raw.strip(): + raise ValueError(f"Frankfurter date missing or invalid: {date_raw!r}") + + as_of = datetime.fromisoformat(date_raw).replace(tzinfo=timezone.utc) + return rate, as_of + + +def _fallback_payload(reason: str, *, response_body: Any = None) -> dict[str, Any]: + logger.error( + "USD/INR FX using hardcoded fallback rate {:.2f} — INR costs may be wrong. reason={} response={}", + _DEFAULT_RATE, + reason, + response_body, + ) + return _cache_payload(_DEFAULT_RATE, datetime.now(timezone.utc), "default") + + def get_usd_inr_rate() -> dict[str, Any]: cached = _read_cached() if cached is not None: return cached - return _cache_payload(_DEFAULT_RATE, datetime.now(timezone.utc), "default") + return refresh_usd_inr_rate() def refresh_usd_inr_rate() -> dict[str, Any]: - rate = _DEFAULT_RATE - as_of = datetime.now(timezone.utc) - source = "default" try: with httpx.Client(timeout=15.0) as client: response = client.get(_FRANKFURTER_URL) response.raise_for_status() data = response.json() - fetched = float(data["rate"]) - if fetched > 0: - rate = fetched - source = "frankfurter" - as_of = datetime.fromisoformat(data["date"]).replace(tzinfo=timezone.utc) except Exception as exc: - logger.warning("USD/INR FX refresh failed, using fallback: {}", exc) + return _fallback_payload(f"Frankfurter request failed: {exc}") - payload = _cache_payload(rate, as_of, source) try: - _client().set(_REDIS_KEY, json.dumps(payload), ex=25 * 3600) - except redis.RedisError as exc: - logger.warning("USD/INR FX cache write failed: {}", exc) + rate, as_of = _parse_frankfurter_payload(data) + except Exception as exc: + return _fallback_payload(f"Frankfurter response parse failed: {exc}", response_body=data) + + payload = _cache_payload(rate, as_of, "frankfurter") + _write_cached(payload) + logger.info( + "USD/INR FX refreshed from Frankfurter: rate={} as_of={}", + rate, + as_of.date().isoformat(), + ) return payload diff --git a/app/services/usage/pricing_overrides.py b/app/services/usage/pricing_overrides.py index 7f55ef01..1214023b 100644 --- a/app/services/usage/pricing_overrides.py +++ b/app/services/usage/pricing_overrides.py @@ -54,9 +54,21 @@ def _known_models(db: Session) -> Set[str]: return models -def validate_model_name(db: Session, model: str) -> None: - if model not in _known_models(db): - raise HTTPException(status_code=400, detail=f"Unknown model: {model}") +def validate_model_name( + db: Session, + model: str, + *, + organization_id: Optional[UUID] = None, +) -> None: + from app.services.usage.enabled_models import org_pricing_eligible_models + + if organization_id is not None: + eligible = set(org_pricing_eligible_models(db, organization_id)) + if model in eligible: + return + if model in _known_models(db): + return + raise HTTPException(status_code=400, detail=f"Unknown model: {model}") def _validate_usage_kind(usage_kind: str) -> str: @@ -185,7 +197,7 @@ def get_effective_rate( usage_kind: str, as_of: date, ) -> Dict[str, Any]: - validate_model_name(db, model) + validate_model_name(db, model, organization_id=organization_id) kind = _validate_usage_kind(usage_kind) resolver = PricingResolver(db) effective = resolver.resolve_rate( @@ -301,7 +313,7 @@ def upsert_override( rates: Dict[str, Any], recompute: bool = True, ) -> Dict[str, Any]: - validate_model_name(db, model) + validate_model_name(db, model, organization_id=organization_id) kind = _validate_usage_kind(usage_kind) if effective_to is not None and effective_to < effective_from: raise HTTPException(status_code=400, detail="effective_to must be >= effective_from") @@ -406,7 +418,7 @@ def delete_override( effective_from: Optional[date] = None, recompute: bool = True, ) -> Dict[str, Any]: - validate_model_name(db, model) + validate_model_name(db, model, organization_id=organization_id) kind = _validate_usage_kind(usage_kind) filters = [ "organization_id = CAST(:organization_id AS uuid)", diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index bcf4ad73..72664080 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -14,8 +14,7 @@ import PlatformAdmin from './pages/platform/PlatformAdmin' import Dashboard from './pages/dashboard/Dashboard' // Usage -import Usage from './pages/usage/Usage' -import UsagePricing from './pages/usage/UsagePricing' +import UsagePage, { UsagePricingRedirect } from './pages/usage/UsagePage' // Prompt Partials import PromptPartials from './pages/promptPartials/PromptPartials' @@ -136,24 +135,6 @@ function EnterpriseGate({ feature, children }: { feature: string; children: Reac return <>{children} } -function EnterpriseLicenseGate({ children }: { children: React.ReactNode }) { - const { hasExtendedUsageHistory, isLoaded } = useLicenseStore() - - if (!isLoaded) { - return ( -
-
-
- ) - } - - if (!hasExtendedUsageHistory()) { - return - } - - return <>{children} -} - function App() { return ( @@ -214,15 +195,8 @@ function App() { } /> } /> } /> - } /> - - - - } - /> + } /> + } /> } diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index ae19a698..aaca0b6c 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -3443,6 +3443,37 @@ class ApiClient { return response.data } + async listUsagePricingAvailableModels(): Promise<{ models: string[] }> { + const response = await this.client.get( + '/api/v1/organizations/usage/pricing/available-models', + ) + return response.data + } + + async getUsagePricingEffective( + model: string, + params?: { usage_kind?: string; as_of?: string }, + ): Promise<{ + model: string + usage_kind: string + as_of: string + catalog_rates?: Record | null + effective_rates?: Record | null + effective_source?: string | null + has_override: boolean + override?: { + effective_from: string + effective_to?: string | null + rates: Record + } | null + }> { + const response = await this.client.get( + `/api/v1/organizations/usage/pricing/overrides/${encodeURIComponent(model)}`, + { params }, + ) + return response.data + } + async listUsagePricingOverrides(params?: { model?: string usage_kind?: string diff --git a/frontend/src/lib/llmModelOptions.ts b/frontend/src/lib/llmModelOptions.ts index d5a556e2..2dfc3fc1 100644 --- a/frontend/src/lib/llmModelOptions.ts +++ b/frontend/src/lib/llmModelOptions.ts @@ -63,6 +63,15 @@ export function resolveLLMModelsForCredential( ) { return { mode: 'gateway_direct', model: gatewayModel } } + const allowlist = credential?.enabled_models?.filter((m) => m?.trim()) ?? [] + if (allowlist.length > 0) { + const allowed = new Set(allowlist) + const filtered = catalogModels.filter((m) => allowed.has(m)) + if (gatewayModel && !filtered.includes(gatewayModel)) { + return { mode: 'catalog', models: [gatewayModel, ...filtered] } + } + return { mode: 'catalog', models: filtered } + } return { mode: 'catalog', models: catalogModels } } diff --git a/frontend/src/lib/usageCurrency.ts b/frontend/src/lib/usageCurrency.ts index 2b2122c4..a110721d 100644 --- a/frontend/src/lib/usageCurrency.ts +++ b/frontend/src/lib/usageCurrency.ts @@ -13,6 +13,15 @@ export function setUsageDisplayCurrency(currency: UsageDisplayCurrency): void { window.localStorage.setItem(STORAGE_KEY, currency) } +function formatUsdAmount(amountUsd: number): string { + const abs = Math.abs(amountUsd) + const digits = abs >= 1 ? 2 : 4 + return `$${amountUsd.toLocaleString('en-US', { + minimumFractionDigits: digits, + maximumFractionDigits: digits, + })}` +} + export function formatUsageCostUsd( usd: number | null | undefined, currency: UsageDisplayCurrency, @@ -21,17 +30,24 @@ export function formatUsageCostUsd( const amountUsd = Number(usd || 0) if (!amountUsd) return '—' if (currency === 'INR') { - return new Intl.NumberFormat(undefined, { + return new Intl.NumberFormat('en-IN', { style: 'currency', currency: 'INR', minimumFractionDigits: 2, maximumFractionDigits: 2, }).format(amountUsd * inrRate) } - return new Intl.NumberFormat(undefined, { - style: 'currency', - currency: 'USD', - minimumFractionDigits: 2, - maximumFractionDigits: 4, - }).format(amountUsd) + return formatUsdAmount(amountUsd) +} + +export function formatFxRateHint( + inrRate: number, + asOf?: string | null, + source?: string | null, +): string { + if (source === 'default') { + return `1 USD ≈ ₹${inrRate.toFixed(2)} (estimate — live FX unavailable)` + } + const dateLabel = asOf ? asOf.slice(0, 10) : 'today' + return `1 USD = ₹${inrRate.toFixed(2)} (Frankfurter, ${dateLabel})` } diff --git a/frontend/src/pages/configurations/AIProviderEnabledModelsStep.tsx b/frontend/src/pages/configurations/AIProviderEnabledModelsStep.tsx new file mode 100644 index 00000000..45ba4b2e --- /dev/null +++ b/frontend/src/pages/configurations/AIProviderEnabledModelsStep.tsx @@ -0,0 +1,213 @@ +import { useMemo, useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import { Check } from 'lucide-react' +import { apiClient } from '../../lib/api' +import type { ModelProvider } from '../../types/api' +import { usageTheme } from '../usage/usageTheme' + +type ModelOptions = { + llm?: string[] + stt?: string[] + tts?: string[] + s2s?: string[] +} + +type SectionKey = 'llm' | 'stt' | 'tts' | 's2s' + +const SECTIONS: Array<{ key: SectionKey; label: string }> = [ + { key: 'llm', label: 'LLM' }, + { key: 'stt', label: 'STT' }, + { key: 'tts', label: 'TTS' }, + { key: 's2s', label: 'Speech-to-speech' }, +] + +type Props = { + provider: ModelProvider + enabledModels: string[] + onChange: (models: string[]) => void + gatewayModel?: string +} + +function ModelChip({ + model, + selected, + onToggle, +}: { + model: string + selected: boolean + onToggle: () => void +}) { + return ( + + ) +} + +export default function AIProviderEnabledModelsStep({ + provider, + enabledModels, + onChange, + gatewayModel, +}: Props) { + const [customModel, setCustomModel] = useState('') + + const { data: options, isLoading } = useQuery({ + queryKey: ['model-options', provider], + queryFn: () => apiClient.getModelOptions(provider), + enabled: Boolean(provider), + }) + + const catalog = (options || {}) as ModelOptions + const selected = useMemo(() => new Set(enabledModels), [enabledModels]) + + const toggle = (model: string) => { + const next = new Set(enabledModels) + if (next.has(model)) next.delete(model) + else next.add(model) + onChange(Array.from(next).sort()) + } + + const selectSection = (key: SectionKey) => { + const models = catalog[key] || [] + const next = new Set(enabledModels) + models.forEach((m) => next.add(m)) + onChange(Array.from(next).sort()) + } + + const clearSection = (key: SectionKey) => { + const models = new Set(catalog[key] || []) + onChange(enabledModels.filter((m) => !models.has(m))) + } + + const addCustomModel = () => { + const name = customModel.trim() + if (!name) return + if (!selected.has(name)) { + onChange([...enabledModels, name].sort()) + } + setCustomModel('') + } + + const gateway = gatewayModel?.trim() + + return ( +
+
+

+ Tap models to allow this integration to use them in evals, agents, and pricing. +

+

+ Leave all off to allow the full provider catalog. +

+
+ + {gateway ? ( +
+ Gateway model {gateway} stays available + when configured. +
+ ) : null} + + {isLoading ? ( +

Loading catalog models…

+ ) : ( + SECTIONS.map(({ key, label }) => { + const models = catalog[key] || [] + if (models.length === 0) return null + const sectionSelected = models.filter((m) => selected.has(m)).length + return ( +
+
+
+ {label} + + {sectionSelected}/{models.length} selected + +
+
+ + +
+
+
+ {models.map((model) => ( + toggle(model)} + /> + ))} +
+
+ ) + }) + )} + +
+ +
+ setCustomModel(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault() + addCustomModel() + } + }} + className={`flex-1 rounded-lg border border-gray-200 px-3 py-2 text-sm focus:outline-none ${usageTheme.focusRing}`} + placeholder="e.g. accounts/fireworks/models/gpt-oss-120b" + /> + +
+
+ +

+ {enabledModels.length > 0 + ? `${enabledModels.length} model${enabledModels.length === 1 ? '' : 's'} enabled` + : 'No restriction — full catalog allowed'} +

+
+ ) +} diff --git a/frontend/src/pages/configurations/Integrations.tsx b/frontend/src/pages/configurations/Integrations.tsx index 4dce71a2..4c2e9ca6 100644 --- a/frontend/src/pages/configurations/Integrations.tsx +++ b/frontend/src/pages/configurations/Integrations.tsx @@ -21,6 +21,7 @@ import { getTelephonyProviderLogo, } from '../../config/providers' import WalkthroughToggleButton from '../../components/walkthrough/WalkthroughToggleButton' +import AIProviderEnabledModelsStep from './AIProviderEnabledModelsStep' type IntegrationType = 'voice_platform' | 'ai_provider' | 'telephony_provider' | null @@ -65,6 +66,8 @@ export default function Integrations() { const [gatewayAuthSecret, setGatewayAuthSecret] = useState('') const [clearGatewayAuthSecret, setClearGatewayAuthSecret] = useState(false) const [gatewayExtraHeadersJson, setGatewayExtraHeadersJson] = useState('') + const [aiProviderWizardStep, setAiProviderWizardStep] = useState<1 | 2>(1) + const [enabledModels, setEnabledModels] = useState([]) const [showDeleteModal, setShowDeleteModal] = useState(false) const [showDeleteAIProviderModal, setShowDeleteAIProviderModal] = useState(false) const [showDeleteTelephonyModal, setShowDeleteTelephonyModal] = useState(false) @@ -363,6 +366,7 @@ export default function Integrations() { setCredentialRoutingMode('inherit'); setGatewayModel(''); setGatewayInterface('inherit'); setGatewayBaseUrl('') setGatewayAuthHeader(''); setGatewayAuthSecretEnv(''); setGatewayAuthSecret(''); setClearGatewayAuthSecret(false) setGatewayExtraHeadersJson('') + setAiProviderWizardStep(1); setEnabledModels([]) setSelectedTelephonyProvider(null); setTelephonyAuthId(''); setTelephonyAuthToken(''); setTelephonyVerifyAppUuid(''); setTelephonyVoiceAppId(''); setTelephonySipDomain('') setEditingTelephonyConfigId(null); setTelephonyName('') } @@ -387,6 +391,8 @@ export default function Integrations() { setGatewayBaseUrl(provider.gateway_base_url || ''); setGatewayAuthHeader(provider.gateway_auth_header || '') setGatewayAuthSecretEnv(provider.gateway_auth_secret_env || ''); setGatewayAuthSecret(''); setClearGatewayAuthSecret(false) setGatewayExtraHeadersJson(formatGatewayExtraHeadersJson(provider.gateway_extra_headers)) + setEnabledModels(provider.enabled_models || []) + setAiProviderWizardStep(1) setShowProviderDropdown(false); setIsEditMode(true); setShowModal(true) } @@ -434,8 +440,29 @@ export default function Integrations() { return } + const resolvedEnabledModels = enabledModels.length > 0 ? enabledModels : null + + if (aiProviderWizardStep === 1) { + if (!isEditMode && !selectedProvider) { + showToast('Please select a provider', 'error') + return + } + if (!isEditMode && aiProviderRequiresApiKey && !apiKey.trim()) { + showToast('Please enter an API key', 'error') + return + } + if (!isEditMode && selectedProvider === ModelProvider.AZURE && !azureEndpointUrl.trim()) { + showToast('Please enter your Azure OpenAI endpoint URL', 'error') + return + } + setAiProviderWizardStep(2) + return + } + if (isEditMode && selectedAIProvider) { - const updateData: Partial = {} + const updateData: Partial = { + enabled_models: resolvedEnabledModels, + } if (apiKey.trim()) updateData.api_key = apiKey if (name !== (selectedAIProvider.name || '')) updateData.name = name || null const trimmedAzureEndpointUrl = azureEndpointUrl.trim() @@ -475,7 +502,6 @@ export default function Integrations() { if (gatewayExtraHeadersJson.trim() !== existingExtraHeadersJson.trim()) { updateData.gateway_extra_headers = parsedGatewayExtraHeaders } - if (Object.keys(updateData).length === 0) { resetForm(); return } updateAIProviderMutation.mutate({ id: selectedAIProvider.id, data: updateData }) } else { if (!selectedProvider) { @@ -503,6 +529,7 @@ export default function Integrations() { gateway_auth_secret_env: gatewayAuthSecretEnv.trim() || undefined, gateway_auth_secret: gatewayAuthSecret.trim() || undefined, gateway_extra_headers: parsedGatewayExtraHeaders || undefined, + enabled_models: resolvedEnabledModels || undefined, }) } } else if (integrationType === 'telephony_provider') { @@ -801,6 +828,14 @@ export default function Integrations() { {provider.gateway_model} )} + {provider.enabled_models && provider.enabled_models.length > 0 && ( + + {provider.enabled_models.length} model{provider.enabled_models.length === 1 ? '' : 's'} + + )} {!provider.is_active && Inactive}
@@ -1119,9 +1154,25 @@ export default function Integrations() { {showModal && renderModal(
-
+
-

{isEditMode ? (integrationType === 'ai_provider' ? 'Edit AI Provider' : integrationType === 'telephony_provider' ? 'Edit Telephony Provider' : 'Edit Integration') : 'Add Integration'}

+
+

+ {isEditMode + ? integrationType === 'ai_provider' + ? 'Edit AI Provider' + : integrationType === 'telephony_provider' + ? 'Edit Telephony Provider' + : 'Edit Integration' + : 'Add Integration'} +

+ {integrationType === 'ai_provider' ? ( +

+ Step {aiProviderWizardStep} of 2 —{' '} + {aiProviderWizardStep === 1 ? 'Credentials' : 'Enabled models'} +

+ ) : null} +
@@ -1140,6 +1191,7 @@ export default function Integrations() { setIntegrationType('ai_provider') setSelectedPlatform(null) setSelectedProvider(null) + setAiProviderWizardStep(1) }} className={`p-3 border-2 rounded-lg text-left transition-all ${integrationType === 'ai_provider' ? 'border-primary-500 bg-primary-50' @@ -1237,7 +1289,7 @@ export default function Integrations() { )} - {integrationType === 'ai_provider' && ( + {integrationType === 'ai_provider' && aiProviderWizardStep === 1 && ( <>
@@ -1448,6 +1500,15 @@ export default function Integrations() { )} + {integrationType === 'ai_provider' && aiProviderWizardStep === 2 && (selectedProvider || selectedAIProvider) && ( + + )} + {integrationType === 'telephony_provider' && ( <>
@@ -1546,11 +1607,58 @@ export default function Integrations() {
- + {integrationType === 'ai_provider' && aiProviderWizardStep === 2 ? ( + <> + + + + ) : ( + + )}
diff --git a/frontend/src/pages/usage/ProviderCredentialSelect.tsx b/frontend/src/pages/usage/ProviderCredentialSelect.tsx new file mode 100644 index 00000000..23a6af2e --- /dev/null +++ b/frontend/src/pages/usage/ProviderCredentialSelect.tsx @@ -0,0 +1,172 @@ +import { useEffect, useRef, useState, type ReactNode } from 'react' +import { createPortal } from 'react-dom' +import { Brain, ChevronDown, Info } from 'lucide-react' +import { getProviderLabel, getProviderLogo } from '../../config/providers' +import type { AIProvider, ModelProvider } from '../../types/api' +import { usageTheme } from './usageTheme' +import { credentialDisplayLabel } from './pricingModelOptions' +import { useAnchoredDropdown } from './useAnchoredDropdown' +function FieldHint({ title, children }: { title: string; children: ReactNode }) { + return ( + + + + + {title} + + {children} + + + ) +} + +function ProviderLogo({ + provider, + size = 'sm', +}: { + provider: ModelProvider + size?: 'sm' | 'md' +}) { + const logo = getProviderLogo(provider) + const label = getProviderLabel(provider) + + if (logo) { + if (size === 'md') { + return ( + + {label} + + ) + } + return {label} + } + + if (size === 'md') { + return ( + + + + ) + } + + return +} + +type Props = { + label: string + hint?: string + value: string + credentials: AIProvider[] + onChange: (credentialId: string) => void + disabled?: boolean + placeholder?: string +} + +export default function ProviderCredentialSelect({ + label, + hint, + value, + credentials, + onChange, + disabled, + placeholder = 'Select integration…', +}: Props) { + const [open, setOpen] = useState(false) + const rootRef = useRef(null) + const anchorRef = useRef(null) + const panelRef = useRef(null) + const coords = useAnchoredDropdown(open && !disabled, anchorRef) + const selected = credentials.find((row) => row.id === value) + + useEffect(() => { + const onDoc = (e: MouseEvent) => { + const target = e.target as Node + if (rootRef.current?.contains(target) || panelRef.current?.contains(target)) return + setOpen(false) + } + document.addEventListener('mousedown', onDoc) + return () => document.removeEventListener('mousedown', onDoc) + }, []) + return ( +
+
+ {label} + {hint ? {hint} : null} +
+ + + {open && !disabled && coords && typeof document !== 'undefined' + ? createPortal( +
+ {credentials.length === 0 ? ( +

No active integrations

+ ) : ( + credentials.map((credential) => { + const isSelected = credential.id === value + const name = credential.name?.trim() + return ( + + ) + }) + )} +
, + document.body, + ) + : null}
+ ) +} diff --git a/frontend/src/pages/usage/SearchableSelect.tsx b/frontend/src/pages/usage/SearchableSelect.tsx index 61da046a..c5f5b1a5 100644 --- a/frontend/src/pages/usage/SearchableSelect.tsx +++ b/frontend/src/pages/usage/SearchableSelect.tsx @@ -1,7 +1,8 @@ -import { useEffect, useMemo, useRef, useState } from 'react' -import { ChevronDown, Search, X } from 'lucide-react' +import { useEffect, useMemo, useRef, useState, type ReactNode } from 'react' +import { createPortal } from 'react-dom' +import { ChevronDown, Info, Search, X } from 'lucide-react' import { usageTheme } from './usageTheme' - +import { useAnchoredDropdown } from './useAnchoredDropdown' export type SearchableOption = { id: string; label: string } type SearchableSelectProps = { @@ -12,6 +13,27 @@ type SearchableSelectProps = { onChange: (id: string) => void disabled?: boolean emptyMessage?: string + hint?: string +} + +function FieldHint({ title, children }: { title: string; children: ReactNode }) { + return ( + + + + + {title} + + {children} + + + ) } export default function SearchableSelect({ @@ -22,11 +44,14 @@ export default function SearchableSelect({ onChange, disabled, emptyMessage = 'No matches', + hint, }: SearchableSelectProps) { const [open, setOpen] = useState(false) const [search, setSearch] = useState('') const rootRef = useRef(null) - + const anchorRef = useRef(null) + const panelRef = useRef(null) + const coords = useAnchoredDropdown(open && !disabled, anchorRef) const selected = options.find( (o) => o.id === value || o.id.toLowerCase() === value.toLowerCase(), ) @@ -39,7 +64,9 @@ export default function SearchableSelect({ useEffect(() => { const onDoc = (e: MouseEvent) => { - if (!rootRef.current?.contains(e.target as Node)) setOpen(false) + const target = e.target as Node + if (rootRef.current?.contains(target) || panelRef.current?.contains(target)) return + setOpen(false) } document.addEventListener('mousedown', onDoc) return () => document.removeEventListener('mousedown', onDoc) @@ -47,17 +74,20 @@ export default function SearchableSelect({ return (
- {label} +
+ {label} + {hint ? {hint} : null} +
- - )) - )} - -
- ) : null} -
+ {open && !disabled && coords && typeof document !== 'undefined' + ? createPortal( +
+
+ + setSearch(e.target.value)} + placeholder="Search…" + className="min-w-0 flex-1 text-sm outline-none" + autoFocus + /> +
+
    + {filtered.length === 0 ? ( +
  • {emptyMessage}
    • + ) : ( + filtered.map((opt) => ( +
  • + +
    • + )) + )} +
+
, + document.body, + ) + : null}
) } diff --git a/frontend/src/pages/usage/Usage.tsx b/frontend/src/pages/usage/Usage.tsx index 469df3c3..1ce1a505 100644 --- a/frontend/src/pages/usage/Usage.tsx +++ b/frontend/src/pages/usage/Usage.tsx @@ -1,16 +1,16 @@ import { useEffect, useMemo, useState } from 'react' import { keepPreviousData, useQuery } from '@tanstack/react-query' -import { Link, useSearchParams } from 'react-router-dom' +import { useSearchParams } from 'react-router-dom' import { Card, CardBody, Spinner } from '@heroui/react' -import { Activity, ChevronRight, CircleDollarSign } from 'lucide-react' +import { ChevronRight, CircleDollarSign } from 'lucide-react' import { apiClient } from '../../lib/api' -import { useIsAdmin } from '../../hooks/useRole' import UsageFiltersBar from './UsageFiltersBar' import UsageDrillPath from './UsageDrillPath' import UsageCostBreakdownModal from './UsageCostBreakdownModal' import { defaultUsageDateRange, isRangeWithinMaxDays, rangeForDays } from './UsageDateRangePicker' import { getUsageTimezone } from './usageTimezone' import { + formatFxRateHint, formatUsageCostUsd, getUsageDisplayCurrency, setUsageDisplayCurrency, @@ -556,7 +556,6 @@ export default function Usage() { const model = searchParams.get('model') || '' const usageKind = (searchParams.get('usage_kind') as Kind) || '' const productSection = searchParams.get('product_section') || '' - const isAdmin = useIsAdmin() const { usagePolicy, isLoaded: licenseLoaded, fetchLicense } = useLicenseStore() const showOssUsageNotice = licenseLoaded && !usagePolicy.extended_history const maxHistoryDays = !licenseLoaded @@ -573,8 +572,10 @@ export default function Usage() { queryKey: ['org-usage', 'fx-rate'], queryFn: () => apiClient.getOrgUsageFxRate(), staleTime: 60 * 60 * 1000, + refetchOnMount: 'always', }) - const inrRate = fxRate?.rate ?? 83 + const inrRate = fxRate?.rate ?? 95 + const fxRateHint = formatFxRateHint(inrRate, fxRate?.as_of, fxRate?.source) const formatCostUsd = (usd?: number | null) => formatUsageCostUsd(usd, displayCurrency, inrRate) @@ -1097,69 +1098,48 @@ export default function Usage() { } return ( -
-
-
-

- - Usage -

-

- Cards show usage for {scopeSubtitle}. - Drill down: workspaces → call imports or product areas → evaluations / models. -

-
-
-
-
+
+
+ {(['USD', 'INR'] as const).map((currency) => ( + - ))} -
- {showCostBreakdown ? ( - - ) : null} - {isAdmin && licenseLoaded && usagePolicy.extended_history ? ( - - Pricing overrides - - ) : null} -
- {summary?.last_updated_at ? ( -

- Updated {new Date(summary.last_updated_at).toLocaleString()} -

- ) : null} + {currency} + + ))}
+ {showCostBreakdown ? ( + + ) : null} + {summary?.last_updated_at ? ( +

+ Updated {new Date(summary.last_updated_at).toLocaleString()} +

+ ) : null}
{summaryError ? ( diff --git a/frontend/src/pages/usage/UsageCostBreakdownModal.tsx b/frontend/src/pages/usage/UsageCostBreakdownModal.tsx index 645729ba..2eafa3b1 100644 --- a/frontend/src/pages/usage/UsageCostBreakdownModal.tsx +++ b/frontend/src/pages/usage/UsageCostBreakdownModal.tsx @@ -1,4 +1,5 @@ import { X } from 'lucide-react' +import { formatUsageCostUsd } from '../../lib/usageCurrency' type UsageCosts = { input_cost_usd: number @@ -21,14 +22,8 @@ type Props = { } function defaultFormatCostUsd(usd?: number | null): string { - const amount = Number(usd || 0) - if (!amount) return '$0.00' - return new Intl.NumberFormat(undefined, { - style: 'currency', - currency: 'USD', - minimumFractionDigits: 2, - maximumFractionDigits: 4, - }).format(amount) + const formatted = formatUsageCostUsd(usd, 'USD', 1) + return formatted === '—' ? '$0.00' : formatted } const LINE_ITEMS: Array<{ key: keyof UsageCosts; label: string }> = [ diff --git a/frontend/src/pages/usage/UsagePage.tsx b/frontend/src/pages/usage/UsagePage.tsx new file mode 100644 index 00000000..9acddfba --- /dev/null +++ b/frontend/src/pages/usage/UsagePage.tsx @@ -0,0 +1,72 @@ +import { useEffect } from 'react' +import { Activity, DollarSign } from 'lucide-react' +import { Navigate, useSearchParams } from 'react-router-dom' +import { useIsAdmin } from '../../hooks/useRole' +import { useLicenseStore } from '../../store/licenseStore' +import Usage from './Usage' +import UsagePricing from './UsagePricing' + +type UsageTab = 'overview' | 'pricing' + +export default function UsagePage() { + const [searchParams, setSearchParams] = useSearchParams() + const isAdmin = useIsAdmin() + const licenseLoaded = useLicenseStore((s) => s.isLoaded) + const hasExtendedHistory = useLicenseStore((s) => s.hasExtendedUsageHistory()) + const canManagePricing = isAdmin && licenseLoaded && hasExtendedHistory + + const tabParam = searchParams.get('tab') + const activeTab: UsageTab = + tabParam === 'pricing' && canManagePricing ? 'pricing' : 'overview' + + useEffect(() => { + if (tabParam === 'pricing' && !canManagePricing) { + setSearchParams({}, { replace: true }) + } + }, [tabParam, canManagePricing, setSearchParams]) + + const setActiveTab = (tab: UsageTab) => { + setSearchParams(tab === 'pricing' ? { tab: 'pricing' } : {}, { replace: true }) + } + + const tabs: Array<{ id: UsageTab; label: string; icon: typeof Activity; hidden?: boolean }> = [ + { id: 'overview', label: 'Overview', icon: Activity }, + { id: 'pricing', label: 'Pricing overrides', icon: DollarSign, hidden: !canManagePricing }, + ] + + return ( +
+

Usage

+ + {canManagePricing ? ( +
+ +
+ ) : null} + + {activeTab === 'overview' ? : } +
+ ) +} + +export function UsagePricingRedirect() { + return +} diff --git a/frontend/src/pages/usage/UsagePricing.tsx b/frontend/src/pages/usage/UsagePricing.tsx index 92d159f3..6796751b 100644 --- a/frontend/src/pages/usage/UsagePricing.tsx +++ b/frontend/src/pages/usage/UsagePricing.tsx @@ -1,22 +1,22 @@ -import { useMemo, useState, type ReactNode } from 'react' +import { useEffect, useMemo, useRef, useState, type ReactNode } from 'react' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import { Card, CardBody, Spinner } from '@heroui/react' -import { Link } from 'react-router-dom' -import { ArrowLeft, DollarSign, Info, Plus, Trash2 } from 'lucide-react' +import { Plus, Trash2, Info } from 'lucide-react' import { apiClient } from '../../lib/api' import { useIsAdmin } from '../../hooks/useRole' import { useToast } from '../../hooks/useToast' import Button from '../../components/Button' import SearchableSelect from './SearchableSelect' +import ProviderCredentialSelect from './ProviderCredentialSelect' import { usageTheme } from './usageTheme' import { - formatUsageCostUsd, - getUsageDisplayCurrency, - setUsageDisplayCurrency, - type UsageDisplayCurrency, -} from '../../lib/usageCurrency' + buildPricingModelOptions, + credentialDisplayLabel, + type PricingUsageKind, +} from './pricingModelOptions' +import type { AIProvider } from '../../types/api' -type UsageKind = 'llm' | 'stt' | 'tts' +type UsageKind = PricingUsageKind type PricingRatesUsd = { input_per_1m?: number | null @@ -51,36 +51,137 @@ const EMPTY_RATES: PricingRatesUsd = { tts_per_1m_characters: undefined, } -const RATE_FIELDS: Array<{ key: RateFieldKey; label: string; kinds: UsageKind[] }> = [ - { key: 'input_per_1m', label: 'Input / 1M tokens (USD)', kinds: ['llm'] }, - { key: 'output_per_1m', label: 'Output / 1M tokens (USD)', kinds: ['llm'] }, - { key: 'cache_read_per_1m', label: 'Cache read / 1M (USD)', kinds: ['llm'] }, - { key: 'cache_write_per_1m', label: 'Cache write / 1M (USD)', kinds: ['llm'] }, - { key: 'reasoning_per_1m', label: 'Reasoning / 1M (USD)', kinds: ['llm'] }, - { key: 'audio_per_minute', label: 'Audio / minute (USD)', kinds: ['llm', 'stt'] }, - { key: 'tts_per_1m_characters', label: 'TTS / 1M characters (USD)', kinds: ['tts'] }, +const RATE_FIELDS: Array<{ + key: RateFieldKey + label: string + kinds: UsageKind[] + hint: string +}> = [ + { + key: 'input_per_1m', + label: 'Input / 1M tokens ($)', + kinds: ['llm'], + hint: 'USD per 1 million prompt or input tokens. Used when costing LLM calls for this model.', + }, + { + key: 'output_per_1m', + label: 'Output / 1M tokens ($)', + kinds: ['llm'], + hint: 'USD per 1 million completion or output tokens returned by the model.', + }, + { + key: 'cache_read_per_1m', + label: 'Cache read / 1M ($)', + kinds: ['llm'], + hint: 'USD per 1 million tokens read from prompt cache, when the provider bills cached input separately.', + }, + { + key: 'cache_write_per_1m', + label: 'Cache write / 1M ($)', + kinds: ['llm'], + hint: 'USD per 1 million tokens written to prompt cache on the first request.', + }, + { + key: 'reasoning_per_1m', + label: 'Reasoning / 1M ($)', + kinds: ['llm'], + hint: 'USD per 1 million reasoning or thinking tokens (e.g. o-series models).', + }, + { + key: 'audio_per_minute', + label: 'Audio / minute ($)', + kinds: ['llm', 'stt'], + hint: 'USD per minute of audio processed — STT transcription or multimodal audio input on LLM calls.', + }, + { + key: 'tts_per_1m_characters', + label: 'TTS / 1M characters ($)', + kinds: ['tts'], + hint: 'USD per 1 million characters sent to the TTS model for synthesis.', + }, ] +const FIELD_HINTS = { + provider: + 'AI company from Integrations (e.g. OpenAI, Anthropic). Pick the credential whose models you want to price.', + model: + 'Model name for the selected provider and usage kind. Only enabled models for that integration are listed.', + usageKind: + 'Choose LLM, STT, or TTS first — this filters which models appear. Rate fields below match the kind.', + effectiveFrom: + 'First calendar day this override applies. Saving recalculates historical usage for this model from this date.', + ratesSection: + 'Enter USD rates only. Leave a field empty to inherit the platform catalog. At least one rate is required to save.', +} as const + const CONTROL_CLASS = `h-10 w-full rounded-lg border border-gray-200 bg-white px-3 text-sm text-gray-900 shadow-sm outline-none transition-colors hover:border-gray-300 ${usageTheme.focusRing}` +function FieldHint({ title, children }: { title: string; children: ReactNode }) { + return ( + + + + + {title} + + {children} + + + ) +} + function FormField({ label, + hint, children, className = '', }: { label: string + hint?: string children: ReactNode className?: string }) { return (
- {label} +
+ {label} + {hint ? ( + {hint} + ) : null} +
{children}
) } +function formatRateUsd(value?: number | null): string { + if (value == null || Number.isNaN(value)) return '—' + return `$${value.toLocaleString('en-US', { + minimumFractionDigits: 2, + maximumFractionDigits: 4, + })}` +} + +function ratesFromApi(raw?: Record | null): PricingRatesUsd { + if (!raw) return { ...EMPTY_RATES } + return { + input_per_1m: raw.input_per_1m ?? undefined, + output_per_1m: raw.output_per_1m ?? undefined, + cache_read_per_1m: raw.cache_read_per_1m ?? undefined, + cache_write_per_1m: raw.cache_write_per_1m ?? undefined, + reasoning_per_1m: raw.reasoning_per_1m ?? undefined, + audio_per_minute: raw.audio_per_minute ?? undefined, + tts_per_1m_characters: raw.tts_per_1m_characters ?? undefined, + } +} + function todayIso(): string { return new Date().toISOString().slice(0, 10) } @@ -90,26 +191,45 @@ export default function UsagePricing() { const queryClient = useQueryClient() const { showToast, ToastContainer } = useToast() - const [displayCurrency, setDisplayCurrency] = useState(() => - getUsageDisplayCurrency(), - ) - const { data: fxRate } = useQuery({ - queryKey: ['org-usage', 'fx-rate'], - queryFn: () => apiClient.getOrgUsageFxRate(), - staleTime: 60 * 60 * 1000, - }) - const inrRate = fxRate?.rate ?? 83 - const formatDisplayRate = (value?: number | null) => - formatUsageCostUsd(value, displayCurrency, inrRate) - + const [credentialId, setCredentialId] = useState('') const [model, setModel] = useState('') const [usageKind, setUsageKind] = useState('llm') const [effectiveFrom, setEffectiveFrom] = useState(todayIso()) const [rates, setRates] = useState(EMPTY_RATES) + const [ratePrefillSource, setRatePrefillSource] = useState<'catalog' | 'override' | null>(null) + const selectionRef = useRef({ model: '', usageKind: '' as UsageKind }) + + const { data: aiProviders = [] } = useQuery({ + queryKey: ['ai-providers'], + queryFn: () => apiClient.listAIProviders(), + enabled: isAdmin, + }) + + const activeCredentials = useMemo( + () => + (aiProviders as AIProvider[]) + .filter((row) => row.is_active) + .sort((a, b) => credentialDisplayLabel(a).localeCompare(credentialDisplayLabel(b))), + [aiProviders], + ) + + const selectedCredential = useMemo( + () => activeCredentials.find((row) => row.id === credentialId), + [activeCredentials, credentialId], + ) + + const { data: providerCatalog } = useQuery({ + queryKey: ['model-options', selectedCredential?.provider], + queryFn: () => apiClient.getModelOptions(selectedCredential!.provider), + enabled: Boolean(selectedCredential?.provider), + }) - const { data: filters } = useQuery({ - queryKey: ['org-usage', 'filters'], - queryFn: () => apiClient.getOrgUsageFilters(), + const { data: availableModels = [] } = useQuery({ + queryKey: ['usage-pricing', 'available-models'], + queryFn: async () => { + const payload = await apiClient.listUsagePricingAvailableModels() + return payload.models || [] + }, enabled: isAdmin, }) @@ -119,13 +239,86 @@ export default function UsagePricing() { enabled: isAdmin, }) - const modelOptions = useMemo(() => { - const fromUsage = filters?.models || [] - const fromOverrides = overrides.map((row) => row.model) - return Array.from(new Set([...fromUsage, ...fromOverrides])) - .sort() - .map((name) => ({ id: name, label: name })) - }, [filters?.models, overrides]) + const { data: effectivePricing, isFetching: effectiveLoading } = useQuery({ + queryKey: ['usage-pricing', 'effective', model, usageKind, effectiveFrom], + queryFn: () => + apiClient.getUsagePricingEffective(model, { + usage_kind: usageKind, + as_of: effectiveFrom, + }), + enabled: isAdmin && Boolean(model), + }) + + const eligibleModelSet = useMemo( + () => new Set(availableModels || []), + [availableModels], + ) + + const modelOptions = useMemo( + () => + buildPricingModelOptions({ + credential: selectedCredential, + catalog: providerCatalog, + kind: usageKind, + eligibleModels: eligibleModelSet, + overrideModels: overrides + .filter((row) => row.usage_kind === usageKind) + .map((row) => row.model), + }), + [selectedCredential, providerCatalog, usageKind, eligibleModelSet, overrides], + ) + + const resetModelAndRates = () => { + setModel('') + setRates(EMPTY_RATES) + setRatePrefillSource(null) + selectionRef.current = { model: '', usageKind: 'llm' } + } + + useEffect(() => { + if (credentialId || activeCredentials.length !== 1) return + setCredentialId(activeCredentials[0].id) + }, [activeCredentials, credentialId]) + + useEffect(() => { + if (!model) { + setRates(EMPTY_RATES) + setRatePrefillSource(null) + selectionRef.current = { model: '', usageKind: usageKind } + return + } + if (!effectivePricing) return + + const selectionChanged = + selectionRef.current.model !== model || + selectionRef.current.usageKind !== usageKind + + const nextRates = ratesFromApi(effectivePricing.effective_rates) + setRates(nextRates) + setRatePrefillSource( + effectivePricing.has_override + ? 'override' + : effectivePricing.effective_rates + ? 'catalog' + : null, + ) + + if (selectionChanged && effectivePricing.override?.effective_from) { + setEffectiveFrom(effectivePricing.override.effective_from) + } + + selectionRef.current = { model, usageKind } + }, [model, usageKind, effectivePricing]) + + const handleCredentialChange = (nextId: string) => { + setCredentialId(nextId) + resetModelAndRates() + } + + const handleUsageKindChange = (nextKind: UsageKind) => { + setUsageKind(nextKind) + resetModelAndRates() + } const visibleRateFields = useMemo( () => RATE_FIELDS.filter((field) => field.kinds.includes(usageKind)), @@ -151,6 +344,7 @@ export default function UsagePricing() { queryClient.invalidateQueries({ queryKey: ['usage-pricing'] }) queryClient.invalidateQueries({ queryKey: ['org-usage'] }) setRates(EMPTY_RATES) + setRatePrefillSource(null) showToast( data.recompute_enqueued ? 'Override saved; cost recompute started' @@ -188,104 +382,62 @@ export default function UsagePricing() { } return ( -
+
-
- - - Back to usage - -
-
- -
-
-
-

Pricing overrides

-
- {(['USD', 'INR'] as const).map((currency) => ( - - ))} -
-
- -
- Set custom USD rates for this organization. Empty fields inherit the platform - catalog. Saving updates costs for matching usage and starts a scoped recompute. -
-
-
-

- Per-model rates for this org. Leave blank to use the default catalog. -

-
-
-
- - +

Add or update override

- Saving triggers a scoped cost recompute for this model. + Per-model rates for this org. Leave blank to use the default catalog. Saving triggers + a scoped cost recompute.

- - + - + +
+ +
+
-
-

- Rate overrides (USD) + {credentialId && modelOptions.length === 0 ? ( +

+ No {usageKind.toUpperCase()} models are available for this integration. Enable models + in Integrations → AI provider → step 2, or pick another usage kind. +

+ ) : null} + + {activeCredentials.length === 0 ? ( +

+ Add an AI provider in Integrations before setting pricing overrides.

+ ) : null} + + {model ? ( +
+ {effectiveLoading ? ( +
+ + Loading current rates… +
+ ) : ratePrefillSource ? ( +

+ {ratePrefillSource === 'override' + ? 'Prefilled from your org override — these are the rates usage is billed at today. Edit and save to update.' + : 'Prefilled from platform catalog — these are the default rates in effect. Edit and save to create an org override.'} +

+ ) : ( +

+ No catalog rate found for this model on the selected date. Enter rates manually. +

+ )} +
+

+ Rate overrides ($) +

+ {FIELD_HINTS.ratesSection} +
- {visibleRateFields.map(({ key, label }) => ( - + {visibleRateFields.map(({ key, label, hint }) => ( + { @@ -321,6 +511,11 @@ export default function UsagePricing() { ))}
+ ) : ( +

+ Select a provider, usage kind, and model to configure rate overrides. +

+ )}
) : null} + {isCustomProvider && !gateway && enabledModels.length === 0 && !isLoading ? ( +
+ Add custom model IDs below, or set a Gateway model on + the previous step for a single pinned Bifrost model. +
+ ) : null} + {isLoading ? (

Loading catalog models…

) : ( @@ -175,38 +205,63 @@ export default function AIProviderEnabledModelsStep({ }) )} -
- -
- setCustomModel(e.target.value)} - onKeyDown={(e) => { - if (e.key === 'Enter') { - e.preventDefault() - addCustomModel() + {customModels.length > 0 ? ( +
+
+ Enabled models + + {customModels.filter((m) => selected.has(m)).length}/{customModels.length} selected + +
+
+ {customModels.map((model) => ( + toggle(model)} + /> + ))} +
+
+ ) : null} + + {showManualAdd ? ( +
+ +
+ setCustomModel(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault() + addCustomModel() + } + }} + className={`flex-1 rounded-lg border border-gray-200 px-3 py-2 text-sm focus:outline-none ${usageTheme.focusRing}`} + placeholder={ + isCustomProvider + ? 'e.g. openai/gpt-4o or production-gpt4' + : 'e.g. accounts/fireworks/models/gpt-oss-120b' } - }} - className={`flex-1 rounded-lg border border-gray-200 px-3 py-2 text-sm focus:outline-none ${usageTheme.focusRing}`} - placeholder="e.g. accounts/fireworks/models/gpt-oss-120b" - /> - + /> + +
-
+ ) : null}

{enabledModels.length > 0 ? `${enabledModels.length} model${enabledModels.length === 1 ? '' : 's'} enabled` - : 'No restriction — full catalog allowed'} + : isCustomProvider + ? 'No models enabled yet — add model IDs above or set a gateway model on step 1' + : 'No restriction — full catalog allowed'}

) diff --git a/frontend/src/pages/configurations/Integrations.tsx b/frontend/src/pages/configurations/Integrations.tsx index 4c2e9ca6..3e0373ec 100644 --- a/frontend/src/pages/configurations/Integrations.tsx +++ b/frontend/src/pages/configurations/Integrations.tsx @@ -617,9 +617,14 @@ export default function Integrations() { aiIntegrationProviders.length > 0 || hasTelephony + const activeAIProvider = + selectedProvider || + (selectedAIProvider?.provider as ModelProvider | undefined) || + null const showGatewayModelField = integrationType === 'ai_provider' && - (credentialRoutingMode === 'gateway' || + (activeAIProvider === ModelProvider.CUSTOM || + credentialRoutingMode === 'gateway' || (credentialRoutingMode === 'inherit' && llmGatewaySettings?.effective_routing && llmGatewaySettings.effective_routing !== 'direct')) diff --git a/tests/test_api/test_gateway_managed_credentials.py b/tests/test_api/test_gateway_managed_credentials.py index 3b59c58f..f420e972 100644 --- a/tests/test_api/test_gateway_managed_credentials.py +++ b/tests/test_api/test_gateway_managed_credentials.py @@ -196,7 +196,31 @@ def test_update_gateway_managed_to_direct_requires_api_key( assert update_response.status_code == 400 -def test_create_aiprovider_rejects_invalid_gateway_base_url(authenticated_client): +def test_list_aiproviders_gateway_without_base_url_returns_200( + authenticated_client, db_session, org_id +): + """Integrations page must not 500 when a gateway credential lacks base_url.""" + _set_platform_gateway_passthrough(False) + settings.LLM_GATEWAY_ENABLED = False + settings.LLM_GATEWAY_BASE_URL = None + + row = AIProvider( + organization_id=org_id, + provider="custom", + api_key="enc-key", + name="Misconfigured gateway", + routing_mode="gateway", + gateway_model="gpt-oss-120b", + is_active=True, + ) + db_session.add(row) + db_session.commit() + + response = authenticated_client.get("/api/v1/aiproviders") + assert response.status_code == 200 + body = response.json() + assert len(body) == 1 + assert body[0]["effective_routing"] == "gateway" _set_platform_gateway_passthrough(False) settings.LLM_GATEWAY_ENABLED = True settings.LLM_GATEWAY_BASE_URL = "http://localhost:8080" diff --git a/tests/test_services/test_ai/test_llm_gateway.py b/tests/test_services/test_ai/test_llm_gateway.py index 3e24de64..1d72295a 100644 --- a/tests/test_services/test_ai/test_llm_gateway.py +++ b/tests/test_services/test_ai/test_llm_gateway.py @@ -392,6 +392,23 @@ def test_credential_gateway_raises_when_no_base_url(): resolve_effective_routing(org_id, db, ctx) +def test_effective_routing_label_gateway_without_base_url_does_not_raise(): + _set_platform_gateway(enabled=False) + org_id, db = _org_db({"enabled": False}) + provider = SimpleNamespace( + provider="custom", + routing_mode="gateway", + gateway_interface="native_openai", + gateway_base_url=None, + gateway_model="gpt-oss-120b", + gateway_auth_header=None, + gateway_auth_secret_env=None, + gateway_auth_secret=None, + gateway_extra_headers=None, + ) + assert get_credential_effective_routing_label(org_id, db, provider) == "gateway" + + def test_effective_routing_label_uses_credential_gateway_base_url(): _set_platform_gateway(enabled=False) org_id, db = _org_db({"enabled": False}) diff --git a/tests/test_services/test_cron/test_job_dispatch.py b/tests/test_services/test_cron/test_job_dispatch.py index ff31ac36..1f938ba0 100644 --- a/tests/test_services/test_cron/test_job_dispatch.py +++ b/tests/test_services/test_cron/test_job_dispatch.py @@ -27,22 +27,33 @@ def test_advance_cron_job_marks_completed_when_max_runs_reached(): assert job.next_run_at is None -def test_enqueue_usage_flush_routes_to_usage_task(monkeypatch): +def test_enqueue_unknown_job_type_returns_unknown(): job = MagicMock() job.job_type = "usage_flush" job.id = uuid4() + meta = enqueue_cron_job(job) + + assert meta["task"] == "unknown" + assert meta["job_type"] == "usage_flush" + + +def test_enqueue_evaluator_run_routes_to_evaluator_task(monkeypatch): + job = MagicMock() + job.job_type = "evaluator_run" + job.id = uuid4() + delayed = MagicMock() - delayed.id = "task-123" + delayed.id = "task-456" task = MagicMock() task.delay.return_value = delayed monkeypatch.setattr( - "app.workers.tasks.flush_usage_counters.flush_usage_counters_task", + "app.workers.tasks.run_cron_evaluator_job.run_cron_evaluator_job_task", task, ) meta = enqueue_cron_job(job) - assert meta["task"] == "flush_usage_counters" - assert meta["celery_task_id"] == "task-123" - task.delay.assert_called_once_with() + assert meta["task"] == "run_cron_evaluator_job" + assert meta["celery_task_id"] == "task-456" + task.delay.assert_called_once_with(str(job.id)) diff --git a/tests/test_services/test_usage/test_enabled_models.py b/tests/test_services/test_usage/test_enabled_models.py index 88d2737a..79610d30 100644 --- a/tests/test_services/test_usage/test_enabled_models.py +++ b/tests/test_services/test_usage/test_enabled_models.py @@ -35,6 +35,20 @@ def test_filter_models_unrestricted_when_allowlist_empty(): assert filter_models_by_credential(cred, ["a", "b"]) == ["a", "b"] +def test_filter_models_by_credential_uses_allowlist_when_catalog_empty(): + cred = _Credential(enabled_models=["openai/gpt-4o", "production-gpt4"], provider="custom") + assert filter_models_by_credential(cred, []) == ["openai/gpt-4o", "production-gpt4"] + + +def test_filter_models_by_credential_empty_catalog_includes_gateway_model(): + cred = _Credential( + enabled_models=["openai/gpt-4o"], + gateway_model="pinned-model", + provider="custom", + ) + assert filter_models_by_credential(cred, []) == ["pinned-model", "openai/gpt-4o"] + + def test_org_pricing_eligible_models_includes_usage_and_overrides(monkeypatch): org_id = uuid4() diff --git a/tests/test_workers/test_beat_schedule.py b/tests/test_workers/test_beat_schedule.py new file mode 100644 index 00000000..88c84aa7 --- /dev/null +++ b/tests/test_workers/test_beat_schedule.py @@ -0,0 +1,33 @@ +"""Tests for Celery Beat platform schedule.""" + +from app.workers.config import _platform_beat_schedule, _usage_flush_beat_seconds + + +def test_usage_flush_beat_seconds_defaults_to_120(): + assert _usage_flush_beat_seconds() >= 30.0 + + +def test_usage_flush_beat_seconds_respects_env(monkeypatch): + monkeypatch.setenv("USAGE_FLUSH_BEAT_SECONDS", "90") + assert _usage_flush_beat_seconds() == 90.0 + + +def test_usage_flush_beat_seconds_clamps_minimum(monkeypatch): + monkeypatch.setenv("USAGE_FLUSH_BEAT_SECONDS", "5") + assert _usage_flush_beat_seconds() == 30.0 + + +def test_platform_beat_schedule_has_four_entries(): + schedule = _platform_beat_schedule() + assert set(schedule.keys()) == { + "flush-usage-counters", + "evaluate-alerts", + "refresh-fx-rates", + "prune-oss-usage-history", + } + + +def test_flush_schedule_uses_env_interval(monkeypatch): + monkeypatch.setenv("USAGE_FLUSH_BEAT_SECONDS", "180") + schedule = _platform_beat_schedule() + assert schedule["flush-usage-counters"]["schedule"] == 180.0 diff --git a/tests/test_workers/test_usage_queue_routing.py b/tests/test_workers/test_usage_queue_routing.py index c3a5776a..d8751354 100644 --- a/tests/test_workers/test_usage_queue_routing.py +++ b/tests/test_workers/test_usage_queue_routing.py @@ -1,6 +1,6 @@ """Tests for usage pricing Celery queue routing.""" -from app.workers.config import USAGE_WORKER_QUEUE, celery_app +from app.workers.config import PLATFORM_WORKER_QUEUE, USAGE_WORKER_QUEUE, celery_app def test_flush_usage_counters_routes_to_usage_queue(): @@ -18,6 +18,16 @@ def test_cron_dispatcher_routes_to_default_worker(): assert routes["dispatch_cron_jobs"]["queue"] == "celery" -def test_beat_schedule_removed(): +def test_beat_schedule_includes_platform_tasks(): schedule = getattr(celery_app.conf, "beat_schedule", None) or {} - assert "flush-llm-usage-counters" not in schedule + assert schedule["flush-usage-counters"]["task"] == "flush_usage_counters" + assert schedule["evaluate-alerts"]["task"] == "evaluate_alerts" + assert schedule["refresh-fx-rates"]["task"] == "refresh_fx_rates" + assert schedule["prune-oss-usage-history"]["task"] == "prune_oss_usage_history" + + +def test_platform_tasks_route_to_platform_queue(): + routes = celery_app.conf.task_routes + assert routes["evaluate_alerts"]["queue"] == PLATFORM_WORKER_QUEUE + assert routes["refresh_fx_rates"]["queue"] == PLATFORM_WORKER_QUEUE + assert routes["prune_oss_usage_history"]["queue"] == PLATFORM_WORKER_QUEUE From 02e8c0a258e76da4ff9885f6f1e319785c5e9bc2 Mon Sep 17 00:00:00 2001 From: M Sami Date: Mon, 17 Aug 2026 18:16:17 +0530 Subject: [PATCH 28/32] fix(migrations): widen source column in model_pricing_rates to VARCHAR(255) and add migration function --- app/migrations/072_usage_pricing_phase1.py | 21 +++++- .../src/pages/configurations/Integrations.tsx | 70 +++++++++++++------ 2 files changed, 66 insertions(+), 25 deletions(-) diff --git a/app/migrations/072_usage_pricing_phase1.py b/app/migrations/072_usage_pricing_phase1.py index 68254deb..3ae097e7 100644 --- a/app/migrations/072_usage_pricing_phase1.py +++ b/app/migrations/072_usage_pricing_phase1.py @@ -77,7 +77,7 @@ def _ensure_model_pricing_rates(db: Session) -> None: effective_from DATE NOT NULL DEFAULT CURRENT_DATE, effective_to DATE, currency VARCHAR(8) NOT NULL DEFAULT 'USD', - source VARCHAR(32) NOT NULL DEFAULT 'catalog', + source VARCHAR(255) NOT NULL DEFAULT 'catalog', input_micro_usd_per_million BIGINT NOT NULL DEFAULT 0, output_micro_usd_per_million BIGINT NOT NULL DEFAULT 0, cache_read_micro_usd_per_million BIGINT NOT NULL DEFAULT 0, @@ -117,12 +117,28 @@ def _ensure_model_pricing_rates(db: Session) -> None: text( """ ALTER TABLE model_pricing_rates - ADD COLUMN source VARCHAR(32) NOT NULL DEFAULT 'catalog' + ADD COLUMN source VARCHAR(255) NOT NULL DEFAULT 'catalog' """ ) ) +def _widen_source_column(db: Session) -> None: + if not _table_exists(db, "model_pricing_rates"): + return + if not _column_exists(db, "model_pricing_rates", "source"): + return + db.execute( + text( + """ + ALTER TABLE model_pricing_rates + ALTER COLUMN source TYPE VARCHAR(255) + """ + ) + ) + print("Widened model_pricing_rates.source to VARCHAR(255)") + + def _add_buffer_cost_columns(db: Session) -> None: if not _table_exists(db, "usage_pending_buffer"): return @@ -161,6 +177,7 @@ def _strip_org_extras(db: Session) -> None: def upgrade(db: Session): _ensure_model_pricing_rates(db) + _widen_source_column(db) _add_buffer_cost_columns(db) _strip_org_extras(db) db.commit() diff --git a/frontend/src/pages/configurations/Integrations.tsx b/frontend/src/pages/configurations/Integrations.tsx index 3e0373ec..0dae3165 100644 --- a/frontend/src/pages/configurations/Integrations.tsx +++ b/frontend/src/pages/configurations/Integrations.tsx @@ -194,6 +194,23 @@ export default function Integrations() { ? llmGatewayType : llmGatewaySettings?.platform_gateway_type || 'bifrost' + const activeAIProvider = + selectedProvider || + (selectedAIProvider?.provider as ModelProvider | undefined) || + null + const isCustomAIProvider = + integrationType === 'ai_provider' && + String(activeAIProvider || '').toLowerCase() === ModelProvider.CUSTOM + const aiProviderUsesModelsStep = integrationType === 'ai_provider' && !isCustomAIProvider + const showGatewayModelField = + integrationType === 'ai_provider' && + (isCustomAIProvider || + credentialRoutingMode === 'gateway' || + (credentialRoutingMode === 'inherit' && + llmGatewaySettings?.effective_routing && + llmGatewaySettings.effective_routing !== 'direct')) + const aiProviderRequiresApiKey = credentialRoutingMode === 'direct' + const showLlmGatewayConfigOptions = llmGatewayMode !== 'disabled' useEffect(() => { @@ -455,8 +472,10 @@ export default function Integrations() { showToast('Please enter your Azure OpenAI endpoint URL', 'error') return } - setAiProviderWizardStep(2) - return + if (!isCustomAIProvider) { + setAiProviderWizardStep(2) + return + } } if (isEditMode && selectedAIProvider) { @@ -617,20 +636,6 @@ export default function Integrations() { aiIntegrationProviders.length > 0 || hasTelephony - const activeAIProvider = - selectedProvider || - (selectedAIProvider?.provider as ModelProvider | undefined) || - null - const showGatewayModelField = - integrationType === 'ai_provider' && - (activeAIProvider === ModelProvider.CUSTOM || - credentialRoutingMode === 'gateway' || - (credentialRoutingMode === 'inherit' && - llmGatewaySettings?.effective_routing && - llmGatewaySettings.effective_routing !== 'direct')) - - const aiProviderRequiresApiKey = credentialRoutingMode === 'direct' - const getPlatformInfo = (platformId: IntegrationPlatform) => { return platforms.find(p => p.id === platformId) } @@ -1159,7 +1164,7 @@ export default function Integrations() { {showModal && renderModal(
-
+

@@ -1171,11 +1176,13 @@ export default function Integrations() { : 'Edit Integration' : 'Add Integration'}

- {integrationType === 'ai_provider' ? ( + {aiProviderUsesModelsStep ? (

Step {aiProviderWizardStep} of 2 —{' '} {aiProviderWizardStep === 1 ? 'Credentials' : 'Enabled models'}

+ ) : isCustomAIProvider ? ( +

Custom Bifrost model integration

) : null}
@@ -1361,16 +1368,20 @@ export default function Integrations() { {showGatewayModelField && ( <>
- + setGatewayModel(e.target.value)} className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-primary-500" - placeholder="e.g., production-gpt4 or openai/gpt-4o" + placeholder={isCustomAIProvider ? 'e.g. openai/gpt-4o or production-gpt4' : 'e.g., production-gpt4 or openai/gpt-4o'} />

- Bifrost custom model ID sent when routing via gateway. Leave blank to use the workload-selected model. + {isCustomAIProvider + ? 'Bifrost model ID for this integration. Each custom credential pins one model.' + : 'Bifrost custom model ID sent when routing via gateway. Leave blank to use the workload-selected model.'}

@@ -1505,7 +1516,7 @@ export default function Integrations() { )} - {integrationType === 'ai_provider' && aiProviderWizardStep === 2 && (selectedProvider || selectedAIProvider) && ( + {aiProviderUsesModelsStep && aiProviderWizardStep === 2 && (selectedProvider || selectedAIProvider) && ( - {integrationType === 'ai_provider' && aiProviderWizardStep === 2 ? ( + {aiProviderUsesModelsStep && aiProviderWizardStep === 2 ? ( <> ) : (
)} diff --git a/frontend/src/pages/usage/Usage.tsx b/frontend/src/pages/usage/Usage.tsx index 1ce1a505..1080ee63 100644 --- a/frontend/src/pages/usage/Usage.tsx +++ b/frontend/src/pages/usage/Usage.tsx @@ -1170,42 +1170,35 @@ export default function Usage() { -
- -
-
- {showAudio ? ( - - ) : null} - {showTts ? ( - - ) : null} - {(totals?.cache_read_tokens || 0) > 0 ? ( - - ) : null} - {(totals?.cache_creation_tokens || 0) > 0 ? ( - - ) : null} - {(totals?.reasoning_tokens || 0) > 0 ? ( - - ) : null} -
-
+ {showAudio ? ( + + ) : null} + {showTts ? ( + + ) : null} + {(totals?.cache_read_tokens || 0) > 0 ? ( + + ) : null} + {(totals?.cache_creation_tokens || 0) > 0 ? ( -
+ ) : null} + {(totals?.reasoning_tokens || 0) > 0 ? ( + + ) : null} +
@@ -1429,7 +1422,7 @@ function StatCard({ return (

{label}

diff --git a/frontend/src/pages/usage/UsageFiltersBar.tsx b/frontend/src/pages/usage/UsageFiltersBar.tsx index 83e7e1c1..67dcef0d 100644 --- a/frontend/src/pages/usage/UsageFiltersBar.tsx +++ b/frontend/src/pages/usage/UsageFiltersBar.tsx @@ -109,19 +109,19 @@ export default function UsageFiltersBar({ const activeChips = useMemo((): ActiveChip[] => { const chips: ActiveChip[] = [] - const wsLabel = workspaces.find((w) => w.id === workspaceId)?.name - if (workspaceId && wsLabel) { + if (workspaceId) { + const wsLabel = workspaces.find((w) => w.id === workspaceId)?.name chips.push({ key: 'workspace', - label: wsLabel, + label: wsLabel || 'Workspace', onClear: () => onWorkspaceChange(''), }) } - const importLabel = callImports.find((c) => c.id === callImportId)?.label - if (callImportId && importLabel) { + if (callImportId) { + const importLabel = callImports.find((c) => c.id === callImportId)?.label chips.push({ key: 'call_import', - label: importLabel, + label: importLabel || 'Call import', onClear: () => onCallImportChange(''), }) } @@ -134,22 +134,20 @@ export default function UsageFiltersBar({ } if (tagId) { const tagLabel = tags.find((t) => t.id === tagId)?.label - if (tagLabel) { - chips.push({ - key: 'tag', - label: tagLabel, - onClear: () => onTagChange(''), - }) - } + chips.push({ + key: 'tag', + label: tagLabel || 'Tag', + onClear: () => onTagChange(''), + }) } - const sourceLabel = - sourceOptions.find((e) => e.id === sourceSelectValue)?.label || - evaluations.find((e) => e.id === evaluationId)?.label || - resources.find((r) => r.id === evaluationId)?.label - if ((evaluationId || productSection) && sourceLabel) { + if (evaluationId || productSection) { + const sourceLabel = + sourceOptions.find((e) => e.id === sourceSelectValue)?.label || + evaluations.find((e) => e.id === evaluationId)?.label || + resources.find((r) => r.id === evaluationId)?.label chips.push({ key: 'evaluation', - label: sourceLabel, + label: sourceLabel || 'Source', onClear: () => onEvaluationChange(''), }) } @@ -193,6 +191,16 @@ export default function UsageFiltersBar({ onModelChange, ]) + const hasActiveScope = Boolean( + workspaceId || + callImportId || + dataset || + tagId || + evaluationId || + model || + usageKind || + productSection, + ) const hasScopeFilters = activeChips.length > 0 const kindHint = (kind: Kind): string | undefined => { @@ -229,7 +237,7 @@ export default function UsageFiltersBar({ Updating options… ) : null} - {hasScopeFilters ? ( + {hasActiveScope ? ( @@ -536,7 +568,7 @@ export default function UsagePricing() { isLoading={saveMutation.isPending} onClick={() => saveMutation.mutate()} > - Save override + {editingOverrideId ? 'Update override' : 'Save override'}
@@ -577,7 +609,7 @@ export default function UsagePricing() { Output Audio TTS - + @@ -604,15 +636,25 @@ export default function UsagePricing() { {formatRateUsd(row.rates.tts_per_1m_characters)} - +
+ + +
))} diff --git a/tests/test_services/test_usage/test_llm_usage.py b/tests/test_services/test_usage/test_llm_usage.py index 79c89c40..d08887bd 100644 --- a/tests/test_services/test_usage/test_llm_usage.py +++ b/tests/test_services/test_usage/test_llm_usage.py @@ -24,6 +24,26 @@ from app.services.usage.normalize import UsageSnapshot +def _stub_stamp_cost_params(*_args, **_kwargs) -> Dict[str, Any]: + return { + "input_cost_micro_usd": 0, + "output_cost_micro_usd": 0, + "cache_read_cost_micro_usd": 0, + "cache_creation_cost_micro_usd": 0, + "reasoning_cost_micro_usd": 0, + "audio_cost_micro_usd": 0, + "tts_cost_micro_usd": 0, + "total_cost_micro_usd": 0, + "pricing_rate_source": None, + "pricing_rate_id": None, + } + + +@pytest.fixture(autouse=True) +def _stub_usage_cost_stamp(monkeypatch): + monkeypatch.setattr(usage_mod, "_stamp_cost_params", _stub_stamp_cost_params) + + class _FakePipeline: def __init__(self, client: "_FakeRedis"): self._client = client @@ -202,7 +222,7 @@ def org_ctx(): return org_id, workspace_id, ctx -def test_record_increments_pending_and_counts_zero_token_calls(fake_redis, org_ctx): +def test_record_increments_pending_and_ignores_zero_token_calls(fake_redis, org_ctx): org_id, _workspace_id, ctx = org_ctx with llm_usage_context(ctx): usage_mod.record_llm_usage( @@ -227,7 +247,7 @@ def test_record_increments_pending_and_counts_zero_token_calls(fake_redis, org_c calls = sum(int(v) for k, v in fields.items() if k.endswith("|call_count")) assert prompt == 10 assert completion == 5 - assert calls == 2 + assert calls == 1 assert any(k.rsplit("|", 1)[0].endswith("|llm") for k in fields) @@ -499,10 +519,7 @@ def test_concurrent_flush_does_not_double_count(fake_redis, org_ctx, monkeypatch db_b = MagicMock() db_b.execute.return_value = MagicMock(rowcount=1) monkeypatch.setattr(usage_mod, "_flush_pending_buffer", lambda *_args, **_kwargs: 0) - monkeypatch.setattr( - "app.services.usage.pricing.apply_cost_to_bucket", - lambda *args, **kwargs: False, - ) + monkeypatch.setattr(usage_mod, "_stamp_cost_params", _stub_stamp_cost_params) first = usage_mod.flush_usage_to_catalog(db_a, org_id) second = usage_mod.flush_usage_to_catalog(db_b, org_id) @@ -549,10 +566,7 @@ def test_flush_redis_processes_multiple_batches_in_one_run( db = MagicMock() db.execute.return_value = MagicMock(rowcount=1) monkeypatch.setattr(usage_mod, "_flush_pending_buffer", lambda *_args, **_kwargs: 0) - monkeypatch.setattr( - "app.services.usage.pricing.apply_cost_to_bucket", - lambda *args, **kwargs: False, - ) + monkeypatch.setattr(usage_mod, "_stamp_cost_params", _stub_stamp_cost_params) flushed = usage_mod.flush_usage_to_catalog(db, org_id) @@ -656,10 +670,6 @@ def test_upsert_bucket_sql_uses_valid_empty_jsonb_literal(): def test_upsert_bucket_matches_legacy_resource_context_key(monkeypatch): """Per-row context must merge into an existing evaluation-level bucket.""" - monkeypatch.setattr( - "app.services.usage.pricing.apply_cost_to_bucket", - lambda *args, **kwargs: False, - ) org_id = uuid4() evaluation_id = uuid4() bucket = { @@ -689,7 +699,7 @@ def test_upsert_bucket_matches_legacy_resource_context_key(monkeypatch): assert "context->>'resource_type'" in legacy_sql -def test_upsert_bucket_applies_cost_after_update(monkeypatch): +def test_upsert_bucket_increments_cost_on_update(monkeypatch): org_id = uuid4() bucket = { "workspace_id": uuid4(), @@ -699,18 +709,46 @@ def test_upsert_bucket_applies_cost_after_update(monkeypatch): "usage_date": date(2026, 8, 12), "usage_kind": "llm", } - deltas = {"prompt_tokens": 10, "completion_tokens": 5} + deltas = {"prompt_tokens": 10, "completion_tokens": 5, "call_count": 1} db = MagicMock() db.execute.return_value = MagicMock(rowcount=1) - called: list[bool] = [] monkeypatch.setattr( - "app.services.usage.pricing.apply_cost_to_bucket", - lambda *args, **kwargs: called.append(True) or True, + usage_mod, + "_stamp_cost_params", + lambda *args, **kwargs: { + "input_cost_micro_usd": 100, + "output_cost_micro_usd": 200, + "cache_read_cost_micro_usd": 0, + "cache_creation_cost_micro_usd": 0, + "reasoning_cost_micro_usd": 0, + "audio_cost_micro_usd": 0, + "tts_cost_micro_usd": 0, + "total_cost_micro_usd": 300, + "pricing_rate_source": "catalog", + "pricing_rate_id": str(uuid4()), + }, ) usage_mod._upsert_bucket(db, org_id, bucket, deltas) - assert called + update_sql = str(db.execute.call_args[0][0]) + assert "input_cost_micro_usd = input_cost_micro_usd + :input_cost_micro_usd" in update_sql + assert "total_cost_micro_usd = total_cost_micro_usd + :total_cost_micro_usd" in update_sql + + +def test_record_llm_usage_skips_zero_token_snapshot(fake_redis, monkeypatch): + org_id = uuid4() + monkeypatch.setattr(usage_mod, "_client", lambda: fake_redis) + usage_mod.record_llm_usage( + "gpt-test", + UsageSnapshot(prompt_tokens=0, completion_tokens=0), + organization_id=org_id, + ctx=LLMUsageContext( + organization_id=org_id, + product_section=LLMUsageProductSection.CHAT, + ), + ) + assert not fake_redis.hgetall(usage_mod._pending_hash_key(org_id)) def test_orphan_recovery_runs_at_most_once_per_interval(fake_redis, monkeypatch): diff --git a/tests/test_services/test_usage/test_pricing.py b/tests/test_services/test_usage/test_pricing.py index 6d01dcaa..eaf63c35 100644 --- a/tests/test_services/test_usage/test_pricing.py +++ b/tests/test_services/test_usage/test_pricing.py @@ -193,9 +193,11 @@ def test_resolve_rate_ignores_stale_negative_cache(monkeypatch): override_result.mappings.return_value.first.return_value = None catalog_result = MagicMock() catalog_result.mappings.return_value.first.return_value = row + cache_table_exists = MagicMock() + cache_table_exists.scalar.return_value = "model_pricing_rates" db = MagicMock() - db.execute.side_effect = [override_result, catalog_result] + db.execute.side_effect = [override_result, cache_table_exists, catalog_result] resolver = PricingResolver(db) card = resolver.resolve_rate( From 49871cbfc22e3afb902d954f29fb729125c6ab99 Mon Sep 17 00:00:00 2001 From: M Sami Date: Tue, 18 Aug 2026 00:09:12 +0530 Subject: [PATCH 31/32] fix(usage): implement _cost_fields_for_pending_deltas function to streamline cost calculations for pending usage deltas --- app/services/ai/llm_service.py | 16 -------- app/services/usage/llm_usage.py | 68 ++++++++++++++++++++++++++++----- 2 files changed, 59 insertions(+), 25 deletions(-) diff --git a/app/services/ai/llm_service.py b/app/services/ai/llm_service.py index a2ea3801..95e64488 100644 --- a/app/services/ai/llm_service.py +++ b/app/services/ai/llm_service.py @@ -490,22 +490,6 @@ def generate_response( credential=credential_ctx, ) - # #region agent log - try: - import json as _json, time as _time - _msg_types = [] - for _m in messages: - _c = _m.get("content") - if isinstance(_c, list): - _msg_types.extend( - p.get("type") for p in _c if isinstance(p, dict) - ) - with open("debug-bfc313.log", "a", encoding="utf-8") as _f: - _f.write(_json.dumps({"sessionId": "bfc313", "runId": "post-fix", "hypothesisId": "C", "location": "llm_service.py:generate_response", "message": "pre-completion routing", "data": {"effective_routing": effective_routing, "credential_mode": getattr(credential_ctx, "routing_mode", None), "credential_id": str(credential_id) if credential_id else None, "model": model_str, "has_api_base": "api_base" in call_kwargs, "custom_llm_provider": call_kwargs.get("custom_llm_provider"), "content_part_types": _msg_types}, "timestamp": int(_time.time() * 1000)}) + "\n") - except Exception: - pass - # #endregion - try: response = litellm.completion(**call_kwargs) except Exception as e: diff --git a/app/services/usage/llm_usage.py b/app/services/usage/llm_usage.py index dea0c400..bbeac232 100644 --- a/app/services/usage/llm_usage.py +++ b/app/services/usage/llm_usage.py @@ -312,6 +312,63 @@ def _deltas_from_tts(characters: int) -> Dict[str, int]: } +def _cost_fields_for_pending_deltas( + organization_id: UUID, + bucket: Dict[str, Any], + deltas: Dict[str, int], + *, + db: Optional[Session] = None, +) -> Dict[str, Any]: + if not _has_billable_usage_deltas(deltas): + return {} + try: + from app.services.usage.pricing import cost_fields_from_deltas + except Exception as exc: + logger.debug("usage pending cost stamp skipped: {}", exc) + return {} + + usage_date = bucket["usage_date"] + if isinstance(usage_date, str): + usage_date = date.fromisoformat(usage_date) + usage_kind = bucket.get("usage_kind") or USAGE_KIND_LLM + + if db is not None: + try: + return cost_fields_from_deltas( + deltas, + organization_id=organization_id, + model=bucket["model"], + usage_kind=usage_kind, + usage_date=usage_date, + db=db, + ) + except Exception as exc: + logger.debug("usage pending cost stamp failed: {}", exc) + return {} + + try: + from app.database import SessionLocal + except Exception as exc: + logger.debug("usage pending cost stamp skipped: {}", exc) + return {} + + session = SessionLocal() + try: + return cost_fields_from_deltas( + deltas, + organization_id=organization_id, + model=bucket["model"], + usage_kind=usage_kind, + usage_date=usage_date, + db=session, + ) + except Exception as exc: + logger.debug("usage pending cost stamp failed: {}", exc) + return {} + finally: + session.close() + + def _buffer_to_postgres( organization_id: UUID, bucket: Dict[str, Any], @@ -347,16 +404,9 @@ def _buffer_to_postgres( } db = SessionLocal() try: - from app.services.usage.pricing import cost_fields_from_deltas - params.update( - cost_fields_from_deltas( - deltas, - organization_id=organization_id, - model=bucket["model"], - usage_kind=usage_kind, - usage_date=usage_date, - db=db, + _cost_fields_for_pending_deltas( + organization_id, bucket, deltas, db=db ) ) db.execute( From a33972d61db06b3ea6aa5b11f053e9c41da4039d Mon Sep 17 00:00:00 2001 From: M Sami Date: Tue, 18 Aug 2026 19:12:13 +0530 Subject: [PATCH 32/32] feat(docs): add usage tracking feature documentation and update related references in the documentation --- docs-fumadocs/content/docs/intro.mdx | 2 + .../content/docs/monitoring/meta.json | 1 + .../content/docs/monitoring/usage.mdx | 198 ++++++++++++++++++ .../content/docs/reference/cli-commands.mdx | 47 +++++ .../content/feature-contributors.json | 17 ++ docs-fumadocs/public/search-index.json | 139 +++++++----- 6 files changed, 357 insertions(+), 47 deletions(-) create mode 100644 docs-fumadocs/content/docs/monitoring/usage.mdx diff --git a/docs-fumadocs/content/docs/intro.mdx b/docs-fumadocs/content/docs/intro.mdx index 36d2e905..f95f6f49 100644 --- a/docs-fumadocs/content/docs/intro.mdx +++ b/docs-fumadocs/content/docs/intro.mdx @@ -33,6 +33,7 @@ EfficientAI gives you an end-to-end loop for voice AI quality: - **Metric-driven evaluation**: built-in and custom metrics with categorization labels, surface targeting, and org-wide or workspace scope. - **Cloud storage**: store audio in Amazon S3, S3-compatible services, or Google Cloud Storage via the Data Sources UI. - **Prompt optimization workflows**: run optimization loops, compare candidates, accept winners, and push selected prompts to providers. +- **Usage tracking**: org-wide LLM, STT, and TTS consumption with cost estimates and drill-down by workspace and product area. ## Platform model @@ -48,6 +49,7 @@ This structure keeps testing reproducible while still reflecting real-world voic ## Key guides +- [Usage](/docs/monitoring/usage/) — LLM/STT/TTS consumption, cost estimates, and drill-down by workspace and product area - [Workspaces](/docs/getting-started/workspaces/) — project isolation within your organization, plus workspace roles (Viewer / Editor / Workspace Admin) and how they interact with org roles - [Cloud Storage](/docs/getting-started/cloud-storage/) — S3 and GCS configuration - [Metrics](/docs/products/metrics/) — custom rubrics, surfaces, and evaluation scope diff --git a/docs-fumadocs/content/docs/monitoring/meta.json b/docs-fumadocs/content/docs/monitoring/meta.json index 2ace091c..e0a482ad 100644 --- a/docs-fumadocs/content/docs/monitoring/meta.json +++ b/docs-fumadocs/content/docs/monitoring/meta.json @@ -1,6 +1,7 @@ { "title": "Monitoring", "pages": [ + "usage", "calls", "alerting", "cron-jobs" diff --git a/docs-fumadocs/content/docs/monitoring/usage.mdx b/docs-fumadocs/content/docs/monitoring/usage.mdx new file mode 100644 index 00000000..1c2cb701 --- /dev/null +++ b/docs-fumadocs/content/docs/monitoring/usage.mdx @@ -0,0 +1,198 @@ +--- +id: usage +title: Usage +sidebar_position: 1 +--- + +# Usage + +**Usage** is org-scoped analytics for LLM, STT, and TTS consumption with estimated costs. It helps you see how much each workspace, product area, call import, and model is consuming — and what that usage likely costs. + +Usage is **not** a quota or billing portal. Optional Flexprice event metering is a separate system and is not shown in the Usage UI. + +Models and providers tracked here come from your enabled [integrations](/docs/getting-started/integrations/). Usage is attributed per [workspace](/docs/getting-started/workspaces/) when the underlying workflow is workspace-scoped. + +--- + +## Opening the Usage page + +In the sidebar, go to **Usage → Overview** (`/usage`). + +The page has two tabs when your org is licensed for enterprise features and you are an org admin: + +| Tab | URL | Who can access | +|-----|-----|----------------| +| **Overview** | `/usage` | All org members | +| **Pricing overrides** | `/usage?tab=pricing` | Org admins with enterprise license | + +--- + +## Summary metrics + +The top of the Overview tab shows rollup cards for the selected date range and filters: + +| Metric | Description | +|--------|-------------| +| **Input tokens** | Prompt or input tokens sent to LLM providers | +| **Output tokens** | Completion or output tokens returned by LLMs | +| **Total tokens** | Sum of input and output tokens | +| **LLM calls** | Number of LLM API calls | +| **STT audio** | Speech-to-text audio duration (shown when non-zero) | +| **TTS characters** | Text-to-speech characters synthesized (shown when non-zero) | +| **Cache read** | Tokens read from provider prompt cache (shown when non-zero) | +| **Cache write** | Tokens written to provider prompt cache (shown when non-zero) | +| **Reasoning** | Reasoning tokens billed separately by some providers (shown when non-zero) | +| **Estimated cost** | Total estimated cost for the filtered range | + +Click **Cost breakdown** to open a modal with line items: + +- Input, output, cache read, cache write, reasoning +- Audio (STT), TTS +- Total estimated cost + +If any usage in the range has no matching catalog rate, the breakdown shows an **unpriced usage** warning. Those rows still appear in token/volume metrics but do not contribute to cost totals. + +### Currency display + +Toggle between **USD** and **INR** in the filter bar. INR amounts use a live USD→INR rate from Frankfurter when available, with a fallback estimate when the FX service is unreachable. + +--- + +## Filters + +Filters narrow the summary cards and drill-down table. All filter state is stored in the URL, so you can bookmark or share a specific view. + +| Filter | Description | +|--------|-------------| +| **Date range** | Start and end dates, interpreted in your browser's IANA timezone | +| **Workspace** | Limit to one workspace | +| **Call import** | Limit to one call import batch | +| **Dataset** | Filter by dataset name on call import rows | +| **Tag** | Filter by call import tag | +| **Evaluation run** | Filter by evaluation resource | +| **Usage kind** | `LLM`, `STT`, or `TTS` | +| **Model** | Provider model identifier | +| **Source / product section** | Product area (playground, evaluators, call imports, etc.) | + +--- + +## Drill-down navigation + +Click rows in the breakdown table to drill deeper. Breadcrumbs at the top show your current path; click a breadcrumb to go back up. + +```mermaid +flowchart TD + Org[Organization] --> Workspace[Workspace] + Workspace --> Composite[Call imports and product areas] + Composite --> CallImport[Call import batch] + CallImport --> EvalRun[Evaluation run] + EvalRun --> Model[Model] + Model --> Kind[Usage kind] + Composite --> ProductSection[Product section] + ProductSection --> Model +``` + +At the organization level, the table groups by **workspace**. Inside a workspace you see a composite view: + +- **Call import batches** — CSV uploads or manual audio recordings +- **Product areas** — usage from other parts of the platform (not tied to a single call import) + +From a call import batch you can drill into evaluation runs, then model, then usage kind. From a product area you drill into model, then usage kind. + +Each drill level returns at most **100 rows**. If more exist, results are truncated at that level. + +### Product sections + +| Section | What it tracks | +|---------|----------------| +| **Call imports** | Call import batch processing | +| **Call import evaluations** | Evaluations run on imported calls | +| **Playground** | Text playground and experiments | +| **Voice playground** | Voice agent playground — LLM, STT, and TTS | +| **Chat** | Chat conversations | +| **Telephony** | Telephony and live calls | +| **Evaluators** | Evaluator definitions and runs | +| **Metrics** | Metrics and scoring | +| **Judge alignment** | Judge alignment workflows | +| **Prompt optimization** | Prompt optimization jobs | +| **Personas** | Persona generation | +| **Agents** | Agent configuration | +| **Prompt partials** | Prompt partials | +| **Conversation evaluations** | Conversation evaluations | +| **Test agent** | Test agent sessions | +| **Other** | Usage not attributed to a named product area | + +--- + +## Data freshness and history + +### Freshness + +Usage counters are buffered in Redis and flushed to Postgres by the `worker-usage` service on a Celery Beat schedule (default: every **2 minutes**). The UI reads Postgres only. + +The page shows **Updated** with a `last_updated_at` timestamp when available. Expect roughly **2 minutes** of lag between new API usage and what appears on this page. + +For fresh data in self-hosted deployments, ensure `beat`, `worker-usage`, and the default `worker` are running (or use `eai start-all`). + +### History limits + +| Deployment | History window | Pricing overrides tab | +|------------|----------------|----------------------| +| **OSS** (no license) | Last **7 days** | Hidden | +| **Enterprise** (`EFFICIENTAI_LICENSE`) | Unlimited | Org admins only | + +On OSS deployments, an amber banner explains the 7-day cap and points to `EFFICIENTAI_LICENSE`. If you pick a wider date range, it is automatically clamped to the allowed window. + +See [Configuration](/docs/reference/configuration/) for license setup. + +--- + +## Pricing overrides + +Enterprise org admins can open **Pricing overrides** (`/usage?tab=pricing`) to set per-model rates that override the built-in catalog for cost estimation. + +### Override fields by usage kind + +| Usage kind | Rate fields (USD) | +|------------|-------------------| +| **LLM** | Input / 1M tokens, output / 1M tokens, cache read / 1M, cache write / 1M, reasoning / 1M, audio / minute | +| **STT** | Audio / minute | +| **TTS** | Characters / 1M | + +For each override you set: + +- **Provider credential** — models come from enabled integrations +- **Usage kind** — LLM, STT, or TTS +- **Model** — provider model identifier +- **Effective from** — date the override starts applying +- **Rates** — USD values for the fields above + +You can prefill rates from the catalog or an existing override. Saving creates or updates the override; deleting removes it. + +### How overrides affect costs + +Overrides apply to **new usage** recorded on or after `effective_from`. Costs already stamped on daily rollup rows are **not** changed automatically. + +To backfill historical costs after a catalog or override change, use the CLI or API recompute workflow. The Usage UI does not expose recompute jobs today — see [CLI Commands](/docs/reference/cli-commands/#usage-pricing). + +--- + +## Operations + +Self-hosted operators manage pricing catalogs and cost backfills outside the UI. + +| Requirement | Purpose | +|-------------|---------| +| `beat` | Schedules usage flush, FX refresh, OSS history prune | +| `worker-usage` | Flushes Redis counters and runs cost recompute jobs | +| Default `worker` | Evaluator cron dispatch (indirectly drives much platform usage) | + +Common tuning variables (see `env.example`): + +| Variable | Default | Purpose | +|----------|---------|---------| +| `USAGE_FLUSH_BEAT_SECONDS` | `120` | Flush interval (~2 min UI lag) | +| `USAGE_READ_CACHE_TTL_SECONDS` | `90` | Redis cache TTL for summary/breakdown/filters | +| `USAGE_FLUSH_MAX_BATCHES_PER_RUN` | `30` | Batches per flush tick | + +For seeding rates, diffing catalogs, and recomputing stored costs, see [CLI Commands — Usage pricing](/docs/reference/cli-commands/#usage-pricing). diff --git a/docs-fumadocs/content/docs/reference/cli-commands.mdx b/docs-fumadocs/content/docs/reference/cli-commands.mdx index 6e09c83a..c0ce1658 100644 --- a/docs-fumadocs/content/docs/reference/cli-commands.mdx +++ b/docs-fumadocs/content/docs/reference/cli-commands.mdx @@ -107,3 +107,50 @@ eai migrate --verbose ``` **Note**: Migrations run automatically on application startup. You only need to run them manually if you want to apply migrations before starting the server. + +## Usage pricing + +Manage model pricing rates and backfill stored usage costs on `llm_usage_daily` rollups. Requires `beat`, `worker-usage`, and the default `worker` (or `eai start-all`). + +```bash +# Upsert model_pricing_rates from app/config/models.json +eai usage seed-rates --config config.yml + +# Compare models.json pricing vs Postgres +eai usage diff-rates --config config.yml + +# Backfill costs in-process (all orgs; use after migrate or catalog change) +eai usage recompute --config config.yml --sync + +# Async recompute via usage queue (requires --organization-id) +eai usage recompute --config config.yml --organization-id + +# Optional scopes: --model, --usage-kind, --start-date, --end-date + +# Optional: fetch LiteLLM prices into pricing_catalog.json +eai usage sync-litellm --local +eai usage sync-litellm --local --write-models +``` + +**After migrations or catalog changes:** + +```bash +eai migrate +eai usage seed-rates --config config.yml +eai usage recompute --config config.yml --sync +``` + +**Flush / Usage UI tuning** — set in `.env` (see `env.example`): + +| Variable | Default | Purpose | +|----------|---------|---------| +| `USAGE_FLUSH_BUCKET_BATCH_SIZE` | `500` | Buckets per DB transaction | +| `USAGE_FLUSH_MAX_BATCHES_PER_RUN` | `30` | Batches per flush tick (≤ **15,000** buckets/run) | +| `USAGE_FLUSH_BEAT_SECONDS` | `120` | Celery Beat flush interval (~2 min lag vs Redis) | +| `USAGE_FLUSH_LOCK_TTL_SECONDS` | `300` | Per-org flush lock TTL | +| `USAGE_READ_CACHE_TTL_SECONDS` | `90` | Redis cache TTL for usage summary/breakdown/filters | +| `CRON_DISPATCH_INTERVAL_SECONDS` | `30` | Evaluator cron dispatcher tick (default worker) | + +The Usage UI reads Postgres only (summary/breakdown/filters). Redis counters flush on the Celery Beat schedule (~2 min eventual consistency). If Redis backlog grows, lower `USAGE_FLUSH_BEAT_SECONDS` or raise `USAGE_FLUSH_MAX_BATCHES_PER_RUN`. + +See [Usage](/docs/monitoring/usage/) for the end-user guide. diff --git a/docs-fumadocs/content/feature-contributors.json b/docs-fumadocs/content/feature-contributors.json index 28ca6227..e7990cc8 100644 --- a/docs-fumadocs/content/feature-contributors.json +++ b/docs-fumadocs/content/feature-contributors.json @@ -287,6 +287,23 @@ ], "lastReviewed": "2026-06-15" }, + { + "featureId": "monitoring/usage", + "docPath": "docs-fumadocs/content/docs/monitoring/usage.mdx", + "historyPath": "docs-fumadocs/content/docs/monitoring/usage.mdx", + "owners": [ + "aadhar-EAI", + "Tejas Narayan" + ], + "contributors": [ + { + "name": "aadhar-EAI", + "email": "aadhar@efficientai.cloud", + "commits": 1 + } + ], + "lastReviewed": "2026-08-18" + }, { "featureId": "monitoring/cron-jobs", "docPath": "docs-fumadocs/content/docs/monitoring/cron-jobs.mdx", diff --git a/docs-fumadocs/public/search-index.json b/docs-fumadocs/public/search-index.json index e2d517c7..2d7daadc 100644 --- a/docs-fumadocs/public/search-index.json +++ b/docs-fumadocs/public/search-index.json @@ -1,19 +1,28 @@ { - "generatedAt": "2026-06-13T19:35:24.170Z", - "count": 30, + "generatedAt": "2026-08-18T13:39:50.745Z", + "count": 35, "records": [ { "id": "advanced/architecture", - "url": "/docs/advanced/architecture", + "url": "/docs/advanced/architecture/", "title": "Architecture", "breadcrumbs": [ "Advanced" ], "content": "Architecture Simple Overview EfficientAI is built like a modern web application. The Brain (API Server) : Controls everything. The Worker : Does the heavy lifting in the background, like processing audio files so the website stays fast. The Interface (Frontend) : The website you see and click on. The Memory (Database) : Where we store all your agents, test results, and user data. Technical Deep Dive EfficientAI is built as a modular, containerized application designed for scalability and extensibility. (See original Architecture documentation below) System Components The platform consists of four primary components: 1. API Server (FastAPI) : The central control plane. 2. Worker (Celery) : Handles asynchronous background tasks (transcription, evaluation). 3. Frontend (React/Vite) : The user interface. 4. Data Stores : PostgreSQL (State) and Redis (Queue/Cache). Core Services 1. API Server ( ) Built with FastAPI, it provides REST endpoints for: Resource Management : CRUD for Agents, Personas, Scenarios. Orchestration : Real time control of test conversations. Analysis : Serving evaluation results and dashboards. 2. Asynchronous Workers ( ) Powered by Celery and Redis, the workers handle long running operations: Transcription : Processing audio files (using Whisper, Deepgram, etc.). Evaluation : Running metric calculations (WER/CER) on completed conversations. 3. Test Agent Service ( ) This is the heart of the testing engine. It: Manages the state of the conversation. Generates accurate system prompts for the Persona. Handles the latency sensitive loop of: ." }, + { + "id": "advanced/call-import-sharding", + "url": "/docs/advanced/call-import-sharding/", + "title": "Call Import Sharding", + "breadcrumbs": [ + "Advanced" + ], + "content": "Call Import Sharding For large batches (10k+ rows), call import row data can be spread across multiple PostgreSQL data shards with a catalog database for metadata, routing, and parent counters. Architecture Catalog DB — , , shard slice registry, dispatch metadata Data shards — , (partitioned by consistent hash on ) Scatter/gather reads — API and workers query each shard and merge results Fair dispatch — import/eval workers respect per shard pending scans and Redis progress keys Enable sharding in under . See and for profiles. Live telephony / evaluator results use a separate payload sharding path ( , ) keyed by workspace — see in the repo. Operations Connection pools When , use smaller per process pools ( 3–5, 5–10) so API + workers × shards stay under Postgres . PgBouncer (optional) Point , , and each shard at PgBouncer ( ) in transaction pooling mode. Keep SQLAlchemy enabled. Observability Row Celery tasks log on the import worker hot path. Watch Redis eval/import progress keys ( , ) alongside catalog parent counters. Rebalance Dry run registry updates: Use only after pausing the import. The rebalance tool copies rows to the target shard, updates the catalog registry, then removes copies from the source shard. Troubleshooting stalled evaluations If evaluation runs stall after recordings import (rows show import but no diarization/scoring): 1. Restart API and after deploying fixes. 2. Retry evaluation from the UI (use Overwrite existing transcripts if diarization previously failed). 3. Ensure the imports worker consumes ." + }, { "id": "advanced/database", - "url": "/docs/advanced/database", + "url": "/docs/advanced/database/", "title": "Database", "breadcrumbs": [ "Advanced" @@ -22,7 +31,7 @@ }, { "id": "advanced/development", - "url": "/docs/advanced/development", + "url": "/docs/advanced/development/", "title": "Development & Troubleshooting", "breadcrumbs": [ "Advanced" @@ -31,25 +40,25 @@ }, { "id": "getting-started/authentication", - "url": "/docs/getting-started/authentication", + "url": "/docs/getting-started/authentication/", "title": "Authentication", "breadcrumbs": [ "Getting Started" ], - "content": "🔐 Authentication EfficientAI ships with a pluggable authentication system that scales from a single operator OSS install to an enterprise deployment behind your existing identity provider. You pick the providers you want in (or via in ) and the API/frontend adapt automatically. Deployment models | Model | Providers | License needed | | | | | | OSS self hosted (default) | , | None | | Enterprise SSO (BYO IdP) | , | | — the header, always available, for programmatic access (CI pipelines, SDKs, scripts). — email + password, verified against the local users table, returns an app signed HS256 Bearer token. Enabled by default. — license gated. Verifies a Bearer JWT issued by your OIDC compliant IdP (Okta, Azure AD / Entra ID, Google Workspace, AWS Cognito, Auth0, Ping, JumpCloud, OneLogin, …) against the issuer's JWKS. :::info Why no bundled IdP? In practice every enterprise already runs one. Shipping our own Keycloak alongside the app just added another thing for you to operate and lock down. talks to whatever you already have. ::: Self hosted (OSS) This is the default after or . No license, no IdP, no external dependencies. The equivalent environment variables (for Docker Compose / ): First time bootstrap 1. Start the stack. 2. Open and click Create account on the login screen. The first user you create becomes the admin of a fresh organization. 3. Mint an API key from Profile → API Keys (or via ) for programmatic access. Password login (email + password) When is enabled, the login screen shows a Sign in and (if ) a Create account tab. Signing up provisions a new user and a new organization, and makes that user the of it. If you leave the organization name blank, the server derives one from the email's local part. Once signed in, the SPA holds a short lived Bearer token and silently re authenticates before it expires. You can change the token lifetime with — the default is 12 hours. Linking a password to an API key only account If you bootstrapped with , the backend provisions a placeholder user behind that key (its email ends in ). You can upgrade this identity to a real email + password login so you can sign in interactively with the same user. Do it from Profile → Sign in Password while signed in via the API key — the page detects the placeholder email and prompts you to pick a real one and a password. After saving, the same user can log in either with the original API key (for machines) or with email + password (for humans). Rules the UI enforces: If the user already has a password, the form asks for the current one before accepting a new one. You can only set the email from that screen while it's still the placeholder address; \"real\" users change their email from the main profile edit flow. Hardening before you expose it to the internet Turn off self service signup once your team is in: Rotate to invalidate existing sessions. Put the app behind a reverse proxy (Nginx, Caddy, Cloudflare) that terminates TLS and enforces HSTS. Restrict to the exact domain(s) serving the SPA. Team management: invitations & organizations EfficientAI is multi tenant from the ground up. Every piece of data is scoped to an organization , a user can be a member of more than one organization, and each membership has a role that controls what they can do. Roles | Role | Can do | | | | | | Read only access to everything in the org. | | | Everything a reader can + create/update/delete most resources. | | | Everything a writer can + manage users, invitations, roles, API keys, and org settings. | The role is stored per membership, so the same user can be an in one org and a in another. Inviting a teammate Admins invite teammates from Settings → Team . An invitation captures an email and a role and stays valid for 7 days. From the same page, admins can also: See the current members of the org and change their role (with a guard so you can't demote the last admin). Remove a member from the organization. Revoke a pending invitation. Delivery. The backend cre" + "content": "🔐 Authentication EfficientAI ships with a pluggable authentication system that scales from a single operator OSS install to an enterprise deployment behind your existing identity provider. You pick the providers you want in (or via in ) and the API/frontend adapt automatically. Deployment models | Model | Providers | License needed | | | | | | OSS self hosted (default) | , | None | | Enterprise SSO (BYO IdP) | , | | — the header, always available, for programmatic access (CI pipelines, SDKs, scripts). — email + password, verified against the local users table, returns an app signed HS256 Bearer token. Enabled by default. — license gated. Verifies a Bearer JWT issued by your OIDC compliant IdP (Okta, Azure AD / Entra ID, Google Workspace, AWS Cognito, Auth0, Ping, JumpCloud, OneLogin, …) against the issuer's JWKS. :::info Why no bundled IdP? In practice every enterprise already runs one. Shipping our own Keycloak alongside the app just added another thing for you to operate and lock down. talks to whatever you already have. ::: Self hosted (OSS) This is the default after or . No license, no IdP, no external dependencies. The equivalent environment variables (for Docker Compose / ): First time bootstrap 1. Start the stack. 2. Open and click Create account on the login screen. The first user you create becomes the admin of a fresh organization. 3. Mint an API key from Profile → API Keys (or via ) for programmatic access. Password login (email + password) When is enabled, the login screen shows a Sign in and (if ) a Create account tab. Signing up provisions a new user and a new organization, and makes that user the of it. If you leave the organization name blank, the server derives one from the email's local part. Once signed in, the SPA holds a short lived access token (15 minutes by default) plus a refresh token. The client silently refreshes the access token before it expires. You can change lifetimes with and . Logout revokes the refresh token and blacklists the current access token server side. Linking a password to an API key only account If you bootstrapped with , the backend provisions a placeholder user behind that key (its email ends in ). You can upgrade this identity to a real email + password login so you can sign in interactively with the same user. Do it from Profile → Sign in Password while signed in via the API key — the page detects the placeholder email and prompts you to pick a real one and a password. After saving, the same user can log in either with the original API key (for machines) or with email + password (for humans). Rules the UI enforces: If the user already has a password, the form asks for the current one before accepting a new one. You can only set the email from that screen while it's still the placeholder address; \"real\" users change their email from the main profile edit flow. Hardening before you expose it to the internet Turn off self service signup once your team is in: Rotate to invalidate existing sessions. Put the app behind a reverse proxy (Nginx, Caddy, Cloudflare) that terminates TLS and enforces HSTS. The bundled FastAPI server sends baseline security headers on all responses ( , , , , and by default). If you terminate traffic at an external reverse proxy, keep those headers (or stricter CSP ) enabled there too. After reviewing CSP violation reports, set to enforce the policy. Pin third party observability container images to fixed tags and rebuild external reverse proxy images on patched runtimes. If a scanner reports a Go stdlib CVE in a binary this repo does not build, identify the flagged container or proxy artifact and upgrade it separately. Restrict to the exact domain(s) serving the SPA. Set in production so , , and are not served on the public hostname. Keep so and return 404 to anonymous public clients (including vulnerability scanners hitting your ALB hostname). AWS ALB target health checks connect directly from VPC addresses (no ); include your VPC/LB CIDRs in . Full migrat" }, { "id": "getting-started/cloud-storage", - "url": "/docs/getting-started/cloud-storage", + "url": "/docs/getting-started/cloud-storage/", "title": "Cloud Storage", "breadcrumbs": [ "Getting Started" ], - "content": "Cloud Storage Overview EfficientAI can store audio files and recordings in the cloud using one active blob backend at a time: Amazon S3 (or any S3 compatible service such as MinIO, DigitalOcean Spaces, Cloudflare R2) Google Cloud Storage (GCS) Cloud storage is useful for: Storing large audio files outside your application server Scaling storage independently from compute Integrating with existing AWS or GCP infrastructure Browsing, uploading, and managing audio from the Data Sources page in the UI Select the backend with in ( or ), then enable and configure the matching block below. Storage block | Option | Description | | | | | | Local fallback directory for uploads when cloud storage is disabled | | | Maximum upload size in megabytes | | | Active cloud backend: or | | | File extensions accepted for audio uploads | Amazon S3 and S3 compatible storage Configuration Set and enable the block: | Option | Required | Description | | | | | | | Yes | Set to to enable S3 storage | | | Yes | Name of your S3 bucket | | | Yes | AWS region (e.g., , ) | | | Yes | AWS Access Key ID | | | Yes | AWS Secret Access Key | | | No | Custom endpoint for S3 compatible services | | | No | Folder prefix for uploaded files (default: ) | S3 compatible examples MinIO DigitalOcean Spaces Cloudflare R2 Google Cloud Storage Configuration Set and enable the block: | Option | Required | Description | | | | | | | Yes | Set to to enable GCS storage | | | Yes | Name of your GCS bucket | | | Yes | GCP project ID | | | No | Path to a service account JSON key file | | | No | Folder prefix for uploaded files (default: ) | Authentication EfficientAI resolves GCS credentials in this order: 1. in (relative paths resolve from the working directory) 2. environment variable 3. Application Default Credentials (ADC) on GCE, GKE, or Cloud Run The client is included in the standard EfficientAI install ( ); no extra Python package step is required for GCS. GCP setup 1. Create a GCS bucket in your project (Console or ). 2. Create a service account with object read/write access (e.g., on the bucket, or a tighter custom role). 3. Download a JSON key and set , or mount the key and set . 4. Set , , and fill in and . Object layout under matches S3 (organization scoped paths), so you can migrate between providers without changing application logic. Data Sources UI Once cloud storage is configured and connected, open Configuration → Data Sources to: Browse folders and audio files in your bucket Upload new audio files Preview playback in the browser Delete files Test the connection (labels show Amazon S3 or Google Cloud Storage based on ) Verifying connection Restart the application after changing storage settings: Use Test Connection on the Data Sources page, or upload a test file and confirm it appears under the configured in your bucket. Troubleshooting | Issue | Solution | | | | | (S3) | Check IAM permissions and bucket policy | | (S3) | Verify bucket name and region | | (S3) | Check for S3 compatible services | | (S3) | Verify and | | GCS | Confirm service account has on the bucket | | GCS bucket not found | Verify , , and that the bucket exists | | GCS auth failure | Set or ; on GCP VMs you can use ADC | | Wrong provider in UI | Ensure matches the enabled block ( vs ) |" + "content": "Cloud Storage Overview EfficientAI can store audio files and recordings in the cloud using one active blob backend at a time: Amazon S3 (or any S3 compatible service such as MinIO, DigitalOcean Spaces, Cloudflare R2) Google Cloud Storage (GCS) Azure Blob Storage Cloud storage is useful for: Storing large audio files outside your application server Scaling storage independently from compute Integrating with existing AWS, GCP, or Azure infrastructure Browsing, uploading, and managing audio from the Data Sources page in the UI Select the backend with in ( , , or ), then enable and configure the matching block below. Storage block | Option | Description | | | | | | Local fallback directory for uploads when cloud storage is disabled | | | Maximum upload size in megabytes | | | Active cloud backend: , , or | | | File extensions accepted for audio uploads | Amazon S3 and S3 compatible storage Configuration Set and enable the block: | Option | Required | Description | | | | | | | Yes | Set to to enable S3 storage | | | Yes | Name of your S3 bucket | | | Yes | AWS region (e.g., , ) | | | Yes | AWS Access Key ID | | | Yes | AWS Secret Access Key | | | No | Custom endpoint for S3 compatible services | | | No | Folder prefix for uploaded files (default: ) | S3 compatible examples MinIO DigitalOcean Spaces Cloudflare R2 Google Cloud Storage Configuration Set and enable the block: | Option | Required | Description | | | | | | | Yes | Set to to enable GCS storage | | | Yes | Name of your GCS bucket | | | Yes | GCP project ID | | | No | Path to a service account JSON key file | | | No | Override service account email for signed URL generation when ADC does not expose it | | | No | Folder prefix for uploaded files (default: ) | Authentication EfficientAI resolves GCS credentials in this order: 1. in (relative paths resolve from the working directory) 2. environment variable 3. Application Default Credentials (ADC) on GCE, GKE, or Cloud Run Uploads and server side downloads work with ADC alone (including GKE Workload Identity ). Signed URLs for browser playback require either a service account JSON key with a private key, or Workload Identity plus IAM signBlob (see below). The client is included in the standard EfficientAI install ( ); no extra Python package step is required for GCS. GCP setup 1. Create a GCS bucket in your project (Console or ). 2. Create a service account with object read/write access (e.g., on the bucket, or a tighter custom role). 3. Authenticate the app using one of: GKE Workload Identity (recommended): bind your Kubernetes service account to the GCP service account. Enable the and grant the GCP service account on itself so EfficientAI can sign playback URLs without a JSON key. Service account JSON key: download a key and set , or mount the key and set . 4. Set , , and fill in and . Example IAM binding for Workload Identity signed URLs: If ADC does not expose the service account email (some federation setups), set to the workload GCP service account email. Object layout under matches S3 (organization scoped paths), so you can migrate between providers without changing application logic. Azure Blob Storage Configuration Set and enable the block: | Option | Required | Description | | | | | | | Yes | Set to to enable Azure Blob Storage | | | Yes | Name of your Azure storage container | | | Yes | Azure storage account name ( not required if is set) | | | Yes | Storage account access key ( not required if is set) | | | No | Full Azure storage connection string (overrides + ) | | | No | Folder prefix for uploaded files (default: ) | Authentication EfficientAI resolves Azure credentials in this order: 1. in 2. + Managed Identity is not supported in this release; use a connection string or account key. SAS URLs (temporary download links in the UI) require an account key. Connection strings include the key automatically. Azure setup 1. Create a Storage Account in the Azure Portal (or via ). 2. Create a container (e.g., ) under tha" }, { "id": "getting-started/installation", - "url": "/docs/getting-started/installation", + "url": "/docs/getting-started/installation/", "title": "Installation", "breadcrumbs": [ "Getting Started" @@ -58,41 +67,50 @@ }, { "id": "getting-started/integrations", - "url": "/docs/getting-started/integrations", + "url": "/docs/getting-started/integrations/", "title": "Integrations", "breadcrumbs": [ "Getting Started" ], - "content": "Integrations Integrations in EfficientAI are not limited to external voice agent platforms. You can connect integrations across three layers of the stack: 1. Voice agent platforms (agent/runtime side) 2. AI providers (LLM/STT/TTS model side) 3. Telephony providers (PSTN/number/routing side) This lets you test and evaluate the complete call path from model behavior to phone network delivery. Integrations UI 1) Voice platform integrations (agent side) Voice platform integrations connect EfficientAI to externally hosted voice agents.