From 9659e35c84f2b4c29c7eb0d3dba6ed4cd69c25b4 Mon Sep 17 00:00:00 2001 From: Tejas Narayan Date: Wed, 5 Aug 2026 23:00:58 +0530 Subject: [PATCH] Revert "fix: updating metric library" This reverts commit f663650c560af8600e8388adc52904cd7ccdd25d. Co-authored-by: Cursor --- app/api/v1/api.py | 4 - app/api/v1/routes/auth.py | 43 +- app/api/v1/routes/call_import_evaluations.py | 14 - app/api/v1/routes/metric_studio.py | 364 ------------ app/api/v1/routes/metrics.py | 272 ++------- app/api/v1/routes/platform_admin.py | 417 -------------- app/config.py | 4 - app/core/auth/api_key.py | 3 - app/core/auth/local.py | 3 - app/core/auth/oidc_common.py | 3 - app/core/auth/org_access.py | 20 - app/core/auth/platform_admin.py | 129 ----- app/migrations/057_metric_draft_lifecycle.py | 64 --- app/migrations/057_platform_admin.py | 144 ----- app/migrations/058_metric_studio_runs.py | 113 ---- app/models/database.py | 155 ----- app/models/enums.py | 7 - app/models/schemas.py | 127 ----- app/services/metric_studio/__init__.py | 1 - .../metric_studio/metric_selection.py | 100 ---- app/services/metric_studio/source_resolver.py | 273 --------- app/services/signup_reference_codes.py | 61 -- app/workers/config.py | 1 - app/workers/tasks/__init__.py | 5 - .../tasks/evaluate_call_import_row_core.py | 3 +- app/workers/tasks/evaluate_studio_run_item.py | 273 --------- app/workers/tasks/process_evaluator_result.py | 2 - config.yml.example | 3 - .../content/docs/products/metrics-studio.mdx | 10 - frontend/src/App.tsx | 16 +- frontend/src/components/Layout.tsx | 17 +- frontend/src/lib/api.ts | 319 +---------- frontend/src/lib/authSession.ts | 45 -- frontend/src/pages/auth/Login.tsx | 56 +- frontend/src/pages/metrics/MetricsLayout.tsx | 20 - .../src/pages/metrics/MetricsManagement.tsx | 77 +-- frontend/src/pages/metrics/MetricsStudio.tsx | 529 ------------------ .../pages/metrics/MetricsStudioRunDetail.tsx | 262 --------- .../metrics/components/MetricPickerPanel.tsx | 160 ------ .../metrics/components/MetricsTabBar.tsx | 35 -- frontend/src/pages/metrics/index.ts | 3 - frontend/src/pages/platform/PlatformAdmin.tsx | 273 --------- frontend/src/pages/platform/PlatformLogin.tsx | 110 ---- .../platform/PlatformOrgMembersModal.tsx | 263 --------- .../pages/platform/PlatformSignupCodes.tsx | 281 ---------- frontend/src/store/platformAdminStore.ts | 49 -- scripts/create_platform_admin.py | 88 --- tests/conftest.py | 2 - tests/test_api/test_metric_studio.py | 64 --- tests/test_api/test_platform_admin.py | 283 ---------- .../test_metric_studio_source_resolver.py | 119 ---- 51 files changed, 98 insertions(+), 5591 deletions(-) delete mode 100644 app/api/v1/routes/metric_studio.py delete mode 100644 app/api/v1/routes/platform_admin.py delete mode 100644 app/core/auth/org_access.py delete mode 100644 app/core/auth/platform_admin.py delete mode 100644 app/migrations/057_metric_draft_lifecycle.py delete mode 100644 app/migrations/057_platform_admin.py delete mode 100644 app/migrations/058_metric_studio_runs.py delete mode 100644 app/services/metric_studio/__init__.py delete mode 100644 app/services/metric_studio/metric_selection.py delete mode 100644 app/services/metric_studio/source_resolver.py delete mode 100644 app/services/signup_reference_codes.py delete mode 100644 app/workers/tasks/evaluate_studio_run_item.py delete mode 100644 docs-fumadocs/content/docs/products/metrics-studio.mdx delete mode 100644 frontend/src/lib/authSession.ts delete mode 100644 frontend/src/pages/metrics/MetricsLayout.tsx delete mode 100644 frontend/src/pages/metrics/MetricsStudio.tsx delete mode 100644 frontend/src/pages/metrics/MetricsStudioRunDetail.tsx delete mode 100644 frontend/src/pages/metrics/components/MetricPickerPanel.tsx delete mode 100644 frontend/src/pages/metrics/components/MetricsTabBar.tsx delete mode 100644 frontend/src/pages/platform/PlatformAdmin.tsx delete mode 100644 frontend/src/pages/platform/PlatformLogin.tsx delete mode 100644 frontend/src/pages/platform/PlatformOrgMembersModal.tsx delete mode 100644 frontend/src/pages/platform/PlatformSignupCodes.tsx delete mode 100644 frontend/src/store/platformAdminStore.ts delete mode 100644 scripts/create_platform_admin.py delete mode 100644 tests/test_api/test_metric_studio.py delete mode 100644 tests/test_api/test_platform_admin.py delete mode 100644 tests/test_services/test_metric_studio_source_resolver.py diff --git a/app/api/v1/api.py b/app/api/v1/api.py index bbf422e0..d6dc6d9d 100644 --- a/app/api/v1/api.py +++ b/app/api/v1/api.py @@ -41,12 +41,10 @@ call_import_tags, call_import_evaluations, judge_alignment, - metric_studio, workspaces, workspace_iam, dashboard, llm_gateway, - platform_admin, ) api_router = APIRouter() @@ -91,9 +89,7 @@ api_router.include_router(call_import_tags.router) api_router.include_router(call_import_evaluations.router) api_router.include_router(judge_alignment.router) -api_router.include_router(metric_studio.router) api_router.include_router(workspaces.router) api_router.include_router(workspace_iam.router) api_router.include_router(dashboard.router) api_router.include_router(llm_gateway.router) -api_router.include_router(platform_admin.router) diff --git a/app/api/v1/routes/auth.py b/app/api/v1/routes/auth.py index 2a4437c5..97dbc838 100644 --- a/app/api/v1/routes/auth.py +++ b/app/api/v1/routes/auth.py @@ -54,10 +54,6 @@ provision_billing_customer, provision_default_workspace, ) -from app.services.signup_reference_codes import ( - consume_reference_code, - validate_reference_code_for_signup, -) router = APIRouter(prefix="/auth", tags=["Authentication"]) @@ -85,7 +81,6 @@ class AuthProviderConfig(BaseModel): class AuthConfigResponse(BaseModel): providers: List[AuthProviderConfig] tier: str # "oss" | "enterprise" - gated_signup: bool = False class SignupRequest(BaseModel): @@ -94,7 +89,6 @@ class SignupRequest(BaseModel): organization_name: Optional[str] = Field(default=None, max_length=255) first_name: Optional[str] = Field(default=None, max_length=255) last_name: Optional[str] = Field(default=None, max_length=255) - reference_code: Optional[str] = Field(default=None, max_length=64) class LoginRequest(BaseModel): @@ -206,15 +200,7 @@ def get_auth_config() -> AuthConfigResponse: ) ) - return AuthConfigResponse( - providers=providers, - tier=tier, - gated_signup=( - settings.AUTH_GATED_SIGNUP_ENABLED - and settings.AUTH_LOCAL_ALLOW_SIGNUP - and "local_password" in enabled - ), - ) + return AuthConfigResponse(providers=providers, tier=tier) # --------------------------------------------------------------------------- @@ -306,10 +292,6 @@ def signup(payload: SignupRequest, db: Session = Depends(get_db)) -> TokenRespon detail="Self-service signup is disabled. Contact your administrator for access.", ) - reference_row = None - if settings.AUTH_GATED_SIGNUP_ENABLED: - reference_row = validate_reference_code_for_signup(db, payload.reference_code) - existing = db.query(User).filter(User.email == payload.email).first() if existing: raise HTTPException( @@ -350,8 +332,6 @@ def signup(payload: SignupRequest, db: Session = Depends(get_db)) -> TokenRespon name=org_name, email=payload.email, ) - if reference_row is not None: - consume_reference_code(db, reference_row) user.last_login_at = datetime.now(timezone.utc) db.commit() db.refresh(user) @@ -388,17 +368,14 @@ def login(payload: LoginRequest, db: Session = Depends(get_db)) -> LoginResponse memberships = ( db.query(OrganizationMember, Organization) .join(Organization, Organization.id == OrganizationMember.organization_id) - .filter( - OrganizationMember.user_id == user.id, - Organization.is_active == True, # noqa: E712 - ) + .filter(OrganizationMember.user_id == user.id) .order_by(OrganizationMember.joined_at.asc()) .all() ) if not memberships: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, - detail="Your account is not a member of any active organization. Contact your administrator.", + detail="Your account is not a member of any organization. Contact your administrator.", ) if len(memberships) > 1 and not payload.organization_id: @@ -535,13 +512,6 @@ def refresh_session(payload: RefreshRequest, db: Session = Depends(get_db)) -> T detail="User is not a member of this organization.", ) - org = db.query(Organization).filter(Organization.id == row.organization_id).first() - if org is None or not org.is_active: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="Organization disabled.", - ) - revoke_refresh_token(db, payload.refresh_token) role_value = membership.role.value if hasattr(membership.role, "value") else membership.role return _issue_session_tokens( @@ -626,13 +596,6 @@ def switch_organization( detail="You are not a member of that organization.", ) - org = db.query(Organization).filter(Organization.id == target_org_id).first() - if org is None or not org.is_active: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="Organization disabled.", - ) - user = db.query(User).filter(User.id == principal.user_id).first() if user is None or not user.is_active: raise HTTPException( diff --git a/app/api/v1/routes/call_import_evaluations.py b/app/api/v1/routes/call_import_evaluations.py index a315546d..f73744ad 100644 --- a/app/api/v1/routes/call_import_evaluations.py +++ b/app/api/v1/routes/call_import_evaluations.py @@ -657,20 +657,6 @@ async def create_call_import_evaluation( "Refresh the metrics list and try again." ), ) - draft_metrics = [ - metric - for metric in org_metrics - if (getattr(metric, "lifecycle", None) or "active") == "draft" - ] - if draft_metrics: - names = ", ".join(metric.name for metric in draft_metrics) - raise HTTPException( - status_code=400, - detail=( - f"Draft metrics cannot be used in call import evaluations: {names}. " - "Promote them in Metrics Studio first." - ), - ) # Parents themselves are containers, not scored rows, so a disabled # parent shouldn't block the run as long as it has enabled children. # We only reject disabled rows that the worker will actually try to diff --git a/app/api/v1/routes/metric_studio.py b/app/api/v1/routes/metric_studio.py deleted file mode 100644 index 484e2afa..00000000 --- a/app/api/v1/routes/metric_studio.py +++ /dev/null @@ -1,364 +0,0 @@ -"""Metrics Studio API routes.""" - -from __future__ import annotations - -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional -from uuid import UUID, uuid4 - -from fastapi import APIRouter, Depends, HTTPException, Query, status -from sqlalchemy.orm import Session - -from app.database import get_db -from app.dependencies import get_api_key, get_organization_id, get_workspace_id -from app.models.database import ( - Metric, - MetricStudioRun, - MetricStudioRunResult, -) -from app.models.schemas import ( - MetricStudioRunCreate, - MetricStudioRunListResponse, - MetricStudioRunResponse, - MetricStudioRunResultListResponse, - MetricStudioRunResultResponse, - MetricStudioRunRetryRequest, -) -from app.services.metric_studio.metric_selection import expand_studio_metric_selection -from app.services.metric_studio.source_resolver import resolve_source - -router = APIRouter(prefix="/metric-studio", tags=["metric-studio"]) - - -def _serialize_run(run: MetricStudioRun) -> MetricStudioRunResponse: - return MetricStudioRunResponse( - id=run.id, - organization_id=run.organization_id, - workspace_id=run.workspace_id, - name=run.name, - selected_metric_ids=[str(mid) for mid in (run.selected_metric_ids or [])], - selected_metric_groups=run.selected_metric_groups, - transcript_source=run.transcript_source or "diarised", - llm_provider=run.llm_provider, - llm_model=run.llm_model, - status=run.status, - total_items=run.total_items or 0, - completed_items=run.completed_items or 0, - failed_items=run.failed_items or 0, - error_message=run.error_message, - started_at=run.started_at, - finished_at=run.finished_at, - created_at=run.created_at, - updated_at=run.updated_at, - ) - - -def _serialize_result(row: MetricStudioRunResult) -> MetricStudioRunResultResponse: - return MetricStudioRunResultResponse( - id=row.id, - run_id=row.run_id, - source_kind=row.source_kind, - source_ref=row.source_ref, - display_label=row.display_label, - source_metadata=row.source_metadata, - status=row.status, - metric_scores=row.metric_scores or {}, - error_message=row.error_message, - started_at=row.started_at, - finished_at=row.finished_at, - created_at=row.created_at, - updated_at=row.updated_at, - ) - - -def _rollup_run_status(db: Session, run: MetricStudioRun) -> None: - results = ( - db.query(MetricStudioRunResult) - .filter(MetricStudioRunResult.run_id == run.id) - .all() - ) - completed = sum(1 for r in results if r.status == "completed") - failed = sum(1 for r in results if r.status == "failed") - pending = sum(1 for r in results if r.status in {"pending", "running"}) - run.completed_items = completed - run.failed_items = failed - if pending: - run.status = "running" - elif failed and completed: - run.status = "partial" - elif failed: - run.status = "failed" - else: - run.status = "completed" - run.finished_at = datetime.now(timezone.utc) - db.flush() - - -@router.post( - "/runs", - response_model=MetricStudioRunResponse, - status_code=status.HTTP_202_ACCEPTED, - operation_id="createMetricStudioRun", -) -async def create_metric_studio_run( - payload: MetricStudioRunCreate, - api_key: str = Depends(get_api_key), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> MetricStudioRunResponse: - del api_key - - org_metrics = ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.id.in_(payload.metric_ids), - ) - .all() - ) - by_id = {metric.id: metric for metric in org_metrics} - unknown_ids = [mid for mid in payload.metric_ids if mid not in by_id] - if unknown_ids: - raise HTTPException( - status_code=400, - detail=f"Unknown metric ids: {', '.join(str(mid) for mid in unknown_ids)}", - ) - - effective_metrics, parent_to_children = expand_studio_metric_selection( - db, organization_id, payload.metric_ids - ) - if not effective_metrics: - raise HTTPException( - status_code=400, - detail="No scorable metrics after expanding the selection.", - ) - - leaf_metric_ids = [m.id for m in effective_metrics] - selected_metric_groups: Dict[str, List[str]] = { - str(pid): [str(c.id) for c in children] - for pid, children in parent_to_children.items() - } - - if payload.llm_provider or payload.llm_model: - if not (payload.llm_provider and payload.llm_model): - raise HTTPException( - status_code=400, - detail="Both llm_provider and llm_model are required when overriding the run LLM.", - ) - - run = MetricStudioRun( - id=uuid4(), - organization_id=organization_id, - workspace_id=workspace_id, - name=payload.name, - selected_metric_ids=[str(mid) for mid in leaf_metric_ids], - selected_metric_groups=selected_metric_groups or None, - transcript_source=payload.transcript_source, - llm_provider=payload.llm_provider, - llm_model=payload.llm_model, - llm_credential_id=payload.llm_credential_id, - llm_config=payload.llm_config, - metric_llm_overrides=payload.metric_llm_overrides, - status="pending", - total_items=len(payload.sources), - started_at=datetime.now(timezone.utc), - ) - db.add(run) - db.flush() - - result_rows: List[MetricStudioRunResult] = [] - for source in payload.sources: - sample = resolve_source( - db, - organization_id=organization_id, - workspace_id=workspace_id, - source_kind=source.source_kind, - source_ref=source.source_ref, - display_label=source.display_label, - ) - result_row = MetricStudioRunResult( - id=uuid4(), - run_id=run.id, - workspace_id=workspace_id, - source_kind=sample.source_kind, - source_ref=sample.source_ref, - display_label=sample.label, - source_metadata=sample.metadata, - status="pending", - ) - db.add(result_row) - result_rows.append(result_row) - - db.commit() - db.refresh(run) - - from app.workers.tasks.evaluate_studio_run_item import ( - evaluate_studio_run_item_task, - ) - - run.status = "running" - db.commit() - - for result_row in result_rows: - async_result = evaluate_studio_run_item_task.delay(str(result_row.id)) - result_row.celery_task_id = async_result.id - result_row.status = "running" - result_row.started_at = datetime.now(timezone.utc) - db.commit() - - return _serialize_run(run) - - -@router.get("/runs", response_model=MetricStudioRunListResponse) -def list_metric_studio_runs( - skip: int = Query(0, ge=0), - limit: int = Query(50, ge=1, le=200), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> MetricStudioRunListResponse: - query = ( - db.query(MetricStudioRun) - .filter( - MetricStudioRun.organization_id == organization_id, - MetricStudioRun.workspace_id == workspace_id, - ) - .order_by(MetricStudioRun.created_at.desc()) - ) - total = query.count() - runs = query.offset(skip).limit(limit).all() - return MetricStudioRunListResponse( - items=[_serialize_run(run) for run in runs], - total=total, - ) - - -@router.get("/runs/{run_id}", response_model=MetricStudioRunResponse) -def get_metric_studio_run( - run_id: UUID, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> MetricStudioRunResponse: - run = ( - db.query(MetricStudioRun) - .filter( - MetricStudioRun.id == run_id, - MetricStudioRun.organization_id == organization_id, - MetricStudioRun.workspace_id == workspace_id, - ) - .first() - ) - if not run: - raise HTTPException(status_code=404, detail="Studio run not found.") - return _serialize_run(run) - - -@router.get("/runs/{run_id}/results", response_model=MetricStudioRunResultListResponse) -def list_metric_studio_run_results( - run_id: UUID, - skip: int = Query(0, ge=0), - limit: int = Query(100, ge=1, le=500), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> MetricStudioRunResultListResponse: - run = ( - db.query(MetricStudioRun) - .filter( - MetricStudioRun.id == run_id, - MetricStudioRun.organization_id == organization_id, - MetricStudioRun.workspace_id == workspace_id, - ) - .first() - ) - if not run: - raise HTTPException(status_code=404, detail="Studio run not found.") - - query = ( - db.query(MetricStudioRunResult) - .filter(MetricStudioRunResult.run_id == run_id) - .order_by(MetricStudioRunResult.created_at.asc()) - ) - total = query.count() - rows = query.offset(skip).limit(limit).all() - return MetricStudioRunResultListResponse( - items=[_serialize_result(row) for row in rows], - total=total, - ) - - -@router.post("/runs/{run_id}/retry", response_model=MetricStudioRunResponse) -def retry_metric_studio_run( - run_id: UUID, - body: MetricStudioRunRetryRequest, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> MetricStudioRunResponse: - run = ( - db.query(MetricStudioRun) - .filter( - MetricStudioRun.id == run_id, - MetricStudioRun.organization_id == organization_id, - MetricStudioRun.workspace_id == workspace_id, - ) - .first() - ) - if not run: - raise HTTPException(status_code=404, detail="Studio run not found.") - - query = db.query(MetricStudioRunResult).filter( - MetricStudioRunResult.run_id == run_id - ) - if body.result_ids: - query = query.filter(MetricStudioRunResult.id.in_(body.result_ids)) - else: - query = query.filter(MetricStudioRunResult.status == "failed") - - rows = query.all() - if not rows: - raise HTTPException(status_code=400, detail="No results eligible for retry.") - - from app.workers.tasks.evaluate_studio_run_item import ( - evaluate_studio_run_item_task, - ) - - run.status = "running" - run.finished_at = None - for row in rows: - row.status = "running" - row.error_message = None - row.metric_scores = {} - row.started_at = datetime.now(timezone.utc) - row.finished_at = None - async_result = evaluate_studio_run_item_task.delay(str(row.id)) - row.celery_task_id = async_result.id - db.commit() - _rollup_run_status(db, run) - db.commit() - db.refresh(run) - return _serialize_run(run) - - -@router.delete("/runs/{run_id}", status_code=status.HTTP_204_NO_CONTENT) -def delete_metric_studio_run( - run_id: UUID, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -) -> None: - run = ( - db.query(MetricStudioRun) - .filter( - MetricStudioRun.id == run_id, - MetricStudioRun.organization_id == organization_id, - MetricStudioRun.workspace_id == workspace_id, - ) - .first() - ) - if not run: - raise HTTPException(status_code=404, detail="Studio run not found.") - db.delete(run) - db.commit() diff --git a/app/api/v1/routes/metrics.py b/app/api/v1/routes/metrics.py index 1c312c87..760580b5 100644 --- a/app/api/v1/routes/metrics.py +++ b/app/api/v1/routes/metrics.py @@ -2,7 +2,6 @@ import json import re -from datetime import datetime, timezone from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, status from sqlalchemy.orm import Session from sqlalchemy import and_, or_ @@ -19,9 +18,6 @@ MetricCreate, MetricCreateWithChildren, MetricChildDraft, - MetricDraftCreate, - MetricDraftCreateWithChildren, - MetricPromoteResponse, MetricUpdate, MetricResponse, PromoteDiscoveredChildRequest, @@ -184,9 +180,6 @@ def _serialize_metric_tree(metric: Metric) -> Dict[str, Any]: "compare_transcripts": bool( getattr(metric, "compare_transcripts", False) ), - "lifecycle": getattr(metric, "lifecycle", None) or "active", - "promoted_from_draft_at": getattr(metric, "promoted_from_draft_at", None), - "studio_notes": getattr(metric, "studio_notes", None), "children": children_payload, "created_at": metric.created_at, "updated_at": metric.updated_at, @@ -327,114 +320,27 @@ def create_metric( @router.post( - "/drafts", + "/with-children", response_model=MetricResponse, status_code=201, - operation_id="createMetricDraft", + operation_id="createMetricWithChildren", ) -def create_metric_draft( - metric_data: MetricDraftCreate, +def create_metric_with_children( + payload: MetricCreateWithChildren, organization_id: UUID = Depends(get_organization_id), workspace_id: UUID = Depends(get_workspace_id), db: Session = Depends(get_db), ): - """Create a draft metric for Metrics Studio (hidden from production flows).""" - _validate_hierarchy_fields( - organization_id, - db, - parent_metric_id=metric_data.parent_metric_id, - selection_mode=metric_data.selection_mode, - metric_type=metric_data.metric_type, - allow_discovery=metric_data.allow_discovery, - ) - - if metric_data.parent_metric_id is not None: - parent_row = ( - db.query(Metric) - .filter( - Metric.id == metric_data.parent_metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if parent_row is None: - raise HTTPException(status_code=400, detail="Parent metric not found.") - effective_workspace_id: Optional[UUID] = parent_row.workspace_id - elif metric_data.scope == "organization": - effective_workspace_id = None - else: - effective_workspace_id = workspace_id - - workspace_filter = ( - Metric.workspace_id.is_(None) - if effective_workspace_id is None - else Metric.workspace_id == effective_workspace_id - ) - parent_filter = ( - Metric.parent_metric_id.is_(None) - if metric_data.parent_metric_id is None - else Metric.parent_metric_id == metric_data.parent_metric_id - ) - existing = ( - db.query(Metric) - .filter( - Metric.name == metric_data.name, - Metric.organization_id == organization_id, - workspace_filter, - parent_filter, - ) - .first() - ) - if existing: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="A metric with this name already exists", - ) - - effective_metric_type = metric_data.metric_type - if metric_data.parent_metric_id is not None: - effective_metric_type = MetricType.BOOLEAN - - metric = Metric( - organization_id=organization_id, - workspace_id=effective_workspace_id, - name=metric_data.name, - description=metric_data.description, - example=metric_data.example, - metric_type=effective_metric_type, - metric_category=metric_data.metric_category, - trigger=metric_data.trigger, - enabled=False, - is_default=False, - metric_origin=metric_data.metric_origin or "custom", - supported_surfaces=metric_data.supported_surfaces or ["agent"], - enabled_surfaces=[], - custom_data_type=metric_data.custom_data_type, - custom_config=metric_data.custom_config, - tags=metric_data.tags, - capture_rationale=bool(metric_data.capture_rationale), - parent_metric_id=metric_data.parent_metric_id, - selection_mode=metric_data.selection_mode, - allow_discovery=bool(metric_data.allow_discovery), - compare_transcripts=bool(metric_data.compare_transcripts), - lifecycle="draft", - studio_notes=metric_data.studio_notes, - ) - db.add(metric) - db.commit() - db.refresh(metric) - return _serialize_metric_tree(metric) - + """Atomically create a parent category metric plus its children. -def _create_metric_with_children( - db: Session, - *, - organization_id: UUID, - workspace_id: UUID, - payload: MetricCreateWithChildren, - lifecycle: str = "active", - studio_notes: Optional[str] = None, -) -> Metric: + The parent gets ``metric_type=text`` (it's a category label, not a + score) and ``selection_mode`` from the payload. Every child is + forced to ``boolean`` so the LLM-evaluation path treats them as + yes/no labels. Both the parent and all children are stamped with + the same scope: either the active workspace (``scope="workspace"``, + default) or ``workspace_id=NULL`` (``scope="organization"``, the + org-shared shape). + """ if payload.selection_mode not in _VALID_SELECTION_MODES: raise HTTPException( status_code=400, @@ -444,6 +350,8 @@ def _create_metric_with_children( ), ) + # Org-shared categories live with ``workspace_id=NULL`` so every + # workspace in the org sees the same category + children. effective_workspace_id: Optional[UUID] = ( None if payload.scope == "organization" else workspace_id ) @@ -468,6 +376,9 @@ def _create_metric_with_children( detail=f"A top-level metric named '{payload.name}' already exists.", ) + # Detect duplicate child names within the same request before any + # writes — the DB has no compound uniqueness constraint, so we + # enforce it in code. child_names_seen: set[str] = set() for child in payload.children: key = (child.name or "").strip().lower() @@ -490,9 +401,10 @@ def _create_metric_with_children( if payload.enabled_surfaces is not None else (payload.supported_surfaces or ["agent"]) if payload.enabled else [] ) - is_draft = lifecycle == "draft" - parent_enabled_surfaces: List[str] = [] if is_draft else enabled_surfaces + # ``allow_discovery`` requires a parent (selection_mode set). + # Both single_choice and multi_label parents are valid hosts; the + # prompt builder + mapper handle the per-mode semantics. if payload.allow_discovery and not payload.selection_mode: raise HTTPException( status_code=400, @@ -507,28 +419,36 @@ def _create_metric_with_children( workspace_id=effective_workspace_id, name=payload.name, description=payload.description, + # The parent itself stores no numeric value — its "result" is the + # set of true children. Treat it as text so the rest of the + # stack (aggregation, CSV export, etc.) renders the chosen child + # name as the parent's "value". metric_type=MetricType.TEXT, metric_category=payload.metric_category, trigger=MetricTrigger.ALWAYS, - enabled=not is_draft and len(parent_enabled_surfaces) > 0, + enabled=len(enabled_surfaces) > 0, is_default=False, metric_origin="custom", supported_surfaces=payload.supported_surfaces or ["agent"], - enabled_surfaces=parent_enabled_surfaces, + enabled_surfaces=enabled_surfaces, tags=payload.tags, + # Hierarchical mode now captures rationale at the PARENT level + # (the LLM emits one rationale per category, never per child), + # so honour the user's toggle here and force children below to + # capture_rationale=False. capture_rationale=bool(payload.capture_rationale), selection_mode=payload.selection_mode, allow_discovery=bool(payload.allow_discovery), - lifecycle=lifecycle, - studio_notes=studio_notes, ) db.add(parent) db.flush() for child_draft in payload.children: - child_enabled = bool(child_draft.enabled) and len(parent_enabled_surfaces) > 0 child = Metric( organization_id=organization_id, + # Children inherit the parent's scope (workspace UUID or + # NULL for org-shared) so the whole category subtree stays + # in one place. workspace_id=effective_workspace_id, name=child_draft.name, description=child_draft.description, @@ -536,120 +456,28 @@ def _create_metric_with_children( metric_type=MetricType.BOOLEAN, metric_category=payload.metric_category, trigger=MetricTrigger.ALWAYS, - enabled=not is_draft and child_enabled, + enabled=bool(child_draft.enabled) and len(enabled_surfaces) > 0, is_default=False, metric_origin="custom", supported_surfaces=payload.supported_surfaces or ["agent"], - enabled_surfaces=parent_enabled_surfaces if child_draft.enabled else [], + enabled_surfaces=( + enabled_surfaces if child_draft.enabled else [] + ), custom_data_type="boolean", custom_config={}, tags=child_draft.tags, + # Children in hierarchical mode never carry their own + # rationale — the parent owns the single rationale string + # for the whole group. Force false regardless of payload so + # legacy clients can't accidentally enable per-child + # rationales that the worker would then ignore. capture_rationale=False, parent_metric_id=parent.id, - lifecycle=lifecycle, ) db.add(child) db.commit() db.refresh(parent) - return parent - - -@router.post( - "/drafts/with-children", - response_model=MetricResponse, - status_code=201, - operation_id="createMetricDraftWithChildren", -) -def create_metric_draft_with_children( - payload: MetricDraftCreateWithChildren, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -): - """Atomically create a draft parent category metric plus its children.""" - parent = _create_metric_with_children( - db, - organization_id=organization_id, - workspace_id=workspace_id, - payload=payload, - lifecycle="draft", - studio_notes=payload.studio_notes, - ) - return _serialize_metric_tree(parent) - - -@router.post( - "/{metric_id}/promote", - response_model=MetricPromoteResponse, - operation_id="promoteMetricDraft", -) -def promote_metric_draft( - metric_id: UUID, - organization_id: UUID = Depends(get_organization_id), - db: Session = Depends(get_db), -): - """Promote a draft metric to active production use.""" - metric = ( - db.query(Metric) - .filter( - Metric.id == metric_id, - Metric.organization_id == organization_id, - ) - .first() - ) - if not metric: - raise HTTPException(status_code=404, detail="Metric not found") - if (metric.lifecycle or "active") != "draft": - raise HTTPException( - status_code=400, - detail="Only draft metrics can be promoted.", - ) - - promoted_at = datetime.now(timezone.utc) - metric.lifecycle = "active" - metric.enabled = True - metric.promoted_from_draft_at = promoted_at - if not (metric.enabled_surfaces or []): - metric.enabled_surfaces = ["agent"] - if not (metric.supported_surfaces or []): - metric.supported_surfaces = ["agent"] - db.commit() - db.refresh(metric) - return MetricPromoteResponse( - metric=_serialize_metric_tree(metric), - promoted_at=promoted_at, - ) - - -@router.post( - "/with-children", - response_model=MetricResponse, - status_code=201, - operation_id="createMetricWithChildren", -) -def create_metric_with_children( - payload: MetricCreateWithChildren, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - db: Session = Depends(get_db), -): - """Atomically create a parent category metric plus its children. - - The parent gets ``metric_type=text`` (it's a category label, not a - score) and ``selection_mode`` from the payload. Every child is - forced to ``boolean`` so the LLM-evaluation path treats them as - yes/no labels. Both the parent and all children are stamped with - the same scope: either the active workspace (``scope="workspace"``, - default) or ``workspace_id=NULL`` (``scope="organization"``, the - org-shared shape). - """ - parent = _create_metric_with_children( - db, - organization_id=organization_id, - workspace_id=workspace_id, - payload=payload, - ) return _serialize_metric_tree(parent) @@ -1013,14 +841,6 @@ def promote_discovered_metric( @router.get("", response_model=List[MetricResponse]) def list_metrics( surface: Optional[str] = None, - include_drafts: bool = Query( - False, - description="When true, include draft metrics (Studio-only) in the listing.", - ), - drafts_only: bool = Query( - False, - description="When true, return only draft metrics.", - ), include_children: bool = Query( True, description=( @@ -1058,12 +878,6 @@ def list_metrics( Metric.metric_origin == "default", ), ) - if drafts_only: - query = query.filter(Metric.lifecycle == "draft") - elif not include_drafts: - query = query.filter( - or_(Metric.lifecycle.is_(None), Metric.lifecycle == "active") - ) metrics = ( query.order_by(Metric.is_default.desc(), Metric.created_at.desc()).all() ) diff --git a/app/api/v1/routes/platform_admin.py b/app/api/v1/routes/platform_admin.py deleted file mode 100644 index 44c392ae..00000000 --- a/app/api/v1/routes/platform_admin.py +++ /dev/null @@ -1,417 +0,0 @@ -"""Platform admin routes for cross-org management.""" - -from __future__ import annotations - -from datetime import datetime, timezone -from typing import List, Optional -from uuid import UUID - -from fastapi import APIRouter, Depends, HTTPException, Query, status -from pydantic import BaseModel, EmailStr, Field -from sqlalchemy import func -from sqlalchemy.orm import Session - -from app.core.auth.platform_admin import ( - PlatformAdminPrincipal, - create_platform_access_token, - get_platform_admin, - platform_admin_feature_enabled, -) -from app.core.auth.refresh_tokens import revoke_all_user_refresh_tokens -from app.core.password import hash_password, validate_password_strength, verify_password -from app.database import get_db -from app.models.database import ( - Organization, - OrganizationMember, - PlatformAdmin, - SignupReferenceCode, - User, -) -from app.services.signup_reference_codes import hash_reference_code - -router = APIRouter(prefix="/platform", tags=["Platform Admin"]) - - -class PlatformLoginRequest(BaseModel): - email: EmailStr - password: str - - -class PlatformAdminSummary(BaseModel): - id: str - email: str - - -class PlatformTokenResponse(BaseModel): - access_token: str - token_type: str = "Bearer" - expires_in: int - admin: PlatformAdminSummary - - -class OrganizationListItem(BaseModel): - id: str - name: str - is_active: bool - member_count: int - created_at: Optional[str] = None - disabled_at: Optional[str] = None - - -class OrganizationListResponse(BaseModel): - items: List[OrganizationListItem] - total: int - offset: int - limit: int - - -class OrganizationStatsResponse(BaseModel): - total: int - active: int - disabled: int - - -class OrganizationUpdateRequest(BaseModel): - is_active: bool - - -class OrgUserItem(BaseModel): - id: str - email: str - role: str - is_active: bool - - -class PlatformPasswordResetRequest(BaseModel): - new_password: str = Field(min_length=8, max_length=32) - - -class PlatformPasswordResetResponse(BaseModel): - user_id: str - email: str - message: str = "Password reset successfully" - - -class SignupCodeCreateRequest(BaseModel): - code: str = Field(min_length=4, max_length=64) - label: Optional[str] = Field(default=None, max_length=255) - max_uses: Optional[int] = Field(default=None, ge=1) - expires_at: Optional[datetime] = None - - -class SignupCodeResponse(BaseModel): - id: str - label: Optional[str] = None - max_uses: Optional[int] = None - use_count: int - expires_at: Optional[str] = None - is_active: bool - created_at: Optional[str] = None - code: Optional[str] = None - - -class SignupCodeUpdateRequest(BaseModel): - is_active: Optional[bool] = None - max_uses: Optional[int] = Field(default=None, ge=1) - label: Optional[str] = Field(default=None, max_length=255) - - -def _validate_password_or_400(password: str) -> None: - try: - validate_password_strength(password) - except ValueError as exc: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc - - -def _serialize_org(org: Organization, member_count: int) -> OrganizationListItem: - return OrganizationListItem( - id=str(org.id), - name=org.name, - is_active=bool(org.is_active), - member_count=member_count, - created_at=org.created_at.isoformat() if org.created_at else None, - disabled_at=org.disabled_at.isoformat() if org.disabled_at else None, - ) - - -def _serialize_signup_code(row: SignupReferenceCode, *, include_code: bool = False, code: Optional[str] = None) -> SignupCodeResponse: - return SignupCodeResponse( - id=str(row.id), - label=row.label, - max_uses=row.max_uses, - use_count=row.use_count or 0, - expires_at=row.expires_at.isoformat() if row.expires_at else None, - is_active=bool(row.is_active), - created_at=row.created_at.isoformat() if row.created_at else None, - code=code if include_code else None, - ) - - -@router.post("/auth/login", response_model=PlatformTokenResponse) -def platform_login(payload: PlatformLoginRequest, db: Session = Depends(get_db)) -> PlatformTokenResponse: - if not platform_admin_feature_enabled(db): - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Not found") - - admin = ( - db.query(PlatformAdmin) - .filter(PlatformAdmin.email == payload.email, PlatformAdmin.is_active == True) # noqa: E712 - .first() - ) - if admin is None or not verify_password(payload.password, admin.password_hash): - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid email or password.", - ) - - admin.last_login_at = datetime.now(timezone.utc) - db.commit() - - access_token, expires_in = create_platform_access_token( - platform_admin_id=admin.id, - email=admin.email, - ) - return PlatformTokenResponse( - access_token=access_token, - expires_in=expires_in, - admin=PlatformAdminSummary(id=str(admin.id), email=admin.email), - ) - - -@router.get("/auth/me", response_model=PlatformAdminSummary) -def platform_me( - principal: PlatformAdminPrincipal = Depends(get_platform_admin), -) -> PlatformAdminSummary: - return PlatformAdminSummary(id=str(principal.platform_admin_id), email=principal.email) - - -@router.get("/organizations", response_model=OrganizationListResponse) -def list_organizations( - offset: int = Query(0, ge=0), - limit: int = Query(50, ge=1, le=200), - search: Optional[str] = Query(default=None, max_length=255), - is_active: Optional[bool] = Query(default=None), - _principal: PlatformAdminPrincipal = Depends(get_platform_admin), - db: Session = Depends(get_db), -) -> OrganizationListResponse: - query = db.query(Organization) - if search: - query = query.filter(Organization.name.ilike(f"%{search}%")) - if is_active is not None: - query = query.filter(Organization.is_active == is_active) - - total = query.count() - orgs = ( - query.order_by(Organization.created_at.desc()) - .offset(offset) - .limit(limit) - .all() - ) - - member_counts = dict( - db.query(OrganizationMember.organization_id, func.count(OrganizationMember.id)) - .filter(OrganizationMember.organization_id.in_([org.id for org in orgs])) - .group_by(OrganizationMember.organization_id) - .all() - ) if orgs else {} - - return OrganizationListResponse( - items=[_serialize_org(org, member_counts.get(org.id, 0)) for org in orgs], - total=total, - offset=offset, - limit=limit, - ) - - -@router.get("/organizations/stats", response_model=OrganizationStatsResponse) -def organization_stats( - _principal: PlatformAdminPrincipal = Depends(get_platform_admin), - db: Session = Depends(get_db), -) -> OrganizationStatsResponse: - total = db.query(func.count(Organization.id)).scalar() or 0 - active = ( - db.query(func.count(Organization.id)) - .filter(Organization.is_active == True) # noqa: E712 - .scalar() - or 0 - ) - disabled = total - active - return OrganizationStatsResponse(total=total, active=active, disabled=disabled) - - -@router.patch("/organizations/{org_id}", response_model=OrganizationListItem) -def update_organization( - org_id: UUID, - payload: OrganizationUpdateRequest, - _principal: PlatformAdminPrincipal = Depends(get_platform_admin), - db: Session = Depends(get_db), -) -> OrganizationListItem: - org = db.query(Organization).filter(Organization.id == org_id).first() - if org is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Organization not found") - - org.is_active = payload.is_active - org.disabled_at = None if payload.is_active else datetime.now(timezone.utc) - db.commit() - db.refresh(org) - - member_count = ( - db.query(func.count(OrganizationMember.id)) - .filter(OrganizationMember.organization_id == org.id) - .scalar() - or 0 - ) - return _serialize_org(org, member_count) - - -@router.get("/organizations/{org_id}/users", response_model=List[OrgUserItem]) -def list_organization_users( - org_id: UUID, - role: Optional[str] = Query(default=None), - _principal: PlatformAdminPrincipal = Depends(get_platform_admin), - db: Session = Depends(get_db), -) -> List[OrgUserItem]: - org = db.query(Organization).filter(Organization.id == org_id).first() - if org is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Organization not found") - - query = ( - db.query(User, OrganizationMember) - .join(OrganizationMember, OrganizationMember.user_id == User.id) - .filter(OrganizationMember.organization_id == org_id) - ) - if role: - query = query.filter(OrganizationMember.role == role) - - rows = query.order_by(User.email.asc()).all() - items: List[OrgUserItem] = [] - for user, member in rows: - role_value = member.role.value if hasattr(member.role, "value") else member.role - items.append( - OrgUserItem( - id=str(user.id), - email=user.email, - role=role_value, - is_active=bool(user.is_active), - ) - ) - return items - - -@router.post( - "/organizations/{org_id}/users/{user_id}/reset-password", - response_model=PlatformPasswordResetResponse, -) -def platform_reset_user_password( - org_id: UUID, - user_id: UUID, - payload: PlatformPasswordResetRequest, - _principal: PlatformAdminPrincipal = Depends(get_platform_admin), - db: Session = Depends(get_db), -) -> PlatformPasswordResetResponse: - member = ( - db.query(OrganizationMember) - .filter( - OrganizationMember.organization_id == org_id, - OrganizationMember.user_id == user_id, - ) - .first() - ) - if member is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="User is not a member of this organization", - ) - - user = db.query(User).filter(User.id == user_id).first() - if user is None or not user.is_active: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="User not found or inactive", - ) - - _validate_password_or_400(payload.new_password) - user.password_hash = hash_password(payload.new_password) - revoke_all_user_refresh_tokens(db, user_id=user.id) - db.commit() - - return PlatformPasswordResetResponse(user_id=str(user.id), email=user.email) - - -@router.get("/signup-codes", response_model=List[SignupCodeResponse]) -def list_signup_codes( - _principal: PlatformAdminPrincipal = Depends(get_platform_admin), - db: Session = Depends(get_db), -) -> List[SignupCodeResponse]: - rows = ( - db.query(SignupReferenceCode) - .order_by(SignupReferenceCode.created_at.desc()) - .all() - ) - return [_serialize_signup_code(row) for row in rows] - - -@router.post("/signup-codes", response_model=SignupCodeResponse, status_code=status.HTTP_201_CREATED) -def create_signup_code( - payload: SignupCodeCreateRequest, - principal: PlatformAdminPrincipal = Depends(get_platform_admin), - db: Session = Depends(get_db), -) -> SignupCodeResponse: - code_hash = hash_reference_code(payload.code) - existing = db.query(SignupReferenceCode).filter(SignupReferenceCode.code_hash == code_hash).first() - if existing is not None: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="A reference code with this value already exists.", - ) - - row = SignupReferenceCode( - code_hash=code_hash, - label=payload.label, - max_uses=payload.max_uses, - expires_at=payload.expires_at, - is_active=True, - created_by=principal.platform_admin_id, - ) - db.add(row) - db.commit() - db.refresh(row) - return _serialize_signup_code(row, include_code=True, code=payload.code.strip()) - - -@router.patch("/signup-codes/{code_id}", response_model=SignupCodeResponse) -def update_signup_code( - code_id: UUID, - payload: SignupCodeUpdateRequest, - _principal: PlatformAdminPrincipal = Depends(get_platform_admin), - db: Session = Depends(get_db), -) -> SignupCodeResponse: - row = db.query(SignupReferenceCode).filter(SignupReferenceCode.id == code_id).first() - if row is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Reference code not found") - - if payload.is_active is not None: - row.is_active = payload.is_active - if payload.max_uses is not None: - row.max_uses = payload.max_uses - if payload.label is not None: - row.label = payload.label - - db.commit() - db.refresh(row) - return _serialize_signup_code(row) - - -@router.delete("/signup-codes/{code_id}", response_model=SignupCodeResponse) -def deactivate_signup_code( - code_id: UUID, - _principal: PlatformAdminPrincipal = Depends(get_platform_admin), - db: Session = Depends(get_db), -) -> SignupCodeResponse: - row = db.query(SignupReferenceCode).filter(SignupReferenceCode.id == code_id).first() - if row is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Reference code not found") - - row.is_active = False - db.commit() - db.refresh(row) - return _serialize_signup_code(row) diff --git a/app/config.py b/app/config.py index ca64ff34..91ecdbb3 100644 --- a/app/config.py +++ b/app/config.py @@ -109,7 +109,6 @@ class Settings(BaseSettings): # Authentication AUTH_PROVIDERS: List[str] = ["api_key"] AUTH_LOCAL_ALLOW_SIGNUP: bool = True - AUTH_GATED_SIGNUP_ENABLED: bool = False AUTH_LOCAL_TOKEN_TTL_MINUTES: int = 15 AUTH_REFRESH_TOKEN_TTL_DAYS: int = 7 AUTH_OIDC_ISSUER: Optional[str] = None @@ -687,9 +686,6 @@ def load_config_from_file(config_path: str) -> None: settings.AUTH_LOCAL_TOKEN_TTL_MINUTES = int(local_config["token_ttl_minutes"]) if "refresh_token_ttl_days" in local_config: settings.AUTH_REFRESH_TOKEN_TTL_DAYS = int(local_config["refresh_token_ttl_days"]) - gated_config = local_config.get("gated_signup", {}) - if isinstance(gated_config, dict) and "enabled" in gated_config: - settings.AUTH_GATED_SIGNUP_ENABLED = bool(gated_config["enabled"]) oidc_config = auth_config.get("oidc", {}) if isinstance(oidc_config, dict): diff --git a/app/core/auth/api_key.py b/app/core/auth/api_key.py index 48fdc346..6ff49ca7 100644 --- a/app/core/auth/api_key.py +++ b/app/core/auth/api_key.py @@ -12,7 +12,6 @@ from sqlalchemy.orm import Session -from app.core.auth.org_access import ensure_organization_active from app.core.auth.principal import AuthMethod, Principal from app.core.auth.providers import AuthError, AuthProvider, RawCredential from app.models.database import APIKey @@ -38,8 +37,6 @@ def authenticate(self, cred: RawCredential, db: Session) -> Principal: if not db_key: raise AuthError("Invalid API key") - ensure_organization_active(db, db_key.organization_id) - db_key.last_used = datetime.now(timezone.utc) db.commit() diff --git a/app/core/auth/local.py b/app/core/auth/local.py index 4408bc3c..bfadc1bb 100644 --- a/app/core/auth/local.py +++ b/app/core/auth/local.py @@ -17,7 +17,6 @@ from jose import JWTError from sqlalchemy.orm import Session -from app.core.auth.org_access import ensure_organization_active from app.core.auth.principal import AuthMethod, Principal from app.core.auth.providers import AuthError, AuthProvider, RawCredential from app.core.auth.tokens import ISSUER, decode_access_token @@ -78,8 +77,6 @@ def authenticate(self, cred: RawCredential, db: Session) -> Principal: if not member: raise AuthError("User is not a member of this organization") - ensure_organization_active(db, org_id) - return Principal( organization_id=org_id, auth_method=AuthMethod.LOCAL_PASSWORD, diff --git a/app/core/auth/oidc_common.py b/app/core/auth/oidc_common.py index 6a70d2bb..6c2542c7 100644 --- a/app/core/auth/oidc_common.py +++ b/app/core/auth/oidc_common.py @@ -20,7 +20,6 @@ from loguru import logger from sqlalchemy.orm import Session -from app.core.auth.org_access import ensure_organization_active from app.core.auth.principal import Principal from app.core.auth.providers import AuthError from app.models.database import Organization, OrganizationMember, User @@ -262,8 +261,6 @@ def principal_from_oidc_claims( last_name=last_name, ) - ensure_organization_active(db, organization.id) - return Principal( organization_id=organization.id, auth_method=auth_method, diff --git a/app/core/auth/org_access.py b/app/core/auth/org_access.py deleted file mode 100644 index 6315bf95..00000000 --- a/app/core/auth/org_access.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Organization access guards shared across auth providers.""" - -from __future__ import annotations - -from uuid import UUID - -from sqlalchemy.orm import Session - -from app.core.auth.providers import AuthError -from app.models.database import Organization - - -def ensure_organization_active(db: Session, organization_id: UUID) -> Organization: - """Raise AuthError when the organization is missing or disabled.""" - org = db.query(Organization).filter(Organization.id == organization_id).first() - if org is None: - raise AuthError("Organization not found") - if not org.is_active: - raise AuthError("Organization disabled", status_code=403) - return org diff --git a/app/core/auth/platform_admin.py b/app/core/auth/platform_admin.py deleted file mode 100644 index d2a4f0bf..00000000 --- a/app/core/auth/platform_admin.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Platform admin JWT issuance and FastAPI dependencies.""" - -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime, timedelta, timezone -from typing import Any, Dict, Optional, Tuple -from uuid import UUID, uuid4 - -from fastapi import Depends, Header, HTTPException, status -from jose import JWTError, jwt -from sqlalchemy.orm import Session - -from app.config import settings -from app.database import get_db -from app.models.database import PlatformAdmin - -PLATFORM_ISSUER = "efficientai-platform" -PLATFORM_SCOPE = "platform_admin" -ALGORITHM = "HS256" - - -@dataclass(frozen=True) -class PlatformAdminPrincipal: - platform_admin_id: UUID - email: str - - -def create_platform_access_token( - *, - platform_admin_id: UUID, - email: str, - expires_in_minutes: Optional[int] = None, -) -> Tuple[str, int]: - ttl_minutes = expires_in_minutes or getattr(settings, "AUTH_LOCAL_TOKEN_TTL_MINUTES", 15) - ttl_seconds = ttl_minutes * 60 - now = datetime.now(timezone.utc) - payload: Dict[str, Any] = { - "iss": PLATFORM_ISSUER, - "sub": str(platform_admin_id), - "email": email, - "scope": PLATFORM_SCOPE, - "jti": str(uuid4()), - "iat": int(now.timestamp()), - "exp": int((now + timedelta(minutes=ttl_minutes)).timestamp()), - } - token = jwt.encode(payload, settings.SECRET_KEY, algorithm=ALGORITHM) - return token, ttl_seconds - - -def decode_platform_access_token(token: str) -> Dict[str, Any]: - return jwt.decode( - token, - settings.SECRET_KEY, - algorithms=[ALGORITHM], - issuer=PLATFORM_ISSUER, - options={"verify_aud": False}, - ) - - -def _extract_bearer(authorization: Optional[str]) -> Optional[str]: - if not authorization: - return None - scheme, _, token = authorization.partition(" ") - if scheme.lower() != "bearer" or not token.strip(): - return None - return token.strip() - - -def platform_admin_feature_enabled(db: Session) -> bool: - return ( - db.query(PlatformAdmin.id) - .filter(PlatformAdmin.is_active == True) # noqa: E712 - .first() - is not None - ) - - -def require_platform_admin_feature(db: Session = Depends(get_db)) -> None: - if not platform_admin_feature_enabled(db): - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Not found") - - -def get_platform_admin( - authorization: Optional[str] = Header(None, alias="Authorization"), - db: Session = Depends(get_db), - _feature: None = Depends(require_platform_admin_feature), -) -> PlatformAdminPrincipal: - bearer = _extract_bearer(authorization) - if not bearer: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Authentication required (send Authorization: Bearer ...)", - ) - - try: - claims = decode_platform_access_token(bearer) - except JWTError as exc: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail=f"Invalid platform admin token: {exc}", - ) from exc - - if claims.get("scope") != PLATFORM_SCOPE: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid platform admin token scope.", - ) - - try: - admin_id = UUID(claims["sub"]) - except (KeyError, ValueError) as exc: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Malformed platform admin token.", - ) from exc - - admin = ( - db.query(PlatformAdmin) - .filter(PlatformAdmin.id == admin_id, PlatformAdmin.is_active == True) # noqa: E712 - .first() - ) - if admin is None: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Platform admin no longer active.", - ) - - return PlatformAdminPrincipal(platform_admin_id=admin.id, email=admin.email) diff --git a/app/migrations/057_metric_draft_lifecycle.py b/app/migrations/057_metric_draft_lifecycle.py deleted file mode 100644 index 8b6dbad4..00000000 --- a/app/migrations/057_metric_draft_lifecycle.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Migration: Add metric draft lifecycle columns.""" - -from sqlalchemy import text -from sqlalchemy.orm import Session - -description = "Add lifecycle, promoted_from_draft_at, studio_notes to metrics." - - -def _column_exists(db: Session, table_name: str, column_name: str) -> bool: - row = db.execute( - text( - """ - SELECT 1 - FROM information_schema.columns - WHERE table_name = :table_name AND column_name = :column_name - """ - ), - {"table_name": table_name, "column_name": column_name}, - ).first() - return row is not None - - -def upgrade(db: Session): - if not _column_exists(db, "metrics", "lifecycle"): - db.execute( - text( - """ - ALTER TABLE metrics - ADD COLUMN lifecycle VARCHAR(20) NOT NULL DEFAULT 'active' - """ - ) - ) - print("Added metrics.lifecycle") - - if not _column_exists(db, "metrics", "promoted_from_draft_at"): - db.execute( - text( - """ - ALTER TABLE metrics - ADD COLUMN promoted_from_draft_at TIMESTAMPTZ NULL - """ - ) - ) - print("Added metrics.promoted_from_draft_at") - - if not _column_exists(db, "metrics", "studio_notes"): - db.execute( - text( - """ - ALTER TABLE metrics - ADD COLUMN studio_notes TEXT NULL - """ - ) - ) - print("Added metrics.studio_notes") - - -def downgrade(db: Session): - if _column_exists(db, "metrics", "studio_notes"): - db.execute(text("ALTER TABLE metrics DROP COLUMN studio_notes")) - if _column_exists(db, "metrics", "promoted_from_draft_at"): - db.execute(text("ALTER TABLE metrics DROP COLUMN promoted_from_draft_at")) - if _column_exists(db, "metrics", "lifecycle"): - db.execute(text("ALTER TABLE metrics DROP COLUMN lifecycle")) diff --git a/app/migrations/057_platform_admin.py b/app/migrations/057_platform_admin.py deleted file mode 100644 index 70a0332e..00000000 --- a/app/migrations/057_platform_admin.py +++ /dev/null @@ -1,144 +0,0 @@ -""" -Migration: platform admin tables, org disable flag, signup reference codes. -""" - -from sqlalchemy import text -from sqlalchemy.orm import Session - -description = "Add platform admin auth, org is_active flag, and signup reference codes." - - -def _column_exists(db: Session, table_name: str, column_name: str) -> bool: - row = db.execute( - text( - """ - SELECT 1 - FROM information_schema.columns - WHERE table_name = :table_name AND column_name = :column_name - """ - ), - {"table_name": table_name, "column_name": column_name}, - ).first() - return row is not None - - -def _table_exists(db: Session, table_name: str) -> bool: - row = db.execute( - text( - """ - SELECT 1 - FROM information_schema.tables - WHERE table_name = :table_name - """ - ), - {"table_name": table_name}, - ).first() - return row is not None - - -def upgrade(db: Session): - if not _column_exists(db, "organizations", "is_active"): - db.execute( - text( - """ - ALTER TABLE organizations - ADD COLUMN is_active BOOLEAN NOT NULL DEFAULT TRUE - """ - ) - ) - db.execute( - text( - """ - CREATE INDEX IF NOT EXISTS ix_organizations_is_active - ON organizations (is_active) - """ - ) - ) - print("Added organizations.is_active") - - if not _column_exists(db, "organizations", "disabled_at"): - db.execute( - text( - """ - ALTER TABLE organizations - ADD COLUMN disabled_at TIMESTAMP WITH TIME ZONE NULL - """ - ) - ) - print("Added organizations.disabled_at") - - if not _table_exists(db, "platform_admins"): - db.execute( - text( - """ - CREATE TABLE platform_admins ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - email VARCHAR(255) NOT NULL UNIQUE, - password_hash VARCHAR(255) NOT NULL, - is_active BOOLEAN NOT NULL DEFAULT TRUE, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - last_login_at TIMESTAMP WITH TIME ZONE NULL - ) - """ - ) - ) - db.execute( - text( - """ - CREATE INDEX IF NOT EXISTS ix_platform_admins_email - ON platform_admins (email) - """ - ) - ) - print("Added platform_admins table") - - if not _table_exists(db, "signup_reference_codes"): - db.execute( - text( - """ - CREATE TABLE signup_reference_codes ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - code_hash VARCHAR(64) NOT NULL UNIQUE, - label VARCHAR(255) NULL, - max_uses INTEGER NULL, - use_count INTEGER NOT NULL DEFAULT 0, - expires_at TIMESTAMP WITH TIME ZONE NULL, - is_active BOOLEAN NOT NULL DEFAULT TRUE, - created_by UUID NULL REFERENCES platform_admins(id) ON DELETE SET NULL, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() - ) - """ - ) - ) - db.execute( - text( - """ - CREATE INDEX IF NOT EXISTS ix_signup_reference_codes_is_active - ON signup_reference_codes (is_active) - """ - ) - ) - print("Added signup_reference_codes table") - - db.commit() - - -def downgrade(db: Session): - if _table_exists(db, "signup_reference_codes"): - db.execute(text("DROP TABLE signup_reference_codes")) - print("Dropped signup_reference_codes table") - - if _table_exists(db, "platform_admins"): - db.execute(text("DROP TABLE platform_admins")) - print("Dropped platform_admins table") - - if _column_exists(db, "organizations", "disabled_at"): - db.execute(text("ALTER TABLE organizations DROP COLUMN disabled_at")) - print("Dropped organizations.disabled_at") - - if _column_exists(db, "organizations", "is_active"): - db.execute(text("DROP INDEX IF EXISTS ix_organizations_is_active")) - db.execute(text("ALTER TABLE organizations DROP COLUMN is_active")) - print("Dropped organizations.is_active") - - db.commit() diff --git a/app/migrations/058_metric_studio_runs.py b/app/migrations/058_metric_studio_runs.py deleted file mode 100644 index 9c265096..00000000 --- a/app/migrations/058_metric_studio_runs.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Migration: Add Metrics Studio run tables.""" - -from sqlalchemy import text -from sqlalchemy.orm import Session - -description = "Add metric_studio_runs and metric_studio_run_results tables." - - -def _table_exists(db: Session, table_name: str) -> bool: - row = db.execute( - text( - """ - SELECT 1 - FROM information_schema.tables - WHERE table_name = :table_name - """ - ), - {"table_name": table_name}, - ).first() - return row is not None - - -def upgrade(db: Session): - if not _table_exists(db, "metric_studio_runs"): - db.execute( - text( - """ - CREATE TABLE metric_studio_runs ( - id UUID PRIMARY KEY, - organization_id UUID NOT NULL REFERENCES organizations(id), - workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE RESTRICT, - created_by_user_id UUID NULL REFERENCES users(id), - name VARCHAR(255) NULL, - selected_metric_ids JSON NOT NULL DEFAULT '[]', - selected_metric_groups JSON NULL, - transcript_source VARCHAR(20) NOT NULL DEFAULT 'diarised', - llm_provider VARCHAR(50) NULL, - llm_model VARCHAR(100) NULL, - llm_credential_id UUID NULL REFERENCES aiproviders(id) ON DELETE SET NULL, - llm_config JSON NULL, - metric_llm_overrides JSON NULL, - status VARCHAR(20) NOT NULL DEFAULT 'pending', - total_items INTEGER NOT NULL DEFAULT 0, - completed_items INTEGER NOT NULL DEFAULT 0, - failed_items INTEGER NOT NULL DEFAULT 0, - error_message TEXT NULL, - started_at TIMESTAMPTZ NULL, - finished_at TIMESTAMPTZ NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() - ) - """ - ) - ) - db.execute( - text( - "CREATE INDEX ix_metric_studio_runs_org ON metric_studio_runs (organization_id)" - ) - ) - db.execute( - text( - "CREATE INDEX ix_metric_studio_runs_workspace ON metric_studio_runs (workspace_id)" - ) - ) - db.execute( - text( - "CREATE INDEX ix_metric_studio_runs_status ON metric_studio_runs (status)" - ) - ) - print("Created metric_studio_runs") - - if not _table_exists(db, "metric_studio_run_results"): - db.execute( - text( - """ - CREATE TABLE metric_studio_run_results ( - id UUID PRIMARY KEY, - run_id UUID NOT NULL REFERENCES metric_studio_runs(id) ON DELETE CASCADE, - workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE RESTRICT, - source_kind VARCHAR(40) NOT NULL, - source_ref VARCHAR(255) NOT NULL, - display_label VARCHAR(512) NULL, - source_metadata JSON NULL, - status VARCHAR(20) NOT NULL DEFAULT 'pending', - metric_scores JSON NOT NULL DEFAULT '{}', - error_message TEXT NULL, - celery_task_id VARCHAR(255) NULL, - started_at TIMESTAMPTZ NULL, - finished_at TIMESTAMPTZ NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() - ) - """ - ) - ) - db.execute( - text( - "CREATE INDEX ix_metric_studio_run_results_run ON metric_studio_run_results (run_id)" - ) - ) - db.execute( - text( - "CREATE INDEX ix_metric_studio_run_results_status ON metric_studio_run_results (status)" - ) - ) - print("Created metric_studio_run_results") - - -def downgrade(db: Session): - if _table_exists(db, "metric_studio_run_results"): - db.execute(text("DROP TABLE metric_studio_run_results")) - if _table_exists(db, "metric_studio_runs"): - db.execute(text("DROP TABLE metric_studio_runs")) diff --git a/app/models/database.py b/app/models/database.py index ad8aa97e..a21a4819 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -57,8 +57,6 @@ class Organization(Base): judge_alignment_settings = Column(JSON, nullable=True) # Per-org LLM gateway overrides (enabled, gateway_type, base_url, keys). llm_gateway_settings = Column(JSON, nullable=True) - is_active = Column(Boolean, default=True, nullable=False, server_default=text("true"), index=True) - disabled_at = Column(DateTime(timezone=True), nullable=True) created_at = Column(DateTime(timezone=True), server_default=func.now()) updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) @@ -263,47 +261,6 @@ class User(Base): refresh_tokens = relationship("RefreshToken", back_populates="user", cascade="all, delete-orphan") -class PlatformAdmin(Base): - """Platform-level administrator (separate from org-scoped users).""" - - __tablename__ = "platform_admins" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - email = Column(String(255), unique=True, nullable=False, index=True) - password_hash = Column(String(255), nullable=False) - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - last_login_at = Column(DateTime(timezone=True), nullable=True) - - signup_reference_codes = relationship( - "SignupReferenceCode", - back_populates="created_by_admin", - foreign_keys="SignupReferenceCode.created_by", - ) - - -class SignupReferenceCode(Base): - """Single- or multi-use reference code required for gated self-service signup.""" - - __tablename__ = "signup_reference_codes" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - code_hash = Column(String(64), unique=True, nullable=False) - label = Column(String(255), nullable=True) - max_uses = Column(Integer, nullable=True) - use_count = Column(Integer, default=0, nullable=False) - expires_at = Column(DateTime(timezone=True), nullable=True) - is_active = Column(Boolean, default=True, nullable=False, index=True) - created_by = Column(UUID(as_uuid=True), ForeignKey("platform_admins.id"), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - created_by_admin = relationship( - "PlatformAdmin", - back_populates="signup_reference_codes", - foreign_keys=[created_by], - ) - - class RefreshToken(Base): """Opaque refresh token for extending local-password sessions.""" @@ -1025,17 +982,6 @@ class Metric(Base): capture_rationale = Column(Boolean, nullable=False, default=False) enabled = Column(Boolean, nullable=False, default=True) - - # Studio draft lifecycle: ``draft`` metrics are visible only in Metrics - # Studio until promoted to ``active``. - lifecycle = Column( - String(20), - nullable=False, - default="active", - server_default="active", - ) - promoted_from_draft_at = Column(DateTime(timezone=True), nullable=True) - studio_notes = Column(Text, nullable=True) # Metadata is_default = Column(Boolean, nullable=False, default=False) # Pre-defined metrics @@ -2556,107 +2502,6 @@ def _call_import_evaluation_row_fill_workspace_id(_mapper, connection, target): target.workspace_id = workspace_id -class MetricStudioRun(Base): - """Ad-hoc metric experiment run in Metrics Studio.""" - - __tablename__ = "metric_studio_runs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - created_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id"), nullable=True - ) - - name = Column(String(255), nullable=True) - selected_metric_ids = Column(JSON, nullable=False, default=list) - selected_metric_groups = Column(JSON, nullable=True) - transcript_source = Column( - String(20), - nullable=False, - default="diarised", - server_default="diarised", - ) - - llm_provider = Column(String(50), nullable=True) - llm_model = Column(String(100), nullable=True) - llm_credential_id = Column( - UUID(as_uuid=True), - ForeignKey("aiproviders.id", ondelete="SET NULL"), - nullable=True, - ) - llm_config = Column(JSON, nullable=True) - metric_llm_overrides = Column(JSON, nullable=True) - - status = Column(String(20), nullable=False, default="pending", index=True) - total_items = Column(Integer, nullable=False, default=0) - completed_items = Column(Integer, nullable=False, default=0) - failed_items = Column(Integer, nullable=False, default=0) - error_message = Column(Text, nullable=True) - - started_at = Column(DateTime(timezone=True), nullable=True) - finished_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - results = relationship( - "MetricStudioRunResult", - back_populates="run", - cascade="all, delete-orphan", - ) - - -class MetricStudioRunResult(Base): - """Per-source scoring output for a MetricStudioRun.""" - - __tablename__ = "metric_studio_run_results" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - run_id = Column( - UUID(as_uuid=True), - ForeignKey("metric_studio_runs.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - source_kind = Column(String(40), nullable=False) - source_ref = Column(String(255), nullable=False) - display_label = Column(String(512), nullable=True) - source_metadata = Column(JSON, nullable=True) - - status = Column(String(20), nullable=False, default="pending", index=True) - metric_scores = Column(JSON, nullable=False, default=dict) - error_message = Column(Text, nullable=True) - celery_task_id = Column(String(255), nullable=True) - - started_at = Column(DateTime(timezone=True), nullable=True) - finished_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - run = relationship("MetricStudioRun", back_populates="results") - - class CallImportEvaluationReportSnapshot(Base): """Persisted PDF-report aggregate used for period-over-period deltas.""" diff --git a/app/models/enums.py b/app/models/enums.py index 140ffe48..86205f5d 100644 --- a/app/models/enums.py +++ b/app/models/enums.py @@ -182,13 +182,6 @@ class MetricCategory(str, enum.Enum): USER_INSIGHT = "user_insight" -class MetricLifecycle(str, enum.Enum): - """Lifecycle state for metrics — drafts are Studio-only until promoted.""" - - ACTIVE = "active" - DRAFT = "draft" - - class MetricTrigger(str, enum.Enum): """Metric trigger enumeration.""" ALWAYS = "always" diff --git a/app/models/schemas.py b/app/models/schemas.py index 4fea3e7e..78dd14f9 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -1966,9 +1966,6 @@ class MetricResponse(BaseModel): # render a "Compare transcripts" badge in the metric picker and # know to skip the run's transcript_source toggle for this metric. compare_transcripts: bool = False - lifecycle: str = "active" - promoted_from_draft_at: Optional[datetime] = None - studio_notes: Optional[str] = None children: List["MetricResponse"] = Field(default_factory=list) created_at: datetime updated_at: datetime @@ -2030,130 +2027,6 @@ def normalize_metric_origin(cls, v): MetricResponse.model_rebuild() -class MetricDraftCreate(MetricCreate): - """Create a draft metric for Metrics Studio experimentation.""" - - studio_notes: Optional[str] = Field( - default=None, - description="Optional notes about what this draft is testing.", - ) - - -class MetricDraftCreateWithChildren(MetricCreateWithChildren): - """Atomically create a draft parent category metric plus its children.""" - - studio_notes: Optional[str] = Field( - default=None, - description="Optional notes about what this draft category is testing.", - ) - - -class MetricPromoteResponse(BaseModel): - """Response after promoting a draft metric to active.""" - - metric: MetricResponse - promoted_at: datetime - - -MetricStudioSourceKind = Literal[ - "call_import_row", "call_recording", "evaluator_result" -] - - -class MetricStudioSourceItem(BaseModel): - """One call source selected for a Studio run.""" - - source_kind: MetricStudioSourceKind - source_ref: str = Field( - ..., - min_length=1, - description="UUID for import rows / evaluator results; call_short_id for recordings.", - ) - display_label: Optional[str] = Field( - default=None, - max_length=512, - description="Optional UI label; resolved server-side when omitted.", - ) - - -class MetricStudioRunCreate(BaseModel): - """Request body for triggering a Metrics Studio evaluation run.""" - - metric_ids: List[UUID] = Field(..., min_length=1) - sources: List[MetricStudioSourceItem] = Field(..., min_length=1) - name: Optional[str] = Field(default=None, max_length=255) - transcript_source: Literal["production", "diarised"] = "diarised" - llm_provider: Optional[str] = Field(default=None, max_length=50) - llm_model: Optional[str] = Field(default=None, max_length=100) - llm_credential_id: Optional[UUID] = None - llm_config: Optional[Dict[str, Any]] = None - metric_llm_overrides: Optional[Dict[str, Any]] = None - - -class MetricStudioRunRetryRequest(BaseModel): - """Retry failed or selected Studio run results.""" - - result_ids: Optional[List[UUID]] = Field( - default=None, - description="When omitted, retry all failed results in the run.", - ) - - -class MetricStudioRunResultResponse(BaseModel): - """Per-source result row for a Studio run.""" - - id: UUID - run_id: UUID - source_kind: str - source_ref: str - display_label: Optional[str] = None - source_metadata: Optional[Dict[str, Any]] = None - status: str - metric_scores: Dict[str, Any] = Field(default_factory=dict) - error_message: Optional[str] = None - started_at: Optional[datetime] = None - finished_at: Optional[datetime] = None - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class MetricStudioRunResponse(BaseModel): - """Metrics Studio run summary.""" - - id: UUID - organization_id: UUID - workspace_id: UUID - name: Optional[str] = None - selected_metric_ids: List[str] = Field(default_factory=list) - selected_metric_groups: Optional[Dict[str, List[str]]] = None - transcript_source: str - llm_provider: Optional[str] = None - llm_model: Optional[str] = None - status: str - total_items: int - completed_items: int - failed_items: int - error_message: Optional[str] = None - started_at: Optional[datetime] = None - finished_at: Optional[datetime] = None - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class MetricStudioRunListResponse(BaseModel): - items: List[MetricStudioRunResponse] - total: int - - -class MetricStudioRunResultListResponse(BaseModel): - items: List[MetricStudioRunResultResponse] - total: int - - # Evaluator Result Schemas class EvaluatorResultCreate(BaseModel): """Schema for creating an evaluator result.""" diff --git a/app/services/metric_studio/__init__.py b/app/services/metric_studio/__init__.py deleted file mode 100644 index 5d04d692..00000000 --- a/app/services/metric_studio/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Metrics Studio services.""" diff --git a/app/services/metric_studio/metric_selection.py b/app/services/metric_studio/metric_selection.py deleted file mode 100644 index 07ea4834..00000000 --- a/app/services/metric_studio/metric_selection.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Metric selection helpers for Metrics Studio runs.""" - -from __future__ import annotations - -from typing import Dict, List, Tuple -from uuid import UUID - -from sqlalchemy.orm import Session - -from app.models.database import Metric - - -def expand_studio_metric_selection( - db: Session, - org_id: UUID, - selected_ids: List[UUID], -) -> Tuple[List[Metric], Dict[UUID, List[Metric]]]: - """Like call-import metric expansion but allows draft/disabled metrics.""" - if not selected_ids: - return [], {} - - requested = list(selected_ids) - initial_rows = ( - db.query(Metric) - .filter( - Metric.organization_id == org_id, - Metric.id.in_(requested), - ) - .all() - ) - initial_by_id = {row.id: row for row in initial_rows} - - parent_ids_requested = { - m.id for m in initial_rows if m.selection_mode and not m.parent_metric_id - } - explicit_children_by_parent: Dict[UUID, List[Metric]] = {} - for m in initial_rows: - if m.parent_metric_id and m.parent_metric_id in parent_ids_requested: - explicit_children_by_parent.setdefault(m.parent_metric_id, []).append(m) - - parents_needing_full_expansion = [ - pid for pid in parent_ids_requested if pid not in explicit_children_by_parent - ] - auto_expanded_children: Dict[UUID, List[Metric]] = {} - if parents_needing_full_expansion: - for pid in parents_needing_full_expansion: - child_rows = ( - db.query(Metric) - .filter( - Metric.organization_id == org_id, - Metric.parent_metric_id == pid, - ) - .order_by(Metric.created_at.asc()) - .all() - ) - auto_expanded_children[pid] = child_rows - - parent_to_children: Dict[UUID, List[Metric]] = {} - for pid in parent_ids_requested: - children = explicit_children_by_parent.get(pid) or auto_expanded_children.get( - pid, [] - ) - parent_to_children[pid] = list(children) - - effective: List[Metric] = [] - seen: set[UUID] = set() - for mid in requested: - m = initial_by_id.get(mid) - if m is None: - continue - if m.selection_mode and not m.parent_metric_id: - for child in parent_to_children.get(m.id, []): - if child.id in seen: - continue - seen.add(child.id) - effective.append(child) - elif m.parent_metric_id is None or m.parent_metric_id not in parent_ids_requested: - if m.id in seen: - continue - seen.add(m.id) - effective.append(m) - - return effective, parent_to_children - - -def load_studio_run_metrics( - db: Session, - organization_id: UUID, - metric_ids: List[UUID], -) -> List[Metric]: - if not metric_ids: - return [] - return ( - db.query(Metric) - .filter( - Metric.organization_id == organization_id, - Metric.id.in_(metric_ids), - ) - .all() - ) diff --git a/app/services/metric_studio/source_resolver.py b/app/services/metric_studio/source_resolver.py deleted file mode 100644 index b8b48fa1..00000000 --- a/app/services/metric_studio/source_resolver.py +++ /dev/null @@ -1,273 +0,0 @@ -"""Resolve heterogeneous call sources into a common evaluation sample.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any, Dict, Optional -from uuid import UUID - -from fastapi import HTTPException -from sqlalchemy.orm import Session - -from app.api.v1.routes.playground import extract_transcript_from_call_data -from app.models.database import ( - Agent, - CallImport, - CallImportRow, - CallRecording, - Evaluator, - EvaluatorResult, - Persona, - Scenario, -) - - -@dataclass -class ResolvedCallSample: - source_kind: str - source_ref: str - label: str - transcript: Optional[str] - diarised_transcript: Optional[str] - audio_s3_key: Optional[str] - call_data: Optional[dict] - agent_id: Optional[UUID] - metadata: Dict[str, Any] = field(default_factory=dict) - - -def _resolve_call_import_row( - db: Session, - *, - organization_id: UUID, - workspace_id: UUID, - source_ref: str, - display_label: Optional[str], -) -> ResolvedCallSample: - try: - row_id = UUID(source_ref) - except ValueError as exc: - raise HTTPException(status_code=400, detail="Invalid call_import_row id.") from exc - - from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row - - row_db = None - extra_catalog = None - try: - try: - row_db, located_catalog, row, _shard_id = locate_call_import_row(row_id) - extra_catalog = located_catalog if located_catalog is not row_db else None - except LookupError as exc: - raise HTTPException(status_code=404, detail="Call import row not found.") from exc - - if row.organization_id != organization_id: - raise HTTPException(status_code=404, detail="Call import row not found.") - - call_import = ( - db.query(CallImport) - .filter( - CallImport.id == row.call_import_id, - CallImport.organization_id == organization_id, - CallImport.workspace_id == workspace_id, - ) - .first() - ) - if not call_import: - raise HTTPException(status_code=404, detail="Call import row not found.") - - label = display_label or row.conversation_id or f"Import row {row.row_index}" - return ResolvedCallSample( - source_kind="call_import_row", - source_ref=str(row.id), - label=label, - transcript=(row.transcript or "").strip() or None, - diarised_transcript=(row.diarised_transcript or "").strip() or None, - audio_s3_key=(row.recording_s3_key or "").strip() or None, - call_data=None, - agent_id=None, - metadata={ - "call_import_id": str(row.call_import_id), - "call_import_name": getattr(call_import, "name", None), - "row_index": row.row_index, - "conversation_id": row.conversation_id, - }, - ) - finally: - if row_db is not None: - close_row_sessions(row_db, extra_catalog) - - -def _resolve_call_recording( - db: Session, - *, - organization_id: UUID, - workspace_id: UUID, - source_ref: str, - display_label: Optional[str], -) -> ResolvedCallSample: - recording = ( - db.query(CallRecording) - .filter( - CallRecording.call_short_id == source_ref, - CallRecording.organization_id == organization_id, - CallRecording.workspace_id == workspace_id, - ) - .first() - ) - if not recording: - raise HTTPException(status_code=404, detail="Call recording not found.") - - call_data = recording.call_data if isinstance(recording.call_data, dict) else {} - platform = (recording.provider_platform or "").lower() - transcript_text, _ = extract_transcript_from_call_data(call_data, platform) - audio_s3_key = None - if recording.evaluator_result_id: - result = ( - db.query(EvaluatorResult) - .filter(EvaluatorResult.id == recording.evaluator_result_id) - .first() - ) - if result and result.audio_s3_key: - audio_s3_key = result.audio_s3_key - - label = display_label or recording.call_short_id - return ResolvedCallSample( - source_kind="call_recording", - source_ref=recording.call_short_id, - label=label, - transcript=transcript_text or None, - diarised_transcript=transcript_text or None, - audio_s3_key=audio_s3_key, - call_data=call_data or None, - agent_id=recording.agent_id, - metadata={ - "call_short_id": recording.call_short_id, - "provider_platform": recording.provider_platform, - "source": getattr(recording.source, "value", recording.source), - }, - ) - - -def _resolve_evaluator_result( - db: Session, - *, - organization_id: UUID, - workspace_id: UUID, - source_ref: str, - display_label: Optional[str], -) -> ResolvedCallSample: - result = None - try: - result_uuid = UUID(source_ref) - result = ( - db.query(EvaluatorResult) - .filter( - EvaluatorResult.id == result_uuid, - EvaluatorResult.organization_id == organization_id, - EvaluatorResult.workspace_id == workspace_id, - ) - .first() - ) - except ValueError: - result = ( - db.query(EvaluatorResult) - .filter( - EvaluatorResult.result_id == source_ref, - EvaluatorResult.organization_id == organization_id, - EvaluatorResult.workspace_id == workspace_id, - ) - .first() - ) - - if not result: - raise HTTPException(status_code=404, detail="Evaluator result not found.") - - persona_name = None - scenario_name = None - evaluator_name = None - if result.persona_id: - persona = db.query(Persona).filter(Persona.id == result.persona_id).first() - persona_name = persona.name if persona else None - if result.scenario_id: - scenario = db.query(Scenario).filter(Scenario.id == result.scenario_id).first() - scenario_name = scenario.name if scenario else None - if result.evaluator_id: - evaluator = db.query(Evaluator).filter(Evaluator.id == result.evaluator_id).first() - evaluator_name = evaluator.name if evaluator else None - - label = display_label or result.name or result.result_id - return ResolvedCallSample( - source_kind="evaluator_result", - source_ref=str(result.id), - label=label, - transcript=(result.transcription or "").strip() or None, - diarised_transcript=(result.transcription or "").strip() or None, - audio_s3_key=(result.audio_s3_key or "").strip() or None, - call_data=result.call_data if isinstance(result.call_data, dict) else None, - agent_id=result.agent_id, - metadata={ - "result_id": result.result_id, - "evaluator_id": str(result.evaluator_id) if result.evaluator_id else None, - "evaluator_name": evaluator_name, - "persona_id": str(result.persona_id) if result.persona_id else None, - "persona_name": persona_name, - "scenario_id": str(result.scenario_id) if result.scenario_id else None, - "scenario_name": scenario_name, - "agent_id": str(result.agent_id) if result.agent_id else None, - }, - ) - - -def resolve_source( - db: Session, - *, - organization_id: UUID, - workspace_id: UUID, - source_kind: str, - source_ref: str, - display_label: Optional[str] = None, -) -> ResolvedCallSample: - if source_kind == "call_import_row": - return _resolve_call_import_row( - db, - organization_id=organization_id, - workspace_id=workspace_id, - source_ref=source_ref, - display_label=display_label, - ) - if source_kind == "call_recording": - return _resolve_call_recording( - db, - organization_id=organization_id, - workspace_id=workspace_id, - source_ref=source_ref, - display_label=display_label, - ) - if source_kind == "evaluator_result": - return _resolve_evaluator_result( - db, - organization_id=organization_id, - workspace_id=workspace_id, - source_ref=source_ref, - display_label=display_label, - ) - raise HTTPException(status_code=400, detail=f"Unknown source_kind: {source_kind}") - - -def preview_source_label( - db: Session, - *, - organization_id: UUID, - workspace_id: UUID, - source_kind: str, - source_ref: str, - display_label: Optional[str] = None, -) -> str: - sample = resolve_source( - db, - organization_id=organization_id, - workspace_id=workspace_id, - source_kind=source_kind, - source_ref=source_ref, - display_label=display_label, - ) - return sample.label diff --git a/app/services/signup_reference_codes.py b/app/services/signup_reference_codes.py deleted file mode 100644 index b7f9027c..00000000 --- a/app/services/signup_reference_codes.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Signup reference code hashing and validation.""" - -from __future__ import annotations - -import hashlib -from datetime import datetime, timezone -from typing import Optional - -from fastapi import HTTPException, status -from sqlalchemy.orm import Session - -from app.config import settings -from app.models.database import SignupReferenceCode - - -def hash_reference_code(code: str) -> str: - normalized = code.strip().upper() - payload = f"{settings.SECRET_KEY}:signup_ref:{normalized}" - return hashlib.sha256(payload.encode("utf-8")).hexdigest() - - -def _to_aware_utc(dt: datetime) -> datetime: - if dt.tzinfo is None: - return dt.replace(tzinfo=timezone.utc) - return dt.astimezone(timezone.utc) - - -def _code_is_usable(row: SignupReferenceCode, *, now: datetime) -> bool: - if not row.is_active: - return False - if row.expires_at is not None and _to_aware_utc(row.expires_at) <= now: - return False - if row.max_uses is not None and row.use_count >= row.max_uses: - return False - return True - - -def validate_reference_code_for_signup(db: Session, code: Optional[str]) -> SignupReferenceCode: - if not code or not code.strip(): - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="A valid reference code is required to sign up.", - ) - - code_hash = hash_reference_code(code) - row = ( - db.query(SignupReferenceCode) - .filter(SignupReferenceCode.code_hash == code_hash) - .first() - ) - now = datetime.now(timezone.utc) - if row is None or not _code_is_usable(row, now=now): - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="A valid reference code is required to sign up.", - ) - return row - - -def consume_reference_code(db: Session, row: SignupReferenceCode) -> None: - row.use_count = (row.use_count or 0) + 1 diff --git a/app/workers/config.py b/app/workers/config.py index f4e44f36..dc1cb711 100644 --- a/app/workers/config.py +++ b/app/workers/config.py @@ -135,7 +135,6 @@ "generate_evaluation_user_insights": {"queue": "evaluations"}, "generate_evaluation_metric_clusters": {"queue": "evaluations"}, "generate_evaluation_prompt_improvements": {"queue": "evaluations"}, - "evaluate_studio_run_item": {"queue": "evaluations"}, "generate_agent_flowchart": {"queue": "celery"}, "map_agent_flowchart_prompt_sections": {"queue": "celery"}, } diff --git a/app/workers/tasks/__init__.py b/app/workers/tasks/__init__.py index e60a9ad2..e4862d8e 100644 --- a/app/workers/tasks/__init__.py +++ b/app/workers/tasks/__init__.py @@ -21,7 +21,6 @@ from . import agent_flowchart_jobs from . import initiate_vobiz_outbound from . import finalize_telephony_recording -from . import evaluate_studio_run_item from . import call_import_bulk_ops from app.workers.concurrency import eval_dispatch from app.workers.concurrency import fair_dispatch @@ -53,7 +52,6 @@ "dispatch_evaluation_rows_task", "dispatch_fair_eval_rows_task", "dispatch_fair_diarization_rows_task", - "evaluate_studio_run_item_task", "dispatch_fair_import_rows_task", "bulk_diarize_call_import_task", "bulk_delete_call_import_rows_task", @@ -114,9 +112,6 @@ call_import_bulk_ops.materialize_call_import_rows_task ) delete_call_import_task = call_import_bulk_ops.delete_call_import_task -evaluate_studio_run_item_task = ( - evaluate_studio_run_item.evaluate_studio_run_item_task -) materialize_call_import_evaluation_task = ( call_import_bulk_ops.materialize_call_import_evaluation_task ) diff --git a/app/workers/tasks/evaluate_call_import_row_core.py b/app/workers/tasks/evaluate_call_import_row_core.py index ad0f42ae..81f71199 100644 --- a/app/workers/tasks/evaluate_call_import_row_core.py +++ b/app/workers/tasks/evaluate_call_import_row_core.py @@ -6,7 +6,7 @@ from typing import Any, List, Optional from uuid import UUID -from sqlalchemy import case, func, or_, update +from sqlalchemy import case, func, update from sqlalchemy.orm import Session from app.models.database import ( @@ -489,7 +489,6 @@ def load_enabled_metrics( Metric.organization_id == evaluation.organization_id, Metric.id.in_(metric_ids), Metric.enabled.is_(True), - or_(Metric.lifecycle.is_(None), Metric.lifecycle == "active"), ) .all() ) diff --git a/app/workers/tasks/evaluate_studio_run_item.py b/app/workers/tasks/evaluate_studio_run_item.py deleted file mode 100644 index 04705867..00000000 --- a/app/workers/tasks/evaluate_studio_run_item.py +++ /dev/null @@ -1,273 +0,0 @@ -"""Celery task: evaluate one Metrics Studio run result.""" - -from __future__ import annotations - -from datetime import datetime, timezone -from typing import Any -from uuid import UUID - -from loguru import logger -from sqlalchemy.orm import Session -from sqlalchemy.orm.attributes import flag_modified - -from app.database import SessionLocal -from app.models.database import ( - AIProvider, - MetricStudioRun, - MetricStudioRunResult, -) -from app.services.metric_studio.metric_selection import load_studio_run_metrics -from app.services.metric_studio.source_resolver import resolve_source -from app.workers.config import celery_app -from app.workers.tasks.evaluate_call_import_row_core import ( - build_parent_groups, - categorize_metrics, -) -from app.workers.tasks.helpers.audio_evaluation import ( - evaluate_audio_metrics, - handle_audio_evaluation_error, -) -from app.workers.tasks.helpers.llm_evaluation import ( - evaluate_with_llm, - handle_llm_evaluation_error, -) - - -def _now_utc() -> datetime: - return datetime.now(timezone.utc) - - -def _rollup_run(db: Session, run: MetricStudioRun) -> None: - results = ( - db.query(MetricStudioRunResult) - .filter(MetricStudioRunResult.run_id == run.id) - .all() - ) - completed = sum(1 for r in results if r.status == "completed") - failed = sum(1 for r in results if r.status == "failed") - pending = sum(1 for r in results if r.status in {"pending", "running"}) - run.completed_items = completed - run.failed_items = failed - if pending: - run.status = "running" - elif failed and completed: - run.status = "partial" - run.finished_at = _now_utc() - elif failed: - run.status = "failed" - run.finished_at = _now_utc() - else: - run.status = "completed" - run.finished_at = _now_utc() - db.commit() - - -@celery_app.task( - bind=True, - name="evaluate_studio_run_item", - max_retries=0, -) -def evaluate_studio_run_item_task(self, result_row_id: str) -> dict[str, Any]: - db = SessionLocal() - try: - try: - row_uuid = UUID(result_row_id) - except ValueError: - return {"status": "error", "detail": "invalid result id"} - - result_row = ( - db.query(MetricStudioRunResult) - .filter(MetricStudioRunResult.id == row_uuid) - .first() - ) - if not result_row: - return {"status": "error", "detail": "result not found"} - - run = ( - db.query(MetricStudioRun) - .filter(MetricStudioRun.id == result_row.run_id) - .first() - ) - if not run: - return {"status": "error", "detail": "run not found"} - - result_row.status = "running" - result_row.started_at = result_row.started_at or _now_utc() - db.commit() - - sample = resolve_source( - db, - organization_id=run.organization_id, - workspace_id=run.workspace_id, - source_kind=result_row.source_kind, - source_ref=result_row.source_ref, - display_label=result_row.display_label, - ) - - transcript_source = (run.transcript_source or "diarised").lower() - if transcript_source == "production": - transcript = sample.transcript - else: - transcript = sample.diarised_transcript or sample.transcript - - metric_ids = [] - for item in run.selected_metric_ids or []: - try: - metric_ids.append(UUID(str(item))) - except (TypeError, ValueError): - continue - - metrics = load_studio_run_metrics(db, run.organization_id, metric_ids) - has_audio = bool(sample.audio_s3_key) - has_production = bool((sample.transcript or "").strip()) - has_diarised = bool((sample.diarised_transcript or "").strip()) - - transcript_metrics, audio_metrics, comparison_metrics, skipped = categorize_metrics( - metrics, - has_audio, - has_production_transcript=has_production, - has_diarised_transcript=has_diarised, - ) - llm_metrics = transcript_metrics + comparison_metrics - metric_scores: dict[str, Any] = dict(skipped) - - if not transcript and not has_audio: - result_row.status = "failed" - result_row.error_message = "No transcript or audio available for this source." - result_row.finished_at = _now_utc() - db.commit() - _rollup_run(db, run) - return {"status": "failed"} - - ai_providers = ( - db.query(AIProvider) - .filter( - AIProvider.organization_id == run.organization_id, - AIProvider.is_active.is_(True), - ) - .all() - ) - - if audio_metrics and sample.audio_s3_key: - try: - audio_scores = evaluate_audio_metrics( - audio_s3_key=sample.audio_s3_key, - audio_metrics=audio_metrics, - result_id=f"studio:{result_row.id}", - ) - metric_scores.update(audio_scores) - except Exception as audio_err: - logger.error( - f"[MetricStudio {result_row.id}] audio evaluation failed: {audio_err}", - exc_info=True, - ) - metric_scores.update( - handle_audio_evaluation_error(audio_metrics, audio_err) - ) - - if llm_metrics and transcript: - parents_by_id, children_by_parent, standalone = build_parent_groups( - db, llm_metrics - ) - result_id = f"studio:{result_row.id}" - - for parent_id, children in children_by_parent.items(): - parent = parents_by_id.get(parent_id) - if not parent or not children: - continue - try: - comparison_pair = None - if any(getattr(m, "compare_transcripts", False) for m in children): - comparison_pair = ( - sample.transcript or "", - sample.diarised_transcript or sample.transcript or "", - ) - scores, _ = evaluate_with_llm( - transcription=transcript, - llm_metrics=children, - ai_providers=ai_providers, - organization_id=run.organization_id, - result_id=result_id, - db=db, - parent_metric=parent, - comparison_pair=comparison_pair, - ) - metric_scores.update(scores) - except Exception as llm_err: - metric_scores.update( - handle_llm_evaluation_error(children, llm_err) - ) - - if standalone: - try: - comparison_standalone = [ - m - for m in standalone - if getattr(m, "compare_transcripts", False) - ] - transcript_standalone = [ - m for m in standalone if m not in comparison_standalone - ] - if transcript_standalone: - scores, _ = evaluate_with_llm( - transcription=transcript, - llm_metrics=transcript_standalone, - ai_providers=ai_providers, - organization_id=run.organization_id, - result_id=result_id, - db=db, - ) - metric_scores.update(scores) - for metric in comparison_standalone: - scores, _ = evaluate_with_llm( - transcription=transcript, - llm_metrics=[metric], - ai_providers=ai_providers, - organization_id=run.organization_id, - result_id=result_id, - db=db, - comparison_pair=( - sample.transcript or "", - sample.diarised_transcript or sample.transcript or "", - ), - ) - metric_scores.update(scores) - except Exception as llm_err: - metric_scores.update(handle_llm_evaluation_error(standalone, llm_err)) - - result_row.metric_scores = metric_scores - flag_modified(result_row, "metric_scores") - result_row.status = "completed" - result_row.error_message = None - result_row.finished_at = _now_utc() - db.commit() - _rollup_run(db, run) - return {"status": "completed", "scores": len(metric_scores)} - except Exception as exc: - logger.error( - f"[MetricStudio] evaluate_studio_run_item failed: {exc}", - exc_info=True, - ) - try: - result_row = ( - db.query(MetricStudioRunResult) - .filter(MetricStudioRunResult.id == UUID(result_row_id)) - .first() - ) - if result_row: - result_row.status = "failed" - result_row.error_message = str(exc) - result_row.finished_at = _now_utc() - db.commit() - run = ( - db.query(MetricStudioRun) - .filter(MetricStudioRun.id == result_row.run_id) - .first() - ) - if run: - _rollup_run(db, run) - except Exception: - db.rollback() - raise - finally: - db.close() diff --git a/app/workers/tasks/process_evaluator_result.py b/app/workers/tasks/process_evaluator_result.py index 89c8184f..d338f294 100644 --- a/app/workers/tasks/process_evaluator_result.py +++ b/app/workers/tasks/process_evaluator_result.py @@ -5,7 +5,6 @@ from uuid import UUID from loguru import logger -from sqlalchemy import or_ from app.database import SessionLocal from app.models.database import ModelProvider @@ -543,7 +542,6 @@ def process_evaluator_result_task(self, result_id: str): enabled_metrics = db.query(Metric).filter( Metric.organization_id == result.organization_id, Metric.enabled == True, - or_(Metric.lifecycle.is_(None), Metric.lifecycle == "active"), ).all() enabled_metrics = [ m for m in enabled_metrics diff --git a/config.yml.example b/config.yml.example index 91b43e2e..935737e6 100644 --- a/config.yml.example +++ b/config.yml.example @@ -164,9 +164,6 @@ auth: refresh_token_ttl_days: 7 # Turn this off in Cloud SaaS to block self-serve signup. allow_signup: true - # When enabled, signup requires a valid reference code from the platform admin API. - gated_signup: - enabled: false # External OIDC (enterprise): bring your own IdP. Works with any # OIDC-compliant provider. See README.md > Authentication & Deployment diff --git a/docs-fumadocs/content/docs/products/metrics-studio.mdx b/docs-fumadocs/content/docs/products/metrics-studio.mdx deleted file mode 100644 index e9992b46..00000000 --- a/docs-fumadocs/content/docs/products/metrics-studio.mdx +++ /dev/null @@ -1,10 +0,0 @@ -# Metrics Studio lets users experiment with metrics against ad-hoc call sources. - -Metrics Studio is a sub-tab under **Metrics → Studio**. Use it to: - -1. Select active or **draft** metrics -2. Pick call sources from call imports, playground/observability recordings, or simulated evaluator results -3. Run ad-hoc evaluations and inspect per-metric scores -4. Promote draft metrics to production when satisfied - -Draft metrics stay hidden from Call Import evaluations and the Manage tab until promoted. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 3d5a7112..2d745a02 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -7,8 +7,6 @@ import Layout from './components/Layout' import Login from './pages/auth/Login' import LoginCallback from './pages/auth/LoginCallback' import SelectOrganization from './pages/auth/SelectOrganization' -import PlatformLogin from './pages/platform/PlatformLogin' -import PlatformAdmin from './pages/platform/PlatformAdmin' // Dashboard import Dashboard from './pages/dashboard/Dashboard' @@ -26,10 +24,8 @@ import Personas from './pages/personas/Personas' import Scenarios from './pages/scenarios/Scenarios' // Metrics -import MetricsLayout from './pages/metrics/MetricsLayout' +import Metrics from './pages/metrics/Metrics' import MetricsManagement from './pages/metrics/MetricsManagement' -import MetricsStudio from './pages/metrics/MetricsStudio' -import MetricsStudioRunDetail from './pages/metrics/MetricsStudioRunDetail' // Playground - Agent import AgentPlayground from './pages/playground/agent/AgentPlayground' @@ -137,8 +133,6 @@ function App() { } /> - } /> - } /> } /> } /> {/* Public blind test form - intentionally outside PrivateRoute and EnterpriseGate. @@ -164,18 +158,14 @@ function App() { } /> } /> } /> - } /> + } /> } /> } /> } /> } /> } /> } /> - }> - } /> - } /> - } /> - + } /> } /> } /> } /> diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index 0103a80d..bfafc1b5 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -146,15 +146,6 @@ function getFlattenedNavItems(): NavItem[] { return items } -function isNavItemActive(href: string, pathname: string): boolean { - if (href === '/metrics-management') { - return ( - pathname === href || pathname.startsWith('/metrics-management/') - ) - } - return pathname === href -} - export default function Layout() { const location = useLocation() const { logout } = useAuthStore() @@ -484,7 +475,7 @@ function SidebarContent({ ))} @@ -535,7 +526,7 @@ function SidebarContent({