diff --git a/src/gumloop/cli/commands/agents.py b/src/gumloop/cli/commands/agents.py index dce14fb..df87526 100644 --- a/src/gumloop/cli/commands/agents.py +++ b/src/gumloop/cli/commands/agents.py @@ -550,3 +550,118 @@ def detach_mcp_server( console.print(f"[green]Detached[/green] MCP server {escape_markup(server_id)} from {agent_id}") console.print(f" detached: {'true' if response.detached else 'false'}", markup=False, highlight=False) + + +@agents_app.command("eval-options", epilog="Example:\n gumloop agents eval-options --json") +def get_evaluation_options( + ctx: typer.Context, + json_output: Annotated[ + bool, + typer.Option("--json", help="Print the raw SDK response as JSON."), + ] = False, +) -> None: + """Show the available agent evaluation settings.""" + cli: CliContext = ctx.obj + try: + response = cli.call_with_refresh(lambda client: client.agents.get_evaluation_options()) + except GumloopError as error: + exit_with_error(error, json_output=json_output) + + if json_output: + print_json(response) + return + + for field in ( + "interaction_types", + "criterion_types", + "criterion_priorities", + "data_point_types", + "frequencies", + "default_interaction_types", + ): + console.print(f"{field}: {', '.join(getattr(response, field))}", markup=False, highlight=False) + for name, value in response.limits.model_dump().items(): + console.print(f"limits.{name}: {value}", markup=False, highlight=False) + + +@agents_app.command( + "eval-metrics", + epilog="Example:\n gumloop agents eval-metrics agent_abc --days 30 --json", +) +def get_evaluation_metrics( + ctx: typer.Context, + agent_id: Annotated[str, typer.Argument(help="ID of the agent whose evaluation metrics should be shown.")], + days: Annotated[ + int, + typer.Option("--days", help="Number of days to include."), + ] = 30, + json_output: Annotated[ + bool, + typer.Option("--json", help="Print the raw SDK response as JSON."), + ] = False, +) -> None: + """Show an agent's evaluation grade and tag counts.""" + cli: CliContext = ctx.obj + try: + response = cli.call_with_refresh(lambda client: client.agents.get_evaluation_metrics(agent_id, days=days)) + except GumloopError as error: + exit_with_error(error, json_output=json_output) + + if json_output: + print_json(response) + return + + console.print("GRADE", "COUNT", sep="\t", soft_wrap=True) + for grade, count in response.grades.items(): + console.print(grade, str(count), sep="\t", soft_wrap=True) + console.print("TAG", "COUNT", sep="\t", soft_wrap=True) + for tag, count in response.tags.items(): + console.print(tag, str(count), sep="\t", soft_wrap=True) + + +@agents_app.command( + "eval-run", + epilog=( + "Examples:\n" + " gumloop agents eval-run agent_abc --session-id session_1\n" + " gumloop agents eval-run agent_abc --session-id session_1 --session-id session_2 --json" + ), +) +def run_evaluations( + ctx: typer.Context, + agent_id: Annotated[str, typer.Argument(help="ID of the agent that owns the sessions.")], + session_ids: Annotated[ + list[str] | None, + typer.Option("--session-id", help="Session ID to evaluate. Repeat for multiple sessions."), + ] = None, + json_output: Annotated[ + bool, + typer.Option("--json", help="Print the raw SDK response as JSON."), + ] = False, +) -> None: + """Queue evaluations for one or more agent sessions.""" + cli: CliContext = ctx.obj + try: + if not session_ids: + raise GumloopError("Pass at least one --session-id.") + response = cli.call_with_refresh(lambda client: client.agents.run_evaluations(agent_id, session_ids)) + except GumloopError as error: + exit_with_error(error, json_output=json_output) + + if json_output: + print_json(response) + return + + console.print(f"[green]Queued[/green] {response.queued} evaluation(s)") + if response.skipped.ineligible: + console.print( + f" Ineligible: {', '.join(response.skipped.ineligible)}", + markup=False, + highlight=False, + ) + if response.skipped.in_flight: + console.print( + f" In flight: {', '.join(response.skipped.in_flight)}", + markup=False, + highlight=False, + ) diff --git a/src/gumloop/resources/agents.py b/src/gumloop/resources/agents.py index 90b13b3..93333ee 100644 --- a/src/gumloop/resources/agents.py +++ b/src/gumloop/resources/agents.py @@ -2,6 +2,7 @@ from collections.abc import Mapping from collections.abc import Sequence +from datetime import datetime from typing import Any from gumloop._http import AsyncHttpClient @@ -18,8 +19,11 @@ from gumloop.types import AgentVersionsResponse from gumloop.types import EvaluationConfigResponse from gumloop.types import EvaluationConfigUpdateRequest +from gumloop.types import EvaluationMetrics +from gumloop.types import EvaluationOptions from gumloop.types import EvaluationResultListResponse from gumloop.types import EvaluationResultResponse +from gumloop.types import EvaluationRunResult from gumloop.types import ModelListResponse from gumloop.types import SkillListResponse @@ -132,6 +136,9 @@ def detach_mcp_server(self, agent_id: str, server_id: str) -> AgentMcpServerDeta def list_mcp_servers(self, agent_id: str) -> AgentMcpServersResponse: return AgentMcpServersResponse.model_validate(self._client.get(f"agents/{agent_id}/mcp-servers")) + def get_evaluation_options(self) -> EvaluationOptions: + return EvaluationOptions.model_validate(self._client.get("agents/evaluation-options")) + def get_evaluation_config(self, agent_id: str) -> EvaluationConfigResponse: return EvaluationConfigResponse.model_validate(self._client.get(f"agents/{agent_id}/evaluation-config")) @@ -156,6 +163,10 @@ def list_evaluations( agent_id: str, *, grade: str | None = None, + status: str | None = None, + created_after: datetime | None = None, + created_before: datetime | None = None, + session_id: str | None = None, page_size: int | None = None, cursor: str | None = None, **kwargs: Any, @@ -163,10 +174,29 @@ def list_evaluations( return EvaluationResultListResponse.model_validate( self._client.get( f"agents/{agent_id}/evaluations", - params={"grade": grade, "page_size": page_size, "cursor": cursor, **kwargs}, + params={ + "grade": grade, + "status": status, + "created_after": created_after.isoformat() if created_after is not None else None, + "created_before": created_before.isoformat() if created_before is not None else None, + "session_id": session_id, + "page_size": page_size, + "cursor": cursor, + **kwargs, + }, ) ) + def get_evaluation_metrics(self, agent_id: str, days: int = 30) -> EvaluationMetrics: + return EvaluationMetrics.model_validate( + self._client.get(f"agents/{agent_id}/evaluations/metrics", params={"days": days}) + ) + + def run_evaluations(self, agent_id: str, session_ids: list[str]) -> EvaluationRunResult: + return EvaluationRunResult.model_validate( + self._client.post(f"agents/{agent_id}/evaluations/run", json={"session_ids": session_ids}) + ) + def get_evaluation(self, agent_id: str, evaluation_id: str) -> EvaluationResultResponse: return EvaluationResultResponse.model_validate( self._client.get(f"agents/{agent_id}/evaluations/{evaluation_id}") @@ -280,6 +310,10 @@ async def list_mcp_servers(self, agent_id: str) -> AgentMcpServersResponse: data = await self._client.get(f"agents/{agent_id}/mcp-servers") return AgentMcpServersResponse.model_validate(data) + async def get_evaluation_options(self) -> EvaluationOptions: + data = await self._client.get("agents/evaluation-options") + return EvaluationOptions.model_validate(data) + async def get_evaluation_config(self, agent_id: str) -> EvaluationConfigResponse: data = await self._client.get(f"agents/{agent_id}/evaluation-config") return EvaluationConfigResponse.model_validate(data) @@ -304,16 +338,40 @@ async def list_evaluations( agent_id: str, *, grade: str | None = None, + status: str | None = None, + created_after: datetime | None = None, + created_before: datetime | None = None, + session_id: str | None = None, page_size: int | None = None, cursor: str | None = None, **kwargs: Any, ) -> EvaluationResultListResponse: data = await self._client.get( f"agents/{agent_id}/evaluations", - params={"grade": grade, "page_size": page_size, "cursor": cursor, **kwargs}, + params={ + "grade": grade, + "status": status, + "created_after": created_after.isoformat() if created_after is not None else None, + "created_before": created_before.isoformat() if created_before is not None else None, + "session_id": session_id, + "page_size": page_size, + "cursor": cursor, + **kwargs, + }, ) return EvaluationResultListResponse.model_validate(data) + async def get_evaluation_metrics(self, agent_id: str, days: int = 30) -> EvaluationMetrics: + data = await self._client.get(f"agents/{agent_id}/evaluations/metrics", params={"days": days}) + return EvaluationMetrics.model_validate(data) + + async def run_evaluations(self, agent_id: str, session_ids: list[str]) -> EvaluationRunResult: + data = await self._client.post( + f"agents/{agent_id}/evaluations/run", + json={"session_ids": session_ids}, + ) + return EvaluationRunResult.model_validate(data) + async def get_evaluation(self, agent_id: str, evaluation_id: str) -> EvaluationResultResponse: data = await self._client.get(f"agents/{agent_id}/evaluations/{evaluation_id}") return EvaluationResultResponse.model_validate(data) diff --git a/src/gumloop/types.py b/src/gumloop/types.py index 74957c9..46ce2ac 100644 --- a/src/gumloop/types.py +++ b/src/gumloop/types.py @@ -685,6 +685,7 @@ class EvaluationConfig(_Model): tags: list[Any] = Field(default_factory=list) data_points: list[Any] = Field(default_factory=list) sentiment: dict[str, Any] | None = None + notifications: dict[str, Any] | None = None updated_ts: str | None = None @@ -703,6 +704,42 @@ class EvaluationConfigUpdateRequest(_Model): tags: list[Any] | None = None data_points: list[Any] | None = None sentiment: dict[str, Any] | None = None + notifications: dict[str, Any] | None = None + + +class EvaluationOptionLimits(_Model): + criteria: int + tags: int + data_points: int + tag_name_max_len: int + tag_description_max_len: int + notification_recipients: int + + +class EvaluationOptions(_Model): + interaction_types: list[str] = Field(default_factory=list) + criterion_types: list[str] = Field(default_factory=list) + criterion_priorities: list[str] = Field(default_factory=list) + data_point_types: list[str] = Field(default_factory=list) + frequencies: list[str] = Field(default_factory=list) + default_interaction_types: list[str] = Field(default_factory=list) + limits: EvaluationOptionLimits + + +class EvaluationMetrics(_Model): + grades: dict[str, int] = Field(default_factory=dict) + tags: dict[str, int] = Field(default_factory=dict) + + +class EvaluationRunSkipped(_Model): + ineligible: list[str] = Field(default_factory=list) + in_flight: list[str] = Field(default_factory=list) + + +class EvaluationRunResult(_Model): + status: Literal["queued"] + queued: int + skipped: EvaluationRunSkipped class EvaluationResult(_Model):