diff --git a/adr/0003-opentelemetry-as-the-operational-record.md b/adr/0003-opentelemetry-as-the-operational-record.md index 56bd95e..bca7def 100644 --- a/adr/0003-opentelemetry-as-the-operational-record.md +++ b/adr/0003-opentelemetry-as-the-operational-record.md @@ -8,7 +8,7 @@ OpenTelemetry spans correlate workflow, process and model operations with a run ## Current implementation -Orbit exports workflow, step, remote-invocation and model spans to a local JSONL OTEL exporter. Run metadata, console output and evidence are retained with the local run record. +Orbit exports workflow, step, remote-invocation, supervisor evaluation and cycle-review model spans to a local JSONL OTEL exporter. It includes redaction-safe request/response fingerprints and lengths, lifecycle metadata and outcomes. Run metadata, console output and full prompt/response evidence are retained with the local run record. ## Planned work diff --git a/backend/app/store.py b/backend/app/store.py index 6b11632..1ec42e3 100644 --- a/backend/app/store.py +++ b/backend/app/store.py @@ -16,10 +16,12 @@ from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any +from urllib.parse import urlparse from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import requests import yaml +from opentelemetry.trace import Status, StatusCode from orbit import load_bundle @@ -32,6 +34,15 @@ ROOT = Path(__file__).resolve().parents[2] +def _telemetry_text_metadata(prefix: str, value: str) -> dict[str, str | int]: + """Describe retained model text without exporting its potentially sensitive contents.""" + encoded = value.encode("utf-8") + return { + f"{prefix}.length": len(value), + f"{prefix}.sha256": hashlib.sha256(encoded).hexdigest(), + } + + def _application_data_pointer() -> Path: """Keep an operator-selected data location outside the data it points to.""" if os.name == "nt": @@ -4342,6 +4353,10 @@ def _execute(self, run_id: str) -> None: "workflow.id": workflow.id, "run.id": run_id, "workflow.kind": workflow.kind, + "orbit.execution.mode": run.execution_mode, + "orbit.execution.type": run.execution_type, + "orbit.loop.limit": run.loop_limit, + "orbit.build.id": run.build_id or "", }, ) as workflow_span: trace_id = f"{workflow_span.get_span_context().trace_id:032x}" @@ -4721,16 +4736,24 @@ def supervisor_result(result: object) -> object: "supervisor.evaluate", attributes={ "run.id": run_id, + "gen_ai.operation.name": "chat", "gen_ai.provider.name": settings.provider, "gen_ai.request.model": settings.model, "orbit.manager.template": ( self.build(run.build_id).get("manager_template_id") if run.build_id else "" ), "orbit.iteration": iteration, + "orbit.candidate.id": candidate_id or "", + "orbit.evidence.step_count": len(cycle_evidence), + **_telemetry_text_metadata("gen_ai.request.prompt", supervisor_prompt), }, ) as span: try: - result = self._validated_supervisor_result(provider.complete(settings, supervisor_prompt)) + span.add_event("gen_ai.request.sent") + response_text = provider.complete(settings, supervisor_prompt) + span.set_attributes(_telemetry_text_metadata("gen_ai.response", response_text)) + span.add_event("gen_ai.response.received") + result = self._validated_supervisor_result(response_text) evaluation = result.get("evaluation") if evaluation is not None: threshold = ( @@ -4767,7 +4790,16 @@ def supervisor_result(result: object) -> object: self._review_cycle_improvement(run, iteration, result, settings, provider) span.set_attribute("orbit.supervisor.improvements", len(result["improvements"])) span.set_attribute("orbit.supervisor.reported_issues", len(result["reported_issues"])) - span.add_event("supervisor.response.validated") + if isinstance(evaluation, dict): + span.set_attribute("orbit.supervisor.score", evaluation.get("score", 0)) + span.set_attribute("orbit.supervisor.approval", str(evaluation.get("approval", ""))) + span.add_event( + "supervisor.response.validated", + { + "orbit.supervisor.improvements": len(result["improvements"]), + "orbit.supervisor.reported_issues": len(result["reported_issues"]), + }, + ) except ValueError as error: run = self._load(run_id) run.supervisor_status, run.supervisor_error, run.updated_at = ( @@ -4786,7 +4818,8 @@ def supervisor_result(result: object) -> object: ) self._save(run) span.record_exception(error) - span.add_event("supervisor.response.invalid", {"reason": str(error)}) + span.set_status(Status(StatusCode.ERROR)) + span.add_event("supervisor.response.invalid", {"error.type": type(error).__name__}) except (RuntimeError, requests.RequestException) as error: run = self._load(run_id) run.supervisor_status, run.supervisor_error, run.updated_at = "failed", str(error), now() @@ -4801,7 +4834,8 @@ def supervisor_result(result: object) -> object: ) self._save(run) span.record_exception(error) - span.add_event("supervisor.request.failed", {"reason": str(error)}) + span.set_status(Status(StatusCode.ERROR)) + span.add_event("supervisor.request.failed", {"error.type": type(error).__name__}) def _review_cycle_improvement( self, run: Run, iteration: int, result: dict[str, Any], settings: ModelSettings, provider: Any @@ -4819,10 +4853,29 @@ def _review_cycle_improvement( ) ) try: - reviewed = json.loads(provider.complete(settings, prompt)) - interventions = reviewed.get("interventions", []) if isinstance(reviewed, dict) else [] - if not isinstance(interventions, list): - return + with self.tracer.start_as_current_span( + "supervisor.cycle_review", + attributes={ + "run.id": run.id, + "gen_ai.operation.name": "chat", + "gen_ai.provider.name": settings.provider, + "gen_ai.request.model": settings.model, + "orbit.iteration": iteration, + **_telemetry_text_metadata("gen_ai.request.prompt", prompt), + }, + ) as span: + span.add_event("gen_ai.request.sent") + response_text = provider.complete(settings, prompt) + span.set_attributes(_telemetry_text_metadata("gen_ai.response", response_text)) + span.add_event("gen_ai.response.received") + reviewed = json.loads(response_text) + interventions = reviewed.get("interventions", []) if isinstance(reviewed, dict) else [] + if not isinstance(interventions, list): + span.set_status(Status(StatusCode.ERROR)) + span.add_event("supervisor.cycle_review.invalid_response") + return + span.set_attribute("orbit.cycle_review.intervention_count", len(interventions)) + span.add_event("supervisor.cycle_review.validated") stored = self.cycle_interventions() for intervention in interventions: if not isinstance(intervention, dict) or not isinstance(intervention.get("title"), str): @@ -4861,6 +4914,11 @@ def _execute_step( "run.id": run_id, "step.id": step.id, "step.phase": step.phase, + "orbit.iteration": loop_index, + "orbit.candidate.id": candidate_id or "", + "process.command.executable": Path(step.command[0]).name if step.command else "", + "process.command.argument_count": max(0, len(step.command) - 1), + "process.timeout.seconds": step.timeout_seconds, }, ) as span: run = self._load(run_id) @@ -4922,6 +4980,7 @@ def _execute_step( creationflags=creation_flags, env=environment, ) + span.add_event("process.started", {"process.pid": process.pid}) interruption_timer: threading.Timer | None = None if ( step.phase == "verify" @@ -5154,7 +5213,13 @@ def capture_output() -> None: run.pid, run.updated_at = None, now() self._save(run) span.add_event("process.completed", {"process.exit_code": process.returncode}) + span.set_attribute("process.exit_code", process.returncode) + span.set_attribute("process.output.line_count", len(captured_lines)) + span.set_attribute("orbit.result.has_structured_output", structured_result is not None) + span.set_attribute("orbit.result.target_log_count", len(target_logs)) + span.set_attribute("orbit.result.data_file_count", len(data_files)) if process.returncode and step.on_failure == "stop": + span.set_status(Status(StatusCode.ERROR)) if step.phase == "verify" and run.advance_requested: run.advance_requested = False self._save(run) @@ -5169,12 +5234,14 @@ def capture_output() -> None: except subprocess.TimeoutExpired: self._stop_process_group(process, force=True) span.add_event("process.timeout", {"timeout.seconds": step.timeout_seconds}) + span.set_status(Status(StatusCode.ERROR)) if self._load(run_id).status == "cancelled": return self._fail(self._load(run_id), step.id, f"timed out after {step.timeout_seconds}s") return except ValueError as error: - span.add_event("step.rejected", {"reason": str(error)}) + span.add_event("step.rejected", {"error.type": type(error).__name__}) + span.set_status(Status(StatusCode.ERROR)) self._fail(self._load(run_id), step.id, str(error)) return finally: @@ -5359,7 +5426,16 @@ def test_build(self, build_id: str, output_locale: str | None = None) -> Run: def _execute_remote(self, run_id: str, executor: dict[str, Any]) -> None: run = self._load(run_id) - with self.tracer.start_as_current_span("remote.agent.run", attributes={"run.id": run_id}) as span: + endpoint = str(executor.get("endpoint", "")) + with self.tracer.start_as_current_span( + "remote.agent.run", + attributes={ + "run.id": run_id, + "http.request.method": str(executor.get("method", "POST")), + "server.address": urlparse(endpoint).hostname or "", + "http.request.timeout_seconds": int(executor.get("timeout_seconds", 0) or 0), + }, + ) as span: run.status, run.current_phase, run.telemetry_trace_id, run.updated_at = ( "running", "execute", @@ -5380,6 +5456,10 @@ def _execute_remote(self, run_id: str, executor: dict[str, Any]) -> None: invocation = RemoteInvocation(**invocation_values) status_code, output = invocation.invoke() span.set_attribute("http.response.status_code", status_code) + span.set_attributes(_telemetry_text_metadata("http.response.body", output)) + span.add_event("remote.response.received", {"http.response.status_code": status_code}) + if not 200 <= status_code < 300: + span.set_status(Status(StatusCode.ERROR)) run = self._load(run_id) run.step_results.append( {"step_id": "execute", "phase": "execute", "http_status": status_code, "output": output} @@ -5391,4 +5471,6 @@ def _execute_remote(self, run_id: str, executor: dict[str, Any]) -> None: self._complete_supervision(run_id) except (ValueError, requests.RequestException) as error: span.record_exception(error) + span.set_status(Status(StatusCode.ERROR)) + span.add_event("remote.request.failed", {"error.type": type(error).__name__}) self._fail(self._load(run_id), "execute", str(error)) diff --git a/frontend/src/features/evaluations/page.tsx b/frontend/src/features/evaluations/page.tsx index 2108606..0ebe158 100644 --- a/frontend/src/features/evaluations/page.tsx +++ b/frontend/src/features/evaluations/page.tsx @@ -1103,7 +1103,6 @@ export function EvaluationsPage({ record={translateSupervisorRecord(supervision)} l={l} telemetry={telemetry} - iteration={iterationTab} renderLineOutput={(value) => } /> diff --git a/frontend/src/features/evaluations/run-detail-supervisor-panel.tsx b/frontend/src/features/evaluations/run-detail-supervisor-panel.tsx index 439fce2..6ec4864 100644 --- a/frontend/src/features/evaluations/run-detail-supervisor-panel.tsx +++ b/frontend/src/features/evaluations/run-detail-supervisor-panel.tsx @@ -4,6 +4,15 @@ import { StatusBadge } from "../../components/ui/status-badge"; type Messages = Record; +function telemetryValue(value: unknown) { + return typeof value === "string" ? value : JSON.stringify(value); +} + +function duration(span: TelemetrySpan) { + if (!span.startTime || !span.endTime || span.endTime < span.startTime) return undefined; + return `${((span.endTime - span.startTime) / 1_000_000).toFixed(1)} ms`; +} + function TelemetryTree({ telemetry, l }: { telemetry?: RunTelemetry; l: Messages }) { const [collapsed, setCollapsed] = useState>(new Set()); const tree = useMemo(() => { @@ -14,14 +23,14 @@ function TelemetryTree({ telemetry, l }: { telemetry?: RunTelemetry; l: Messages if (!telemetry) return

{l.loadingOpenTelemetryTrace}

; if (!tree.roots.length) return

{l.noOpenTelemetrySpans}

; const render = (span: TelemetrySpan): ReactNode => { - const children = tree.children.get(span.spanId) ?? [], expandable = children.length > 0, isCollapsed = collapsed.has(span.spanId); - return
  • {expandable && !isCollapsed &&
      {children.map(render)}
    }
  • ; + const children = tree.children.get(span.spanId) ?? [], expandable = children.length > 0, isCollapsed = collapsed.has(span.spanId), attributes = Object.entries(span.attributes ?? {}), events = span.events ?? []; + const summary = [span.status || "UNSET", duration(span), ...events.map((event) => event.name)].filter(Boolean).join(" · "); + return
  • {!isCollapsed && (attributes.length > 0 || events.some((event) => Object.keys(event.attributes ?? {}).length > 0)) &&
    {attributes.map(([key, value]) =>
    {key}
    {telemetryValue(value)}
    )}{events.flatMap((event) => Object.entries(event.attributes ?? {}).map(([key, value]) =>
    {event.name} · {key}
    {telemetryValue(value)}
    ))}
    }{expandable && !isCollapsed &&
      {children.map(render)}
    }
  • ; }; return
      {tree.roots.map(render)}
    ; } -export function SupervisorPanel({ record, l, telemetry, iteration, renderLineOutput }: { record?: SupervisorRecord; l: Messages; telemetry?: RunTelemetry; iteration: number; renderLineOutput: (value: string) => ReactNode }) { +export function SupervisorPanel({ record, l, telemetry, renderLineOutput }: { record?: SupervisorRecord; l: Messages; telemetry?: RunTelemetry; renderLineOutput: (value: string) => ReactNode }) { const response = record?.response; - const iterationTelemetry = telemetry ? { ...telemetry, spans: telemetry.spans.filter((span) => span.name === "supervisor.evaluate" && Number(span.attributes?.["orbit.iteration"]) === iteration) } : undefined; - return
    {l.supervisorPrompt}
    {renderLineOutput(record?.prompt || l.noSupervisorPrompt || "")}
    {l.supervisorResponse}{response ? renderLineOutput(JSON.stringify(response, null, 2)) :

    {record?.error || l.supervisorWaiting}

    }
    {l.openTelemetryTrace}
    ; + return
    {l.supervisorPrompt}
    {renderLineOutput(record?.prompt || l.noSupervisorPrompt || "")}
    {l.supervisorResponse}{response ? renderLineOutput(JSON.stringify(response, null, 2)) :

    {record?.error || l.supervisorWaiting}

    }
    {l.openTelemetryTrace}
    ; } diff --git a/frontend/src/theme-overrides.css b/frontend/src/theme-overrides.css index 27c7922..5b4ee04 100644 --- a/frontend/src/theme-overrides.css +++ b/frontend/src/theme-overrides.css @@ -119,6 +119,7 @@ footer { display:flex; align-items:center; justify-content:space-between; margin .catalog-list__header { position:sticky; top:0; z-index:1; display:grid; min-height:28px; grid-template-columns:minmax(0,1fr) 140px 56px; align-items:center; gap:8px; padding:0 6px; border-bottom:1px solid var(--line); background:var(--surface); }.catalog-list__header button,.catalog-list__header > span { display:inline-flex; align-items:center; gap:4px; padding:0; border:0; color:var(--muted); background:transparent; font:10px 'DM Mono',monospace; text-transform:uppercase; text-align:left; }.catalog-list__header button { cursor:pointer; }.catalog-list__header button:hover { color:var(--accent); } .catalog-row-wrap { display:grid; grid-template-columns:minmax(0,1fr) 140px 56px; align-items:center; gap:8px; padding-inline:6px; border-top:1px solid var(--line); }.catalog-row-wrap:hover,.catalog-row-wrap:has(.catalog-row:focus-visible) { background:var(--surface-raised); }.catalog-row-wrap:hover .catalog-row,.catalog-row-wrap:has(.catalog-row:focus-visible) .catalog-row { color:var(--accent); background:transparent; }.catalog-row-wrap:hover .catalog-row__created,.catalog-row-wrap:has(.catalog-row:focus-visible) .catalog-row__created { color:var(--accent); }.catalog-row-wrap .catalog-row { min-width:0; border-top:0; }.catalog-row-wrap .icon-button,.catalog-row__created { min-width:0; }.catalog-row__created { white-space:nowrap; } .telemetry-tree,.telemetry-tree ul { display:grid; gap:6px; margin:0; padding:0; list-style:none; }.telemetry-tree ul { margin-left:13px; padding-left:15px; border-left:1px solid var(--line); }.trace-node { display:flex; width:100%; align-items:flex-start; gap:8px; padding:8px 10px; border:1px solid var(--line); border-radius:7px; color:var(--text); background:var(--bg); text-align:left; }.trace-node:not(:disabled) { cursor:pointer; }.trace-node:not(:disabled):hover { border-color:var(--accent); background:var(--surface-raised); }.trace-node:disabled { cursor:default; }.trace-node > div { display:grid; gap:3px; min-width:0; }.trace-node strong { font:600 11px 'DM Mono',monospace; }.trace-node small { overflow:hidden; color:var(--muted); text-overflow:ellipsis; white-space:nowrap; font-size:10px; }.trace-status { width:7px; height:7px; flex:none; margin-top:4px; border-radius:50%; }.trace-status--ok { background:#bce989; }.trace-status--error { background:#eaa89f; } +.trace-details { display:grid; gap:4px; margin:5px 10px 8px 25px; padding:7px 9px; border-left:2px solid var(--line); color:var(--muted); font:10px 'DM Mono',monospace; }.trace-details div { display:grid; grid-template-columns:minmax(120px, 38%) minmax(0, 1fr); gap:8px; }.trace-details dt { overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }.trace-details dd { min-width:0; margin:0; overflow-wrap:anywhere; color:var(--text); } .test-case-hint { margin:0; color:var(--muted); font-size:11px; line-height:1.55; }.test-case-editor fieldset label { display:grid; gap:5px; color:var(--muted); font-size:11px; }.test-case-editor fieldset textarea { min-height:84px; } .build-wizard .test-case-editor { display:none; } .build-wizard .modal-setting-row:has(textarea[placeholder]) { display:none; } diff --git a/specs/observability/open-telemetry.md b/specs/observability/open-telemetry.md index c2db550..f115e5f 100644 --- a/specs/observability/open-telemetry.md +++ b/specs/observability/open-telemetry.md @@ -5,12 +5,15 @@ Status: accepted ## Trace model - One run creates one root trace. -- Workflow, step, remote-invocation and model calls are spans. +- Workflow, step, remote-invocation, supervisor evaluation and cycle-review model calls are spans. - Dashboard rows expose the local trace ID and its exported spans. ## Logs and metrics -- Console output is retained in the local run record; OTEL records subprocess completion and failures as span events. +- Console output and full supervisor prompts/responses are retained in the local run record. OTEL records + redaction-safe request/response lengths and SHA-256 fingerprints, model/provider metadata, evaluation + outcomes, subprocess lifecycle and output counts, and remote-response metadata as span attributes/events. +- Prompt, response, console and HTTP-body text are not exported in OTEL records. - Dashboard metrics currently include build/run counts and supervisor-feedback summaries. ## Data protection