From e24148b0d5369e5484ad52110c2db824a3b39d68 Mon Sep 17 00:00:00 2001 From: forthfate Date: Sat, 12 Sep 2026 01:47:02 +0900 Subject: [PATCH 1/2] feat: improve run controls and workflow progress --- backend/app/store.py | 74 +++++++++++------- backend/orbit_sdk.py | 7 ++ backend/tests/test_api.py | 87 ++++++++++++++++++++++ backend/tests/test_orbit_sdk.py | 12 +++ frontend/src/components/ui/modal.tsx | 11 ++- frontend/src/domain/models.ts | 2 +- frontend/src/features/evaluations/page.tsx | 19 +++-- frontend/src/theme-overrides.css | 4 +- 8 files changed, 177 insertions(+), 39 deletions(-) diff --git a/backend/app/store.py b/backend/app/store.py index 473cf8d..d9abc56 100644 --- a/backend/app/store.py +++ b/backend/app/store.py @@ -4575,6 +4575,21 @@ def interrupt_overdue_evaluation() -> None: captured_lines: list[tuple[str, str]] = [] live_step_key = f"{step.id}:{loop_index}:{candidate_id or '-'}:{started}" + def live_workflow_functions() -> list[dict[str, Any]]: + functions: list[dict[str, Any]] = [] + for _, line in captured_lines: + if not line.startswith("__ORBIT_RESULT__"): + continue + try: + emitted = json.loads(line.removeprefix("__ORBIT_RESULT__")) + except json.JSONDecodeError: + continue + if isinstance(emitted, dict) and isinstance(emitted.get("workflow_functions"), list): + functions.extend( + item for item in emitted["workflow_functions"] if isinstance(item, dict) + ) + return functions + def retained_visible_lines() -> list[tuple[str, str]]: retained: list[tuple[str, str]] = [] retained_size = 0 @@ -4602,6 +4617,9 @@ def persist_live_output() -> None: item["log_lines"] = [ {"timestamp": timestamp, "value": line} for timestamp, line in retained ] + functions = live_workflow_functions() + if functions: + item["result"] = {"workflow_functions": functions} current.updated_at = now() self._save(current) return @@ -4638,6 +4656,7 @@ def capture_output() -> None: } ) self._save(run) + persist_live_output() span.set_attribute("process.pid", process.pid) process.wait(timeout=step.timeout_seconds) if interruption_timer: @@ -4834,8 +4853,8 @@ def _wait_for_schedule(self, run_id: str) -> bool: def retry(self, run_id: str, restart_from_first: bool) -> Run: run = self._load(run_id) - if run.execution_type != "pipeline" or run.status not in {"failed", "cancelled"}: - raise ValueError("Only failed or cancelled pipeline runs can be retried") + if run.execution_type != "pipeline" or run.status not in {"succeeded", "failed", "cancelled"}: + raise ValueError("Only completed, failed, or cancelled pipeline runs can be retried") latest_iteration = max( ( int(item.get("loop_index", 0)) @@ -4844,31 +4863,32 @@ def retry(self, run_id: str, restart_from_first: bool) -> Run: ), default=1, ) - return self.create_run( - run.workflow_id, - execution_mode=run.execution_mode, - build_id=run.build_id, - build_name=run.build_name, - supervisor_profile_name=run.supervisor_profile_name, - prompt_source=run.prompt_source, - prompt_snapshot=run.prompt_snapshot, - loop_limit=run.loop_limit, - timezone=run.timezone, - schedule_enabled=run.schedule_enabled, - schedule_weekdays=run.schedule_weekdays, - schedule_start_time=run.schedule_start_time, - schedule_end_time=run.schedule_end_time, - start_iteration=1 if restart_from_first else min(latest_iteration, run.loop_limit), - repeat_interval_minutes=run.repeat_interval_minutes, - cadence_mode=run.cadence_mode, - overrun_policy=run.overrun_policy, - approval_score=run.approval_score, - iteration_strategy=run.iteration_strategy, - candidates_per_iteration=run.candidates_per_iteration, - repository=run.repository, - retry_of_run_id=run.id, - retry_mode="restart" if restart_from_first else "resume", - ) + restarted_at = now() + run.created_at = restarted_at + run.updated_at = restarted_at + run.finished_at = None + run.status = "queued" + run.start_iteration = 1 if restart_from_first else min(latest_iteration, run.loop_limit) + run.retry_of_run_id = None + run.retry_mode = "restart" if restart_from_first else "resume" + run.iteration_candidates = [] + run.iteration_deadline_at = None + run.advance_requested = False + run.current_step = None + run.current_phase = None + run.pid = None + run.last_pid = None + run.telemetry_trace_id = None + run.supervisor_status = "pending" + run.supervisor_response = None + run.supervisor_error = None + run.supervisor_results = [] + run.runner_output = "" + run.step_results = [] + run.approval_reason = None + self._save(run) + self._start(run.id) + return self._load(run.id) def emergency_stop(self) -> list[Run]: stopped = [] diff --git a/backend/orbit_sdk.py b/backend/orbit_sdk.py index f7958b2..7889183 100644 --- a/backend/orbit_sdk.py +++ b/backend/orbit_sdk.py @@ -246,6 +246,7 @@ class RunnerContext: mode: str loop_index: int environment: dict[str, str] = field(default_factory=lambda: dict(os.environ)) + _active_workflow_functions: set[str] = field(default_factory=set, init=False, repr=False) def __post_init__(self) -> None: self.phase = canonical_phase(self.phase) @@ -255,6 +256,10 @@ def function(self, function_id: str): """Record one graph-annotated function's outcome within this lifecycle phase.""" if not function_id.strip(): raise ValueError("function_id must not be empty") + if function_id in self._active_workflow_functions: + yield + return + self._active_workflow_functions.add(function_id) started = datetime.now(UTC) self.log(f"workflow function started: {function_id}") self.emit_result( @@ -299,6 +304,8 @@ def function(self, function_id: str): ] } ) + finally: + self._active_workflow_functions.discard(function_id) @property def resources(self) -> dict[str, object]: diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index a6b3028..f96af4f 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -1,6 +1,9 @@ import base64 import json import sys +import threading +import time +from types import SimpleNamespace import orbit_sdk as sdk import pytest @@ -106,6 +109,54 @@ def test_runner_target_logs_are_retained_separately_from_runner_output(tmp_path, assert all(entry["iteration"] == 3 and entry["phase"] == "execute" for entry in step["target_logs"]) +def test_running_workflow_function_is_retained_before_its_step_finishes(tmp_path, monkeypatch): + monkeypatch.setattr(store_module, "RUNS", tmp_path / "runs") + project = tmp_path / "target" + project.mkdir() + runner = project / "runner.py" + runner.write_text( + "import time\n" + "from orbit_sdk import runner\n" + "@runner.phase('execute')\n" + "def run(ctx):\n" + " with ctx.function('collect-source-evidence'):\n" + " time.sleep(1)\n" + "if __name__ == '__main__': runner.main()\n", + encoding="utf-8", + ) + timestamp = store_module.now() + store = store_module.ConsoleStore() + store._save( + Run( + id="live-function-run", + workflow_id="workflow", + workflow_name="Workflow", + repository=str(project), + status="running", + created_at=timestamp, + updated_at=timestamp, + ) + ) + step = Step( + id="run", + phase="execute", + name="Run", + command=[sys.executable, str(runner), "--phase", "execute"], + working_directory=str(project), + ) + + thread = threading.Thread(target=store._execute_step, args=("live-function-run", step, 1)) + thread.start() + for _ in range(20): + results = store._load("live-function-run").step_results + if results and results[-1].get("result", {}).get("workflow_functions"): + break + time.sleep(0.1) + thread.join() + + assert results[-1]["result"]["workflow_functions"][0]["status"] == "running" + + def test_runner_data_files_are_retained_for_the_iteration(tmp_path, monkeypatch): app_data = tmp_path / "orbit-data" monkeypatch.setattr(store_module, "APP_DATA", app_data) @@ -489,6 +540,42 @@ def test_deleting_a_completed_run_removes_its_history(tmp_path, monkeypatch): assert not (store_module.RUNS / "completed-run.json").exists() +def test_completed_pipeline_run_can_be_retried(tmp_path, monkeypatch): + monkeypatch.setattr(store_module, "RUNS", tmp_path / "runs") + store = store_module.ConsoleStore() + timestamp = store_module.now() + store._save( + Run( + id="completed-run", + workflow_id="workflow", + workflow_name="Workflow", + execution_type="pipeline", + status="succeeded", + created_at=timestamp, + updated_at=timestamp, + finished_at=timestamp, + step_results=[{"step_id": "execute", "loop_index": 1, "output": "previous output"}], + supervisor_results=[{"iteration": 1, "response": {"evaluation": {"score": 8}}}], + runner_output="previous runner output", + ) + ) + monkeypatch.setattr( + store, "_runner_execution_plan", lambda runner_id: SimpleNamespace(id=runner_id, name="Workflow") + ) + monkeypatch.setattr(store, "_runner_graph_definition", lambda *_: None) + monkeypatch.setattr(store, "_start", lambda _: None) + + retried = store.retry("completed-run", restart_from_first=True) + + assert retried.id == "completed-run" + assert retried.retry_of_run_id is None + assert retried.retry_mode == "restart" + assert retried.status == "queued" + assert retried.step_results == [] + assert retried.supervisor_results == [] + assert retried.runner_output == "" + + def test_active_evaluations_count_feedback_across_all_iterations(monkeypatch): store = store_module.ConsoleStore() timestamp = store_module.now() diff --git a/backend/tests/test_orbit_sdk.py b/backend/tests/test_orbit_sdk.py index 88c9016..ca65f43 100644 --- a/backend/tests/test_orbit_sdk.py +++ b/backend/tests/test_orbit_sdk.py @@ -94,6 +94,18 @@ def collect(ctx) -> None: assert '"status": "failed"' in output +def test_nested_function_trace_for_the_same_node_is_emitted_once(tmp_path, capsys): + ctx = context(tmp_path, iteration=1) + + with ctx.function("collect-source-evidence"): + with ctx.function("collect-source-evidence"): + pass + + output = capsys.readouterr().out + assert output.count('"status": "running"') == 1 + assert output.count('"status": "succeeded"') == 1 + + def context(project, *, iteration: int, run_id: str = "run-123"): return sdk.RunnerContext( phase="execute", diff --git a/frontend/src/components/ui/modal.tsx b/frontend/src/components/ui/modal.tsx index 09b3b3c..63519c2 100644 --- a/frontend/src/components/ui/modal.tsx +++ b/frontend/src/components/ui/modal.tsx @@ -8,12 +8,14 @@ export function Modal({ onClose, children, className = "", + headerActions, }: { open: boolean; title: string; onClose: () => void; children: ReactNode; className?: string; + headerActions?: ReactNode; }) { useEffect(() => { if (!open) return; @@ -37,9 +39,12 @@ export function Modal({ >

{title}

- +
+ {headerActions} + +
{children} diff --git a/frontend/src/domain/models.ts b/frontend/src/domain/models.ts index 744eabf..91245a8 100644 --- a/frontend/src/domain/models.ts +++ b/frontend/src/domain/models.ts @@ -6,7 +6,7 @@ export type TargetEnvironment = {id:string;name:string;repository:string;browser export type Build = { id:string; name:string; enabled:boolean; runner_id:string; execution_environment_id?:string;target_environment_id?:string; repository:string; repository_name?:string; repository_is_git?:boolean; repository_error?:string; purpose:string;manager_template_id?:string;model_profile_name?:string;test_case_set_id?:string;test_cases?:TestCase[];browser_base_url?:string;browser_executable_path?:string;browser_library_path?:string;timezone:string;repeat_interval_minutes:number;run_limit:number;cadence_mode?:'after_completion'|'fixed';overrun_policy?:'wait'|'interrupt_eval';schedule_enabled?:boolean;schedule_weekdays?:number[];schedule_start_time?:string;schedule_end_time?:string;iteration_strategy?:'linear'|'score_select';candidates_per_iteration?:number;approval_score:number;require_human_approval_before_apply?:boolean;created_at?:string;last_run_at?:string;executor?:{type?:'local'|'remote-http';endpoint?:string;method?:'GET'|'POST'|'PUT';timeout_seconds?:number;headers?:Record} } export type PromptTemplate = { id:string; name:string; version:number; content:string; versions?:{version:number;content:string}[];created_at?:string } export type SavedDataFile = { label?:string; filename:string; path:string; relative_path?:string; sha256?:string; size?:number; content_type?:string } -export type RunStepResult = { step_id:string; phase?:'before_all'|'before_each'|'execute'|'verify'|'after_each'|'after_all'; loop_index?:number; candidate_id?:string|null; name?:string; command?:string[]; working_directory?:string; started_at?:string; ended_at?:string; exit_code?:number; output?:string; error?:string; log_lines?:{timestamp:string;value:string}[]; target_logs?:{timestamp?:string;level?:string;source?:string;message:string;run_id?:string;iteration?:number;phase?:string}[]; data_files?:SavedDataFile[]; result?:Record } +export type RunStepResult = { step_id:string; phase?:'before_all'|'before_each'|'execute'|'verify'|'after_each'|'after_all'; loop_index?:number; candidate_id?:string|null; name?:string; command?:string[]; working_directory?:string; started_at?:string; ended_at?:string; in_progress?:boolean; exit_code?:number; output?:string; error?:string; log_lines?:{timestamp:string;value:string}[]; target_logs?:{timestamp?:string;level?:string;source?:string;message:string;run_id?:string;iteration?:number;phase?:string}[]; data_files?:SavedDataFile[]; result?:Record } export type WorkflowGraphNode = { id:string; title:string; phase?:string|null; inputs?:string[]; outputs?:string[]; description?:string|null; status?:'idle'|'running'|'succeeded'|'failed'|'skipped' } export type WorkflowGraphEdge = { source:string; target:string; kind?:'execution'|'data'|'condition'|'loop'|'error'; label?:string|null; source_port?:string|null; target_port?:string|null } export type WorkflowGraphDefinition = { nodes:WorkflowGraphNode[]; edges:WorkflowGraphEdge[] } diff --git a/frontend/src/features/evaluations/page.tsx b/frontend/src/features/evaluations/page.tsx index c03fc94..6e62163 100644 --- a/frontend/src/features/evaluations/page.tsx +++ b/frontend/src/features/evaluations/page.tsx @@ -524,7 +524,7 @@ export function EvaluationsPage({ )} - {["failed", "cancelled"].includes(r.status) && r.execution_type === "pipeline" && ( + {terminal(r.status) && r.execution_type === "pipeline" && ( @@ -591,23 +591,26 @@ export function EvaluationsPage({ const workflowGraph = useMemo(() => { const definition = selected?.workflow_graph; if (!definition?.nodes.length) return null; - const hasFunctionTrace = steps.some((step) => Array.isArray(step.result?.workflow_functions)); return { ...definition, nodes: definition.nodes.map((node) => { const phaseSteps = steps.filter((step) => (step.phase ?? step.step_id) === node.phase); const latestStep = phaseSteps.at(-1); + const phaseHasFunctionTrace = phaseSteps.some((step) => Array.isArray(step.result?.workflow_functions)); + const firstNodeInPhase = definition.nodes.find((candidate) => candidate.phase === node.phase); const functionTrace = [...steps].reverse().flatMap((step) => { const traces = step.result?.workflow_functions; - return Array.isArray(traces) ? traces : []; + return Array.isArray(traces) ? [...traces].reverse() : []; }).find((trace) => typeof trace === "object" && trace !== null && trace.id === node.id) as { status?: WorkflowGraphNode["status"] } | undefined; const status: WorkflowGraphNode["status"] = functionTrace?.status ? functionTrace.status - : selected?.status === "running" && node.phase === selected.current_phase && !hasFunctionTrace + : selected?.status === "running" && node.phase === selected.current_phase && !phaseHasFunctionTrace && firstNodeInPhase?.id === node.id ? "running" + : latestStep?.in_progress + ? "idle" : latestStep?.error || (latestStep?.exit_code ?? 0) !== 0 - ? "failed" - : latestStep + ? phaseHasFunctionTrace ? "idle" : "failed" + : latestStep ? "succeeded" : "idle"; return { ...node, status }; @@ -932,6 +935,10 @@ export function EvaluationsPage({ onSelectedRunClose?.(); }} className="modal--run-detail" + headerActions={<> + + + } >
diff --git a/frontend/src/theme-overrides.css b/frontend/src/theme-overrides.css index f1a9ab7..7156cba 100644 --- a/frontend/src/theme-overrides.css +++ b/frontend/src/theme-overrides.css @@ -39,7 +39,7 @@ footer { display:flex; align-items:center; justify-content:space-between; margin .approve,.run-button { background:var(--accent); } .modal-backdrop { position:fixed; inset:0; z-index:20; display:grid; place-items:center; padding:24px; background:rgb(0 0 0 / .58); } .modal { box-sizing:border-box; width:min(760px,100%); min-width:0; max-height:calc(100vh - 48px); overflow:auto; border:1px solid var(--line); border-radius:12px; background:var(--surface); box-shadow:0 18px 60px rgb(0 0 0 / .38); padding:22px; } -.modal-header,.panel-title-action { display:flex; align-items:flex-start; justify-content:space-between; gap:16px; } +.modal-header,.panel-title-action { display:flex; align-items:flex-start; justify-content:space-between; gap:16px; }.modal-header-actions { display:flex; align-items:center; gap:6px; }.modal-header-actions .icon-button.danger { border-color:transparent; background:transparent; } .modal-header { margin-bottom:18px; }.modal-header h2 { margin:0; font-size:18px; white-space:nowrap; }.modal-close { border:0; color:var(--muted); background:transparent; padding:3px; } .modal-form { display:grid; gap:10px; }.build-form { grid-template-columns:repeat(2,minmax(0,1fr)); }.profile-form { grid-template-columns:repeat(2,minmax(0,1fr)); } .build-form { grid-template-columns:1fr; }.modal-setting-row { display:grid; grid-template-columns:180px minmax(0,1fr); align-items:center; gap:18px; padding:11px 0; border-bottom:1px solid var(--line); color:var(--text); font-size:12px; }.modal-setting-row textarea { min-height:150px; resize:vertical; }.build-approval-toggle { display:flex; align-items:center; gap:7px; }.prompt-changes { display:grid; gap:12px; }.prompt-changes > section { overflow:hidden; border:1px solid var(--line); border-radius:8px; background:var(--surface-raised); }.prompt-version-navigator { display:flex; align-items:center; justify-content:center; gap:10px; }.prompt-version-navigator span { min-width:94px; color:var(--muted); font:11px 'DM Mono',monospace; text-align:center; }.prompt-changes__head { display:flex; align-items:center; justify-content:space-between; gap:12px; padding:10px 12px; border-bottom:1px solid var(--line); }.prompt-changes__head div { display:grid; gap:3px; min-width:0; }.prompt-changes__head small { overflow:hidden; color:var(--muted); text-overflow:ellipsis; white-space:nowrap; }.prompt-change-meta,.prompt-changes > section > .hint { margin:0; padding:10px 12px; color:var(--muted); font:11px 'DM Mono',monospace; }.prompt-diff { max-height:340px; overflow:auto; margin:0; color:var(--text); background:var(--bg); font:11px/1.6 'DM Mono',monospace; }.prompt-diff__row { display:grid; grid-template-columns:46px 46px 22px minmax(max-content,1fr); min-width:max-content; }.prompt-diff__line-number { padding:0 10px; border-right:1px solid var(--line); color:var(--muted); text-align:right; user-select:none; }.prompt-diff__marker { padding:0 7px; color:var(--muted); text-align:center; user-select:none; }.prompt-diff__row code { min-width:0; padding-right:12px; color:inherit; white-space:pre; }.prompt-diff__row--added { color:#c5edbd; background:#203c2a; }.prompt-diff__row--added .prompt-diff__line-number,.prompt-diff__row--added .prompt-diff__marker { border-color:rgb(142 211 132 / .22); color:#a9db9d; }.prompt-diff__row--removed { color:#ffc2bb; background:#492a2a; }.prompt-diff__row--removed .prompt-diff__line-number,.prompt-diff__row--removed .prompt-diff__marker { border-color:rgb(235 142 132 / .24); color:#efaaa1; }.prompt-diff::selection { background:color-mix(in srgb,var(--accent) 35%,transparent); } @@ -146,6 +146,6 @@ main > aside,main > .content { transition:width .2s ease,margin-left .2s ease,pa .run-detail-tab-panel { display:grid; gap:14px; padding-top:14px; }.run-detail-tab-panel__description { margin:0; color:var(--muted); font-size:11px; line-height:1.5; }.run-detail-tab-panel__intro { display:flex; align-items:center; justify-content:space-between; gap:12px; }.run-detail-tab-panel__intro .iteration-navigator { flex:none; min-height:30px; padding:0; border:0; }.tooltip-box { display:grid; grid-template-columns:auto minmax(0,1fr); align-items:start; gap:8px; padding:10px 12px; border:1px solid color-mix(in srgb,var(--accent) 40%,var(--line)); border-radius:8px; color:var(--text); background:color-mix(in srgb,var(--accent) 14%,var(--surface-raised)); font-size:11px; line-height:1.55; }.tooltip-box > svg { margin-top:1px; color:var(--accent); }.tooltip-box > span { min-width:0; }.iteration-candidates { display:flex; justify-content:flex-end; flex-wrap:wrap; gap:6px; padding:8px 0; border-bottom:1px solid var(--line); }.iteration-candidates button { display:flex; align-items:center; gap:6px; padding:4px 7px; border:1px solid var(--line); border-radius:999px; color:var(--muted); background:transparent; font:10px 'DM Mono',monospace; cursor:pointer; }.iteration-candidates button.selected { border-color:color-mix(in srgb,var(--accent) 45%,var(--line)); color:var(--accent); background:color-mix(in srgb,var(--accent) 9%,var(--bg)); }.iteration-candidates small { font:inherit; }.iteration-candidates em { font-style:normal; } @media(max-width:520px){.run-detail-tab-panel__intro { align-items:flex-start; flex-direction:column; }} .browser-evidence { display:grid; gap:6px; margin:10px 0; padding:10px; border:1px solid var(--line); border-radius:8px; background:var(--surface-raised); font-size:12px; }.browser-evidence > small,.browser-evidence div small { overflow-wrap:anywhere; color:var(--muted); }.browser-evidence div { display:grid; gap:3px; padding-top:7px; border-top:1px solid var(--line); }.browser-evidence b { width:max-content; padding:2px 6px; border-radius:999px; font-size:10px; }.browser-evidence__pass { color:#bdecc8; background:#1e4832; }.browser-evidence__fail { color:#ffc0b7; background:#4b2925; } .run-build code { overflow:hidden; color:#839088; text-overflow:ellipsis; white-space:nowrap; font:9px 'DM Mono',monospace; }.table .tr > span { min-width:0; overflow:visible; text-overflow:clip; white-space:normal; overflow-wrap:anywhere; word-break:break-word; line-height:1.35; font-size:clamp(9px,.76vw,11px); }.table .tr.th > span { font-size:clamp(8px,.7vw,10px); line-height:1.25; } -.workflow-graph { height:378px; min-height:243px; border:1px solid var(--line); border-radius:10px; overflow:hidden; background:var(--surface); }.workflow-graph-node { position:relative; width:250px; border:1px solid var(--line); border-radius:8px; background:var(--surface-raised); color:var(--text); box-shadow:0 8px 22px rgb(0 0 0 / .18); }.workflow-graph .react-flow__handle { pointer-events:none; opacity:0; }.workflow-graph-node header,.workflow-graph-node footer { display:grid; gap:4px; padding:10px; }.workflow-graph-node header { border-bottom:1px solid var(--line); }.workflow-graph-node small { color:var(--muted); font:9px 'DM Mono',monospace; text-transform:uppercase; }.workflow-graph-node strong { font-size:12px; }.workflow-graph-node p { margin:0; padding:10px; color:var(--muted); font-size:11px; }.workflow-graph-node footer { grid-template-columns:1fr 1fr; color:var(--muted); font-size:10px; }.workflow-graph-node--running { border-color:var(--accent); box-shadow:0 0 0 1px color-mix(in srgb,var(--accent) 38%,transparent),0 0 20px color-mix(in srgb,var(--accent) 42%,transparent); animation:workflow-node-pulse 1.8s ease-in-out infinite; }.workflow-graph-node--running::after { content:""; position:absolute; top:0; left:0; width:18px; height:3px; border-radius:999px; pointer-events:none; background:var(--accent); box-shadow:0 0 7px 2px color-mix(in srgb,var(--accent) 82%,transparent),0 0 16px 3px color-mix(in srgb,var(--accent) 42%,transparent); offset-path:inset(-2px round 9px); offset-anchor:center; offset-rotate:0deg; animation:workflow-border-orbit 2.4s linear infinite; }.workflow-graph-node--failed { border-color:#eaa89f; }.workflow-graph-node--succeeded { border-color:#79c99e; }.workflow-graph-edge--condition .react-flow__edge-path { stroke:#f1d292; stroke-dasharray:5 4; }.workflow-graph-edge--loop .react-flow__edge-path { stroke:#8fb8ff; stroke-dasharray:6 4; }.workflow-graph-edge--error .react-flow__edge-path { stroke:#eaa89f; } @keyframes workflow-node-pulse { 50% { box-shadow:0 0 0 4px color-mix(in srgb,var(--accent) 10%,transparent),0 0 30px color-mix(in srgb,var(--accent) 56%,transparent); } } @keyframes workflow-border-orbit { to { offset-distance:100%; } } @media(prefers-reduced-motion:reduce) { .workflow-graph-node--running,.workflow-graph-node--running::after { animation:none; } } +.workflow-graph { height:378px; min-height:243px; border:1px solid var(--line); border-radius:10px; overflow:hidden; background:var(--surface); }.workflow-graph-node { position:relative; width:250px; border:1px solid var(--line); border-radius:8px; background:var(--surface-raised); color:var(--text); box-shadow:0 8px 22px rgb(0 0 0 / .18); }.workflow-graph .react-flow__handle { pointer-events:none; opacity:0; }.workflow-graph-node header,.workflow-graph-node footer { display:grid; gap:4px; padding:10px; }.workflow-graph-node header { border-bottom:1px solid var(--line); }.workflow-graph-node small { color:var(--muted); font:9px 'DM Mono',monospace; text-transform:uppercase; }.workflow-graph-node strong { font-size:12px; }.workflow-graph-node p { margin:0; padding:10px; color:var(--muted); font-size:11px; }.workflow-graph-node footer { grid-template-columns:1fr 1fr; color:var(--muted); font-size:10px; }.workflow-graph-node--idle { border-color:color-mix(in srgb,var(--line) 72%,var(--bg)); background:color-mix(in srgb,var(--surface-raised) 52%,var(--bg)); color:var(--muted); box-shadow:none; }.workflow-graph-node--running { border-color:var(--accent); box-shadow:0 0 0 1px color-mix(in srgb,var(--accent) 38%,transparent),0 0 20px color-mix(in srgb,var(--accent) 42%,transparent); animation:workflow-node-pulse 1.8s ease-in-out infinite; }.workflow-graph-node--running::after { content:""; position:absolute; top:0; left:0; width:18px; height:3px; border-radius:999px; pointer-events:none; background:var(--accent); box-shadow:0 0 7px 2px color-mix(in srgb,var(--accent) 82%,transparent),0 0 16px 3px color-mix(in srgb,var(--accent) 42%,transparent); offset-path:inset(-2px round 9px); offset-anchor:center; offset-rotate:0deg; animation:workflow-border-orbit 2.4s linear infinite; }.workflow-graph-node--failed { border-color:#eaa89f; }.workflow-graph-node--succeeded { border-color:#79c99e; }.workflow-graph-edge--condition .react-flow__edge-path { stroke:#f1d292; stroke-dasharray:5 4; }.workflow-graph-edge--loop .react-flow__edge-path { stroke:#8fb8ff; stroke-dasharray:6 4; }.workflow-graph-edge--error .react-flow__edge-path { stroke:#eaa89f; } @keyframes workflow-node-pulse { 50% { box-shadow:0 0 0 4px color-mix(in srgb,var(--accent) 10%,transparent),0 0 30px color-mix(in srgb,var(--accent) 56%,transparent); } } @keyframes workflow-border-orbit { to { offset-distance:100%; } } @media(prefers-reduced-motion:reduce) { .workflow-graph-node--running,.workflow-graph-node--running::after { animation:none; } } .workflow-graph .react-flow__controls { overflow:hidden; border:1px solid var(--line); border-radius:8px; background:var(--surface-raised); box-shadow:0 8px 22px rgb(0 0 0 / .22); }.workflow-graph .react-flow__controls-button { display:grid; width:30px; height:30px; place-items:center; border:0; border-bottom:1px solid var(--line); color:var(--muted); background:transparent; transition:color .15s ease,background-color .15s ease; }.workflow-graph .react-flow__controls-button:last-child { border-bottom:0; }.workflow-graph .react-flow__controls-button:hover { color:var(--text); background:var(--bg); }.workflow-graph .react-flow__controls-button svg { width:14px; height:14px; fill:currentColor; }.workflow-graph .react-flow__attribution { padding:3px 6px; border:1px solid color-mix(in srgb,var(--line) 82%,transparent); border-top:0; border-left:0; border-radius:0 0 5px 0; background:color-mix(in srgb,var(--surface-raised) 88%,transparent); }.workflow-graph .react-flow__attribution a { color:var(--muted); font:9px 'DM Mono',monospace; letter-spacing:.02em; transition:color .15s ease; }.workflow-graph .react-flow__attribution a:hover,.workflow-graph .react-flow__attribution a:focus-visible { color:var(--text); outline:0; } .workflow-graph-zone { box-sizing:border-box; width:100%; height:100%; padding:12px; border:1px dashed color-mix(in srgb,var(--accent) 45%,var(--line)); border-radius:10px; color:var(--muted); background:color-mix(in srgb,var(--accent) 5%,transparent); }.workflow-graph-zone strong,.workflow-graph-zone small { display:block; }.workflow-graph-zone strong { color:var(--text); font:600 11px 'DM Mono',monospace; text-transform:uppercase; }.workflow-graph-zone small { margin-top:3px; font:9px 'DM Mono',monospace; } From 55b2b99add2fdeb8850e59eaea929b592dcf3d34 Mon Sep 17 00:00:00 2001 From: forthfate Date: Sat, 12 Sep 2026 11:00:15 +0900 Subject: [PATCH 2/2] feat: instrument runner lifecycle templates --- backend/app/store.py | 201 +++++++++++++++++++-- backend/orbit_sdk.py | 10 +- backend/tests/test_api.py | 65 +++++++ frontend/src/features/evaluations/page.tsx | 16 +- 4 files changed, 272 insertions(+), 20 deletions(-) diff --git a/backend/app/store.py b/backend/app/store.py index d9abc56..7650c49 100644 --- a/backend/app/store.py +++ b/backend/app/store.py @@ -115,9 +115,16 @@ def _application_data_dir() -> Path: import json import re -from orbit_sdk import runner +from orbit_sdk import graph, runner REQUIRED_SUFFICIENT_EVALUATIONS = 3 + +graph.connect("validate-target", "prepare-prompt") +graph.connect("prepare-prompt", "exercise-target", label="managed prompt") +graph.connect("exercise-target", "assess-candidate", kind="data", label="responses") +graph.connect("assess-candidate", "retain-iteration") +graph.connect("retain-iteration", "prepare-prompt", kind="loop", label="next evaluation") +graph.connect("retain-iteration", "restore-baseline", kind="condition", label="completed") # Marker comments make replacement idempotent and preserve the surrounding # target prompt content that OpenOrbit does not own. PROMPT_BLOCK_START = "" @@ -204,6 +211,7 @@ def managed_prompt_evidence(ctx): } +@graph.step("validate-target", title="Validate target", phase="before_all", outputs=["evaluation_contract"]) @runner.phase("before_all") def before_all(ctx): # Process-level validation runs once before the iteration loop begins. @@ -215,6 +223,7 @@ def before_all(ctx): ctx.log("Validated an OpenOrbit-native target-AI prompt improvement cycle") +@graph.step("prepare-prompt", title="Prepare prompt candidate", phase="before_each", inputs=["evaluation_contract"], outputs=["managed_prompt"]) @runner.phase("before_each") def before_each(ctx): # Keep the target's complete pre-evaluation state outside commit history. @@ -259,6 +268,7 @@ def before_each(ctx): ctx.log("Refreshed the rollback-protected prompt from accepted supervisor feedback") +@graph.step("exercise-target", title="Exercise target AI", phase="execute", inputs=["managed_prompt"], outputs=["target_responses"]) @runner.phase("execute") def execute(ctx): # Exercise the evaluated AI with the current managed prompt. The raw reply @@ -306,6 +316,7 @@ def execute(ctx): ) +@graph.step("assess-candidate", title="Assess candidate evidence", phase="verify", inputs=["target_responses"], outputs=["candidate_verdict"]) @runner.phase("verify") def verify(ctx): # Promote a candidate only after the required number of stable evaluations. @@ -340,6 +351,7 @@ def verify(ctx): ctx.log(f"Candidate verdict: {verdict}") +@graph.step("retain-iteration", title="Retain iteration evidence", phase="after_each", inputs=["candidate_verdict"], outputs=["iteration_snapshot"]) @runner.phase("after_each") def after_each(ctx): # Preserve the first evaluated state as a named recovery checkpoint. @@ -348,6 +360,7 @@ def after_each(ctx): ctx.log("Retained prompt versions, decisions, and validation evidence") +@graph.step("restore-baseline", title="Restore baseline", phase="after_all", inputs=["iteration_snapshot"], outputs=["restored_target"]) @runner.phase("after_all") def after_all(ctx): # Return the target to its exact baseline without creating a Git commit. @@ -372,7 +385,11 @@ def after_all(ctx): from langgraph.graph import END, START, StateGraph import orbit_sdk -from orbit_sdk import runner +from orbit_sdk import graph as orbit_graph, runner + +orbit_graph.connect("validate-site", "explore-site") +orbit_graph.connect("explore-site", "review-evidence", kind="data", label="rendered pages") +orbit_graph.connect("review-evidence", "finalize-review") class ExplorerState(TypedDict, total=False): @@ -421,22 +438,109 @@ def graph(ctx): return workflow.compile() +@orbit_graph.step("validate-site", title="Validate site", phase="before_all", outputs=["site_target"]) @runner.phase("before_all") def before_all(ctx): if not ctx.build.get("browser_base_url"): raise ValueError("Set a browser base URL before exploring a site") +@orbit_graph.step("explore-site", title="Explore rendered site", phase="execute", inputs=["site_target"], outputs=["rendered_pages"]) @runner.phase("execute") def execute(ctx): result = graph(ctx).invoke({"base_url": ctx.build["browser_base_url"], "max_clicks": 3}) ctx.emit_result({"site_exploration": {"opinion": result["opinion"], "evidence": result["evidence"]}}) +@orbit_graph.step("review-evidence", title="Review exploration evidence", phase="verify", inputs=["rendered_pages"], outputs=["product_review"]) +@runner.phase("verify") +def verify(ctx): + ctx.log("Retained rendered exploration evidence for review") + + +@orbit_graph.step("finalize-review", title="Finalize site review", phase="after_all", inputs=["product_review"], outputs=["completed_review"]) +@runner.phase("after_all") +def after_all(ctx): + ctx.log("Finalized the bounded site exploration review") + + if __name__ == "__main__": runner.main() """ +EXTERNAL_COMMAND_ADAPTER_TEMPLATE = r'''"""Run a bounded external automation through its explicit action contract.""" + +import json +import os +import shlex + +from orbit_sdk import ORBIT_PROJECT_PATH, graph, runner + +graph.connect("check-adapter", "prepare-adapter") +graph.connect("prepare-adapter", "run-adapter", label="prepared target") +graph.connect("run-adapter", "collect-adapter-evidence", kind="data", label="adapter output") +graph.connect("collect-adapter-evidence", "close-adapter-cycle") +graph.connect("close-adapter-cycle", "prepare-adapter", kind="loop", label="next cycle") +graph.connect("close-adapter-cycle", "finalize-adapter", kind="condition", label="completed") + + +def adapter_command(): + configured = os.environ.get("ORBIT_ADAPTER_COMMAND", "").strip() + if not configured: + raise ValueError("Set ORBIT_ADAPTER_COMMAND to an external tool command") + if configured.startswith("["): + value = json.loads(configured) + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ValueError("ORBIT_ADAPTER_COMMAND JSON must be an array of strings") + return value + return shlex.split(configured) + + +def invoke(ctx, action): + return ctx.exec([*adapter_command(), action], cwd=ORBIT_PROJECT_PATH(), timeout=3600) + + +@graph.step("check-adapter", title="Check adapter readiness", phase="before_all", outputs=["adapter_status"]) +@runner.phase("before_all") +def before_all(ctx): + ctx.emit_result({"external_adapter": {"status": invoke(ctx, "status")}}) + + +@graph.step("prepare-adapter", title="Prepare adapter cycle", phase="before_each", inputs=["adapter_status"], outputs=["prepared_target"]) +@runner.phase("before_each") +def before_each(ctx): + ctx.emit_result({"external_adapter": {"iteration": ctx.loop_index, "prepared": invoke(ctx, "prepare")}}) + + +@graph.step("run-adapter", title="Run bounded adapter task", phase="execute", inputs=["prepared_target"], outputs=["adapter_result"]) +@runner.phase("execute") +def execute(ctx): + ctx.emit_result({"external_adapter": {"iteration": ctx.loop_index, "result": invoke(ctx, "run-once")}}) + + +@graph.step("collect-adapter-evidence", title="Collect adapter evidence", phase="verify", inputs=["adapter_result"], outputs=["adapter_evidence"]) +@runner.phase("verify") +def verify(ctx): + ctx.emit_result({"external_adapter": {"iteration": ctx.loop_index, "evidence": invoke(ctx, "collect-evidence")}}) + + +@graph.step("close-adapter-cycle", title="Close adapter cycle", phase="after_each", inputs=["adapter_evidence"], outputs=["cycle_complete"]) +@runner.phase("after_each") +def after_each(ctx): + ctx.log("Completed one bounded external adapter cycle") + + +@graph.step("finalize-adapter", title="Finalize external automation", phase="after_all", inputs=["cycle_complete"], outputs=["final_status"]) +@runner.phase("after_all") +def after_all(ctx): + ctx.log("Finalized the external automation evaluation") + + +if __name__ == "__main__": + runner.main() +''' + + JSON_AGENT_CYCLE_TEMPLATE = r'''"""Run a portable, bounded external agent cycle. Set ORBIT_AGENT_COMMAND to a JSON argument array or a shell-like command @@ -449,7 +553,14 @@ def execute(ctx): import os import shlex -from orbit_sdk import runner +from orbit_sdk import graph, runner + +graph.connect("check-agent", "record-inputs") +graph.connect("record-inputs", "run-agent", label="bounded input") +graph.connect("run-agent", "confirm-agent-state", kind="data", label="agent result") +graph.connect("confirm-agent-state", "close-cycle") +graph.connect("close-cycle", "record-inputs", kind="loop", label="next cycle") +graph.connect("close-cycle", "finalize-agent", kind="condition", label="completed") def agent_command(): @@ -495,6 +606,7 @@ def invoke(ctx, action): return result +@graph.step("check-agent", title="Check agent readiness", phase="before_all", outputs=["agent_status"]) @runner.phase("before_all") def before_all(ctx): # Check availability once; later phases must not start an independent loop. @@ -502,6 +614,7 @@ def before_all(ctx): ctx.emit_result({"agent_cycle": {"status": status}}) +@graph.step("record-inputs", title="Record cycle inputs", phase="before_each", inputs=["agent_status"], outputs=["cycle_input"]) @runner.phase("before_each") def before_each(ctx): # Record the fixed inputs so every external action is auditable. @@ -515,6 +628,7 @@ def before_each(ctx): ) +@graph.step("run-agent", title="Run bounded agent cycle", phase="execute", inputs=["cycle_input"], outputs=["agent_result"]) @runner.phase("execute") def execute(ctx): # Exactly one unit of agent work; OpenOrbit schedules a future iteration. @@ -522,6 +636,7 @@ def execute(ctx): ctx.emit_result({"agent_cycle": {"iteration": ctx.loop_index, "result": result}}) +@graph.step("confirm-agent-state", title="Confirm agent state", phase="verify", inputs=["agent_result"], outputs=["verified_status"]) @runner.phase("verify") def verify(ctx): # Re-read status rather than assuming the prior action completed correctly. @@ -529,12 +644,14 @@ def verify(ctx): ctx.emit_result({"agent_cycle": {"iteration": ctx.loop_index, "status": status}}) +@graph.step("close-cycle", title="Close cycle", phase="after_each", inputs=["verified_status"], outputs=["cycle_complete"]) @runner.phase("after_each") def after_each(ctx): # The external process has already returned; no daemon cleanup is required. ctx.log("Completed one bounded external agent cycle") +@graph.step("finalize-agent", title="Finalize agent evaluation", phase="after_all", inputs=["cycle_complete"], outputs=["final_status"]) @runner.phase("after_all") def after_all(ctx): ctx.log("Finalized the external agent evaluation") @@ -557,7 +674,14 @@ def after_all(ctx): import os import shlex -from orbit_sdk import runner +from orbit_sdk import graph, runner + +graph.connect("preflight-probes", "prepare-probes") +graph.connect("prepare-probes", "run-probe-matrix", label="prepared inputs") +graph.connect("run-probe-matrix", "collect-probe-evidence", kind="data", label="probe report") +graph.connect("collect-probe-evidence", "close-probe-cycle") +graph.connect("close-probe-cycle", "prepare-probes", kind="loop", label="next cycle") +graph.connect("close-probe-cycle", "finalize-probe-monitor", kind="condition", label="completed") def probe_command(): @@ -603,6 +727,7 @@ def invoke(ctx, action): return result +@graph.step("preflight-probes", title="Preflight probe matrix", phase="before_all", outputs=["probe_contract"]) @runner.phase("before_all") def before_all(ctx): # A fixed probe set keeps the gate repeatable and its evidence comparable. @@ -612,6 +737,7 @@ def before_all(ctx): ctx.emit_result({"probe_gate": {"preflight": preflight}}) +@graph.step("prepare-probes", title="Prepare probes", phase="before_each", inputs=["probe_contract"], outputs=["prepared_probes"]) @runner.phase("before_each") def before_each(ctx): # Prepare disposable inputs without mutating the target repository. @@ -619,6 +745,7 @@ def before_each(ctx): ctx.emit_result({"probe_gate": {"iteration": ctx.loop_index, "prepared": prepared}}) +@graph.step("run-probe-matrix", title="Run probe matrix", phase="execute", inputs=["prepared_probes"], outputs=["probe_report"]) @runner.phase("execute") def execute(ctx): # Run the complete fixed matrix once and retain the tool's structured report. @@ -626,6 +753,7 @@ def execute(ctx): ctx.emit_result({"probe_gate": {"iteration": ctx.loop_index, "report": report}}) +@graph.step("collect-probe-evidence", title="Collect probe evidence", phase="verify", inputs=["probe_report"], outputs=["evidence_gate"]) @runner.phase("verify") def verify(ctx): # Collect final evidence separately so a supervisor can make an independent decision. @@ -633,11 +761,13 @@ def verify(ctx): ctx.emit_result({"probe_gate": {"iteration": ctx.loop_index, "evidence": evidence}}) +@graph.step("close-probe-cycle", title="Close probe cycle", phase="after_each", inputs=["evidence_gate"], outputs=["cycle_complete"]) @runner.phase("after_each") def after_each(ctx): ctx.log("Completed one evidence-gated probe matrix") +@graph.step("finalize-probe-monitor", title="Finalize drift monitor", phase="after_all", inputs=["cycle_complete"], outputs=["final_status"]) @runner.phase("after_all") def after_all(ctx): ctx.log("Finalized the evidence-gated probe evaluation") @@ -733,6 +863,19 @@ def shutdown(self) -> None: processes = list(self._processes.values()) for process in processes: self._stop_process_group(process) + for run in self.runs(): + if run.status not in {"queued", "running", "awaiting_approval"}: + continue + run.status, run.current_step, run.current_phase = "cancelled", None, None + run.updated_at, run.finished_at = now(), now() + run.step_results.append( + { + "step_id": "orbit-shutdown", + "error": "OpenOrbit stopped before this pipeline completed.", + "ended_at": now(), + } + ) + self._save(run) def _recover_interrupted_runs(self) -> None: """Do not present orphaned in-memory pipelines as still running. @@ -799,7 +942,14 @@ def runner_templates() -> list[dict[str, str]]: import json import re -from orbit_sdk import runner +from orbit_sdk import graph, runner + +graph.connect("validate-journey", "plan-journey") +graph.connect("plan-journey", "run-journey", label="focused cases") +graph.connect("run-journey", "review-journey", kind="data", label="browser evidence") +graph.connect("review-journey", "retain-journey") +graph.connect("retain-journey", "plan-journey", kind="loop", label="next iteration") +graph.connect("retain-journey", "finalize-journey", kind="condition", label="completed") # Validate only configuration that the runner cannot safely infer. This runs # once when an evaluation process starts, before its iteration loop. @@ -851,12 +1001,14 @@ def plan(ctx, state): reason = "Previously failed journeys require confirmation." if failed else "Rotate one fixed journey to retain broad, bounded coverage." return {"case_ids": [str(case.get("id")) for case in focused], "rules": rules, "reason": reason, "supervisor_feedback": feedback} +@graph.step("validate-journey", title="Validate journey contract", phase="before_all", outputs=["journey_contract"]) @runner.phase("before_all") def before_all(ctx): # Process-level preparation: run once before OpenOrbit starts repeating. validate(ctx) ctx.log("Validated the bounded user-journey contract") +@graph.step("plan-journey", title="Plan focused journey", phase="before_each", inputs=["journey_contract"], outputs=["journey_plan"]) @runner.phase("before_each") def before_each(ctx): # Iteration-level preparation: persist a plan that the execute phase consumes. @@ -867,6 +1019,7 @@ def before_each(ctx): ctx.emit_result({"user_journey": {"iteration": ctx.loop_index, "case_count": len(ctx.test_cases), "plan": journey_plan}}) ctx.log(f"Planned {len(journey_plan['case_ids'])} focused journey case(s): {journey_plan['reason']}") +@graph.step("run-journey", title="Run browser journey", phase="execute", inputs=["journey_plan"], outputs=["journey_evidence"]) @runner.phase("execute") def execute(ctx): # Execute only the focused fixed cases; Playwright returns screenshots and @@ -887,14 +1040,17 @@ def execute(ctx): save_state(ctx, state) ctx.emit_result({"user_journey": {"iteration": ctx.loop_index, "plan": journey_plan, "passed": passed, "failed": len(results) - passed, "results": results, "evidence": evidence, "handoff": state["handoff"]}}) +@graph.step("review-journey", title="Review journey evidence", phase="verify", inputs=["journey_evidence"], outputs=["journey_handoff"]) @runner.phase("verify") def verify(ctx): # Expose the persisted handoff as structured run output for supervision. state = load_state(ctx) ctx.emit_result({"user_journey": {"next_iteration": state.get("handoff", {}), "state_path": str(state_path(ctx))}}) ctx.log("Stored the journey summary, reasons, and behavior rules for the next iteration") +@graph.step("retain-journey", title="Retain journey result", phase="after_each", inputs=["journey_handoff"], outputs=["iteration_complete"]) @runner.phase("after_each") def after_each(ctx): ctx.log("Closed this bounded browser journey") +@graph.step("finalize-journey", title="Finalize journey evaluation", phase="after_all", inputs=["iteration_complete"], outputs=["final_status"]) @runner.phase("after_all") def after_all(ctx): ctx.log("Finalized the user-journey evaluation") @@ -914,6 +1070,7 @@ def after_all(ctx): ctx.log("Finalized the user-journey evaluation") "source": """# Requirements\n# - PROJECT_ROOT is a Git repository.\n# - The build selects fixed browser test cases and a browser base URL.\n# - Candidate source changes are supplied through the normal reviewed change flow.\n# This runner never launches an external improvement script or commits a change.\n\nimport hashlib\nimport json\nimport re\nfrom pathlib import Path\n\nfrom orbit_sdk import runner\n\nREQUIRED_SUFFICIENT_EVALUATIONS = 3\n\ndef state_path(ctx):\n build_id = re.sub(r"[^a-zA-Z0-9_-]+", "-", str(ctx.build.get("id") or "manual"))\n directory = ctx.app_data / "improvement-cycles"\n directory.mkdir(parents=True, exist_ok=True)\n return directory / f"{build_id}.json"\n\ndef load_state(ctx):\n path = state_path(ctx)\n if not path.exists():\n return {"candidate_fingerprint": None, "sufficient_evaluations": 0, "history": []}\n return json.loads(path.read_text(encoding="utf-8"))\n\ndef save_state(ctx, state):\n state["history"] = state.get("history", [])[-24:]\n state_path(ctx).write_text(json.dumps(state, ensure_ascii=False, indent=2), encoding="utf-8")\n\ndef git(ctx, *args):\n return ctx.exec(["git", *args], cwd=ctx.project_root, timeout=300)\n\ndef candidate(ctx):\n patch = git(ctx, "diff", "--binary", "--")\n changed = [line for line in git(ctx, "diff", "--name-only").splitlines() if line]\n return (hashlib.sha256(patch.encode("utf-8")).hexdigest() if patch else None), changed\n\n@runner.phase("before_all")\ndef before_all(ctx):\n git(ctx, "rev-parse", "--show-toplevel")\n if not ctx.build.get("browser_base_url") or not ctx.test_cases:\n raise ValueError("Select a browser base URL and fixed test cases for a native improvement cycle")\n ctx.log("Validated a Git-backed, OpenOrbit-native improvement cycle")\n\n@runner.phase("before_each")\ndef before_each(ctx):\n fingerprint, changed = candidate(ctx)\n ctx.emit_result({"improvement_cycle": {"iteration": ctx.loop_index, "candidate_fingerprint": fingerprint, "changed_paths": changed}})\n ctx.log("Captured the candidate baseline before validation")\n\n@runner.phase("execute")\ndef execute(ctx):\n evidence = ctx.playwright_journey()\n results = evidence["results"]\n passed = all(item["passed"] for item in results)\n fingerprint, changed = candidate(ctx)\n ctx.emit_result({"improvement_cycle": {"iteration": ctx.loop_index, "candidate_fingerprint": fingerprint, "changed_paths": changed, "passed": passed, "evidence": evidence}})\n if not passed:\n raise SystemExit("A fixed validation journey failed")\n\n@runner.phase("verify")\ndef verify(ctx):\n state = load_state(ctx)\n fingerprint, changed = candidate(ctx)\n if not fingerprint:\n state["candidate_fingerprint"] = None\n state["sufficient_evaluations"] = 0\n verdict = "no_candidate"\n elif state.get("candidate_fingerprint") == fingerprint:\n state["sufficient_evaluations"] = int(state.get("sufficient_evaluations", 0)) + 1\n verdict = "ready_for_approval" if state["sufficient_evaluations"] >= REQUIRED_SUFFICIENT_EVALUATIONS else "continue_validation"\n else:\n state["candidate_fingerprint"] = fingerprint\n state["sufficient_evaluations"] = 1\n verdict = "continue_validation"\n state.setdefault("history", []).append({"iteration": ctx.loop_index, "fingerprint": fingerprint, "paths": changed, "verdict": verdict})\n save_state(ctx, state)\n ctx.emit_result({"improvement_cycle": {"candidate_fingerprint": fingerprint, "changed_paths": changed, "sufficient_evaluations": state["sufficient_evaluations"], "required_evaluations": REQUIRED_SUFFICIENT_EVALUATIONS, "verdict": verdict}})\n ctx.log(f"Candidate verdict: {verdict}")\n\n@runner.phase("after_each")\ndef after_each(ctx): ctx.log("Retained native improvement evidence for supervision")\n@runner.phase("after_all")\ndef after_all(ctx): ctx.log("Finalized the native improvement cycle without committing changes")\n\nif __name__ == "__main__": runner.main()\n""", }, ] + templates[1]["source"] = EXTERNAL_COMMAND_ADAPTER_TEMPLATE templates[-1] = { "id": "native-improvement-cycle", "name": "Prompt improvement validation", @@ -1121,23 +1278,35 @@ def save_template_translation( @staticmethod def _quick_start_browser_runner() -> str: - return """from orbit_sdk import runner + return """from orbit_sdk import graph, runner + +graph.connect("validate-browser", "run-browser-journey") +graph.connect("run-browser-journey", "verify-browser-evidence", kind="data", label="journey evidence") +graph.connect("verify-browser-evidence", "finalize-browser-evaluation") +@graph.step("validate-browser", title="Validate browser target", phase="before_all", outputs=["browser_target"]) @runner.phase("before_all") def before_all(ctx): if not ctx.build.get("browser_base_url"): raise ValueError("Quick start browser evaluation requires a browser base URL") +@graph.step("run-browser-journey", title="Run browser journey", phase="execute", inputs=["browser_target"], outputs=["journey_evidence"]) @runner.phase("execute") def execute(ctx): evidence = ctx.playwright_journey() if not all(item["passed"] for item in evidence["results"]): raise SystemExit("A browser journey failed") +@graph.step("verify-browser-evidence", title="Verify journey evidence", phase="verify", inputs=["journey_evidence"], outputs=["journey_verdict"]) @runner.phase("verify") def verify(ctx): ctx.log("Quick start browser evaluation completed") +@graph.step("finalize-browser-evaluation", title="Finalize browser evaluation", phase="after_all", inputs=["journey_verdict"], outputs=["completed_evaluation"]) +@runner.phase("after_all") +def after_all(ctx): + ctx.log("Finalized the one-shot browser evaluation") + if __name__ == "__main__": runner.main() """ @@ -1147,7 +1316,7 @@ def _built_in_quick_starts(self) -> list[dict[str, Any]]: { "schema_version": 1, "id": "openorbit.user-journey-smoke-test", - "version": "1.0.1", + "version": "1.0.2", "name": "User journey smoke test", "description": "Create a browser-based smoke test. Requires a running app and Playwright browser.", "publisher": {"name": "OpenOrbit"}, @@ -1315,7 +1484,7 @@ def _built_in_quick_starts(self) -> list[dict[str, Any]]: { "schema_version": 1, "id": "openorbit.site-exploration-review", - "version": "1.0.0", + "version": "1.0.1", "name": "Site exploration review", "description": "Explore a site through safe links and leave evidence-backed product feedback. Requires a running app, Playwright browser, and LangGraph.", "publisher": {"name": "OpenOrbit"}, @@ -1450,7 +1619,7 @@ def _built_in_quick_starts(self) -> list[dict[str, Any]]: { "schema_version": 1, "id": "openorbit.agent-self-improvement", - "version": "1.1.0", + "version": "1.1.1", "name": "Agent self-improvement", "description": "Improve a managed prompt from retained responses of the real target AI. Requires a Git repository, prompt file, and configured model profile.", "publisher": {"name": "OpenOrbit"}, @@ -1610,7 +1779,7 @@ def _built_in_quick_starts(self) -> list[dict[str, Any]]: { "schema_version": 1, "id": "openorbit.ai-slo-drift-monitor", - "version": "1.0.0", + "version": "1.0.1", "name": "AI SLO and behavior drift monitor", "description": "Repeatedly assess AI quality, safety, latency, and cost against a fixed baseline. Connects an existing structured AI evaluator; OpenOrbit retains the evidence, supervision, and improvement decisions.", "publisher": {"name": "OpenOrbit"}, @@ -4107,12 +4276,6 @@ def _execute(self, run_id: str) -> None: self._execute_step(run_id, step, loop_index, resources, allow_terminal=True) if self._load(run_id).status in {"failed", "cancelled"}: break - if ( - self._load(run_id).status == "running" - and run.execution_mode == "run" - and self._latest_cycle_has_persona_evidence(self._load(run_id)) - ): - self._complete_supervision(run_id) if ( loop_index < run.loop_limit and run.repeat_interval_minutes @@ -4147,6 +4310,12 @@ def _execute(self, run_id: str) -> None: now(), ) self._save(run) + if ( + run.status == "succeeded" + and run.execution_mode == "run" + and self._latest_cycle_has_persona_evidence(run) + ): + self._complete_supervision(run_id) @staticmethod def _latest_cycle_has_persona_evidence(run: Run) -> bool: diff --git a/backend/orbit_sdk.py b/backend/orbit_sdk.py index 7889183..c651ec9 100644 --- a/backend/orbit_sdk.py +++ b/backend/orbit_sdk.py @@ -1266,10 +1266,13 @@ def complete_model(self, prompt: str) -> dict[str, str]: aws_profile=str(profile.get("aws_profile", "")), ) provider = AzureOpenAIProvider() if settings.provider == "azure-openai" else BedrockProvider() + self.log(f"target model request started: {settings.provider}/{settings.model}") + response = provider.complete(settings, prompt) + self.log(f"target model request completed: {settings.provider}/{settings.model}") return { "profile_name": str(profile.get("profile_name", "")), "model": settings.model, - "response": provider.complete(settings, prompt), + "response": response, } @property @@ -1403,6 +1406,7 @@ def playwright_journey(self, cases: list[dict[str, object]] | None = None) -> di selected_cases = cases if cases is not None else self.test_cases if not selected_cases: raise ValueError("at least one fixed test case is required for a Playwright journey") + self.log(f"browser journey started: {len(selected_cases)} case(s) against {base_url}") artifacts = ( self.app_data / "artifacts" @@ -1445,6 +1449,8 @@ def playwright_journey(self, cases: list[dict[str, object]] | None = None) -> di except (json.JSONDecodeError, IndexError) as error: raise RuntimeError("Playwright did not return structured journey evidence") from error evidence["artifacts_directory"] = str(artifacts) + passed = sum(bool(item.get("passed")) for item in evidence.get("results", [])) + self.log(f"browser journey completed: {passed}/{len(selected_cases)} case(s) passed") self.emit_result({"browser_journey": evidence}) return evidence @@ -1504,7 +1510,9 @@ def forward_output() -> None: reader.join() output = "".join(lines) if process.returncode: + self.log(f"exec failed with exit code {process.returncode}: {' '.join(command)}") raise SystemExit(process.returncode) + self.log(f"exec completed: {' '.join(command)}") return output diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index f96af4f..4882865 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -965,6 +965,71 @@ def test_site_exploration_quick_start_uses_the_langgraph_runner(): assert "logout|signout|delete" in runner["source"] +@pytest.mark.parametrize( + ("quick_start_id", "phases"), + [ + ("openorbit.user-journey-smoke-test", ["before_all", "execute", "verify", "after_all"]), + ("openorbit.site-exploration-review", ["before_all", "execute", "verify", "after_all"]), + ( + "openorbit.agent-self-improvement", + ["before_all", "before_each", "execute", "verify", "after_each", "after_all"], + ), + ( + "openorbit.ai-slo-drift-monitor", + ["before_all", "before_each", "execute", "verify", "after_each", "after_all"], + ), + ], +) +def test_quick_start_runner_graph_matches_its_execution_purpose(monkeypatch, quick_start_id, phases): + store = store_module.ConsoleStore() + quick_start = next(item for item in store._built_in_quick_starts() if item["id"] == quick_start_id) + monkeypatch.setattr(sdk, "graph", sdk.Graph()) + + exec( + compile(quick_start["assets"]["runner"]["source"], quick_start_id, "exec"), + {"__name__": quick_start_id}, + ) + + definition = sdk.graph.definition() + assert [node["phase"] for node in definition["nodes"]] == phases + assert definition["edges"] + + +@pytest.mark.parametrize( + ("template_id", "phases"), + [ + ("user-journey-cycle", ["before_all", "before_each", "execute", "verify", "after_each", "after_all"]), + ( + "external-command-adapter", + ["before_all", "before_each", "execute", "verify", "after_each", "after_all"], + ), + ( + "native-improvement-cycle", + ["before_all", "before_each", "execute", "verify", "after_each", "after_all"], + ), + ("site-exploration", ["before_all", "execute", "verify", "after_all"]), + ("json-agent-cycle", ["before_all", "before_each", "execute", "verify", "after_each", "after_all"]), + ( + "evidence-gated-probe-cycle", + ["before_all", "before_each", "execute", "verify", "after_each", "after_all"], + ), + ], +) +def test_runner_templates_publish_a_lifecycle_graph(monkeypatch, template_id, phases): + source = next( + template["source"] + for template in store_module.ConsoleStore.runner_templates() + if template["id"] == template_id + ) + monkeypatch.setattr(sdk, "graph", sdk.Graph()) + + exec(compile(source, template_id, "exec"), {"__name__": template_id}) + + definition = sdk.graph.definition() + assert [node["phase"] for node in definition["nodes"]] == phases + assert definition["edges"] + + def test_ai_slo_drift_quick_start_uses_a_recurring_evidence_gate(): store = store_module.ConsoleStore() quick_start = next( diff --git a/frontend/src/features/evaluations/page.tsx b/frontend/src/features/evaluations/page.tsx index 6e62163..9c3cc68 100644 --- a/frontend/src/features/evaluations/page.tsx +++ b/frontend/src/features/evaluations/page.tsx @@ -176,7 +176,7 @@ export function EvaluationsPage({ [pageSize, setPageSize] = useState(15), [deleteSelectionOpen, setDeleteSelectionOpen] = useState(false), [retryingRun, setRetryingRun] = useState(null); - const retryCopy = locale === "ko" ? { title: "실행 재시도", warning: "재시도는 작업 디렉터리 또는 외부 대상의 중간 결과를 변경할 수 있습니다.", restart: "1부터 다시 시작", resume: "마지막 이터레이션부터 재시도", cancel: "취소" } : locale === "ja" ? { title: "実行を再試行", warning: "再試行により作業ディレクトリまたは外部ターゲットの中間結果が変わる可能性があります。", restart: "反復 1 から再開", resume: "最後の反復から再試行", cancel: "キャンセル" } : { title: "Retry run", warning: "Retrying can change intermediate results in the working directory or external target.", restart: "Restart from iteration 1", resume: "Retry from the last iteration", cancel: "Cancel" }; + const retryCopy = locale === "ko" ? { title: "실행 재시도", warning: "재시도는 작업 디렉터리 또는 외부 대상의 중간 결과를 변경할 수 있습니다.", restart: "1부터 다시 시작", resume: "마지막 이터레이션부터 재시도" } : locale === "ja" ? { title: "実行を再試行", warning: "再試行により作業ディレクトリまたは外部ターゲットの中間結果が変わる可能性があります。", restart: "反復 1 から再開", resume: "最後の反復から再試行" } : { title: "Retry run", warning: "Retrying can change intermediate results in the working directory or external target.", restart: "Restart from iteration 1", resume: "Retry from the last iteration" }; const selectedSource = initialSelectedRun ?? selectedInternal; const selected = selectedSource ? runs.find((run) => run.id === selectedSource.id) ?? selectedSource @@ -602,8 +602,19 @@ export function EvaluationsPage({ const traces = step.result?.workflow_functions; return Array.isArray(traces) ? [...traces].reverse() : []; }).find((trace) => typeof trace === "object" && trace !== null && trace.id === node.id) as { status?: WorkflowGraphNode["status"] } | undefined; - const status: WorkflowGraphNode["status"] = functionTrace?.status + const supervisingEvidence = selected?.status === "running" + && selected.current_phase === "verify" + && selected.supervisor_status === "pending" + && node.phase === "verify" + && functionTrace?.status === "succeeded"; + const interruptedFunction = selected && terminal(selected.status) && functionTrace?.status === "running"; + const status: WorkflowGraphNode["status"] = interruptedFunction + ? "skipped" + : functionTrace?.status + && !supervisingEvidence ? functionTrace.status + : supervisingEvidence + ? "running" : selected?.status === "running" && node.phase === selected.current_phase && !phaseHasFunctionTrace && firstNodeInPhase?.id === node.id ? "running" : latestStep?.in_progress @@ -1294,7 +1305,6 @@ export function EvaluationsPage({

{retryCopy.warning}

-