From 4a60fd336464250dcd305e8147222834c887e766 Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Thu, 3 Sep 2026 15:45:29 +0300 Subject: [PATCH] feat(agents): add the UiPath Delegate SDK agent as a built-in harness [PILOT-7267] Move `DelegateSdkAgent` (agent.type `delegate-sdk`) out of the internal coder_eval_uipath plugin into core: it drives UiPath Autopilot's Delegate agent through the `@uipath/delegate-stdio` Node host (JSON-Lines over stdio), with the S2S token-file refresher, per-generation transcript reconstruction, stall recovery and failure classification carried over intact. - `AgentKind.DELEGATE_SDK`, `DelegateSdkAgentConfig` (sdk_options / project_id / session_id), registered via `register_builtins`; empty `delegate-sdk` extra. - The layer-5 `sdk_options` override guard is now registry-driven: any kind whose config declares `sdk_options` qualifies (claude-code + delegate-sdk today). - Delegate-routed pricing rows (hyphenated ids) added to pricing.py and mirrored in evalboard/lib/pricing.ts, verbatim from the plugin. - Host stderr tracing forwards at INFO when DELEGATE_STDIO_VERBOSE forces it on a quiet run (replaces the plugin's logging bridge). - docs/agents/DELEGATE_SDK.md, parity table column, smoke task, ported test suites. Related PRs: UiPath/coder_eval_uipath (drops the plugin agent, nightly on public npm) and UiPath/Autopilot (delegate-stdio 0.1.3 public-npm prep). Co-Authored-By: Claude Fable 5.1 --- CLAUDE.md | 2 +- README.md | 5 +- docker/Dockerfile | 4 + docs/EXTENDING.md | 5 +- docs/USER_GUIDE.md | 2 +- docs/agents/DELEGATE_SDK.md | 347 ++ docs/agents/HARNESS_PARITY.md | 46 +- docs/index.md | 7 +- docs/llms.txt | 3 +- evalboard/app/_components/harness-badge.tsx | 4 +- evalboard/lib/pricing.ts | 19 + mkdocs.yml | 2 + pyproject.toml | 20 +- src/coder_eval/agents/__init__.py | 7 +- .../agents/_delegate_s2s_token_file.py | 339 ++ src/coder_eval/agents/delegate_sdk_agent.py | 2043 ++++++++++++ src/coder_eval/models/__init__.py | 2 + src/coder_eval/models/agent_config.py | 53 +- src/coder_eval/models/enums.py | 1 + src/coder_eval/orchestration/overrides.py | 38 +- src/coder_eval/pricing.py | 33 +- tasks/delegate_sdk_smoke_test.yaml | 42 + tests/test_custom_lint.py | 4 +- tests/test_delegate_s2s_token_file.py | 455 +++ tests/test_delegate_sdk_agent.py | 2885 +++++++++++++++++ tests/test_delegate_sdk_agent_live.py | 133 + tests/test_delegate_sdk_builtin.py | 185 ++ tests/test_overrides_engine.py | 42 + uv.lock | 2 +- 29 files changed, 6691 insertions(+), 39 deletions(-) create mode 100644 docs/agents/DELEGATE_SDK.md create mode 100644 src/coder_eval/agents/_delegate_s2s_token_file.py create mode 100644 src/coder_eval/agents/delegate_sdk_agent.py create mode 100644 tasks/delegate_sdk_smoke_test.yaml create mode 100644 tests/test_delegate_s2s_token_file.py create mode 100644 tests/test_delegate_sdk_agent.py create mode 100644 tests/test_delegate_sdk_agent_live.py create mode 100644 tests/test_delegate_sdk_builtin.py diff --git a/CLAUDE.md b/CLAUDE.md index ac00d5ef..13d6cfe6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -144,7 +144,7 @@ action.yml # Published composite GitHub Action (coder-ev - **Sub-agent token accounting**: There is NO separate per-sub-agent field. Every sub-agent generation is captured as a `parent_tool_use_id`-tagged `AssistantMessage` in the turn transcript, so per-sub-agent usage is derived by grouping those messages on that id (the evalboard's `aggregateSubAgentUsage` does exactly this). Claude bubbles its sub-agent's intermediate generations into the parent stream natively, and the **terminal** generation (delivered as the Agent tool result, never streamed) is synthesized into one via `_synthesize_subagent_terminal_message` from `tool_use_result.usage`. Codex reconstructs all child generations from the child rollout (`_recover_subagent_tool_calls`). The turn total already includes sub-agent cost — Claude via the SDK's cumulative `model_usage`; Codex via `_fold_subagent_tokens`, which folds the child messages (their real per-generation tokens) into the parent total. `CommandTelemetry.result_summary` is stored **untruncated** (no 200-char cap) so sub-agent returns are preserved whole. Set `CODER_EVAL_RAW_SDK_LOG=1` to dump every raw SDK event to the task log for inspection. - **Reconciliation message (stream self-reconciles to the turn total)**: The per-message stream consistently under-reports the authoritative turn total — a fixed prompt slice (~512 input tokens on Claude) is billed on no SDK-emitted message, and sub-agent input/cache only partially bubbles up. So `EventCollector.build_turn_record` appends one synthetic `ReconciliationMessage` (`role="reconciliation"`, in the `TranscriptMessage` union) per turn, carrying the per-bucket residual = `token_usage` − Σ(assistant message buckets). The invariant: **summing the four token buckets across `TurnRecord.messages` (assistant + reconciliation) equals `token_usage` exactly**, for both Claude and Codex (Codex's stream is already complete after `_recover_subagent_tool_calls`, so its residual is usually 0 and no entry is emitted). This is what lets the evalboard SUM the message stream as the source of truth instead of reading a separate aggregate ("agent tokens"): `selectTokenTotals` returns the stream sum whenever a reconciliation entry is present, and the timeline renders it as its own row. It is agent-agnostic (booked at the single `EventCollector` seam), carries no cost (cost stays on `token_usage`), and is excluded from generation/turn counts and the cost simulator. The LiteLLM open-weight actual-cost join (`litellm_cost.apply_actual_cost`) deliberately writes cost at the TURN level only (`token_usage.total_cost_usd` = the real OpenRouter bill) plus the per-call `TurnRecord.provider_call_costs` audit record; it does NOT touch the message token buckets, so `EventCollector` stays the single writer and this invariant holds on every backend. The Python `token_usage`/`total_token_usage` aggregate is unchanged and still authoritative for budget/judges/reports. - **Reference solutions are directory-only, and shielded (partially) from the agent**: `task.reference` is a single required `directory:` (relative to the task YAML) — the inline `code:` / single-file `file:` forms are gone, because a directory is the only shape that can be permission-gated as a unit; a `model_validator(mode="before")` gives the removed forms a migration error. The orchestrator stages a **per-run private copy** (`orchestration/evaluation.py::stage_reference_dir`, symlinks stripped) into a tempdir, removed in `_cleanup` via `path_utils.rmtree_restrictive` (keyed on `_reference_staging_root`, recorded BEFORE the copy so a failed copy still cleans up; `rmtree(ignore_errors=True)` silently declines on a tree left at 000) and deliberately never preserved into `run_dir/artifacts`. That copy is held at mode `000` for the whole of every `agent.communicate` call via **`Sandbox.set_permissions`**, the driver-aware wrapper over `fs_permissions.py::set_permissions`. Windows **stack**: exiting restores the *enclosing* window's mode, only the outermost exit restores the pre-window mode — that is what makes a mid-turn re-grant (`mode=READ_ONLY_MODE`) expressible, and it covers two windows at the same mode so no refcount is needed. The window is enforced **only inside a docker container** (`Sandbox.enforces_permission_windows`) and is a no-op on the host, where the agent shares our uid. **That gate keys on the `CODER_EVAL_IN_CONTAINER` env var, NOT `sandbox.driver`** — `run_task_internal_command` rewrites `driver: docker` → `tempdir` before building the in-container Orchestrator, so a driver-based gate would silently disable the anti-cheat on exactly the path that needs it (regression-guarded by `TestSandboxDriverGate`); `resolve_reference_dir` gates its `/work/references` branch on the same var for the same reason. The task directory is **not** shielded (`:ro` mount → EROFS, and the same YAML is readable at `/work/input`). Criteria address reference files with the `$REFERENCE_DIR` token (same resolver as `$TASK_DIR`) and the `REFERENCE_DIR` env var for `run_command`; `reference_comparison` names one file via `reference_file`. Docker mounts a throwaway **read-write** copy at `/work/references` (a `:ro` mount cannot be chmod'd — EROFS), masks the in-task-dir original with an empty tmpfs, and drops `DAC_OVERRIDE`/`DAC_READ_SEARCH`. `FOWNER`/`CHOWN` are deliberately **NOT** dropped: the in-container orchestrator that applies the window is the same root process with the same caps, so dropping `FOWNER` breaks *the harness's own* chmod wherever the bind mount preserves a non-root owner (native Linux — verified: `chmod: Operation not permitted`), i.e. exactly where the drop would otherwise bite. A window that cannot be applied is now a hard error, not a warning: `Sandbox.set_permissions` passes `strict=True` whenever it enforces, so an unprotected run fails instead of producing a normal-looking score. **KNOWN GAP — this is defense-in-depth, not a boundary**: (a) `chmod(2)` is gated on owner-or-`CAP_FOWNER` and the container runs as root owning the copy, so a deliberate `chmod 755 /work/references` restores access; (b) the window spans `agent.communicate` only, and nothing reaps agent child processes at turn end, so a backgrounded read loop succeeds once the window closes. The **write** half of (b) is closed — `path_utils.digest_tree` hashes the tree at staging and `Orchestrator._verify_reference_integrity` re-checks before grading, raising `ReferenceTamperedError` (→ `FinalStatus.ERROR`) on a mismatch so an agent cannot overwrite the reference to drive `reference_comparison` to 1.0. Passive reads are blocked; an adversarial agent is not. Full containment requires running the agent as a non-root uid AND holding the window for the agent's whole lifetime — follow-up. `tasks/anti_cheat_reference` probes the passive-read half. -- **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. OpenCode likewise keeps a native unit — the CLI streams a real multi-step loop per `communicate()` (`step_start`/`step_finish`), so `max_turns: N` allows N complete steps and cuts cleanly when step N+1 begins. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex and Antigravity (both run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it), `allowed_tools`/`disallowed_tools`/`system_prompt` on OpenCode (no CLI knob; warned at `start()`, not enforced), and **`agent.plugins[].path` depth** — claude-code REQUIRES a plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, while Codex and Antigravity scan both depths and accept either. That is the costly direction: the wrong depth produces no error, every positive row of an activation suite scores 0, and the suite reports recall 0.0, which reads exactly like a skill that never triggers. Held to the plugin-root shape (for `SKILL_SOURCE_PATH` only) by lint rule CE045. `plugins` on OpenCode is **honored for skills**: each local plugin root is mapped to the `skills` dir its `.claude-plugin/plugin.json` declares (default `/skills`, never the root itself — `skills.paths` is scanned recursively and a plugin root can hold a self-referential symlink) and injected via `OPENCODE_CONFIG_CONTENT`, which `--pure` does not suppress; a plugin's agents/hooks/commands/MCP servers are still dropped. Full table + rationale: docs/agents/HARNESS_PARITY.md. +- **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. OpenCode likewise keeps a native unit — the CLI streams a real multi-step loop per `communicate()` (`step_start`/`step_finish`), so `max_turns: N` allows N complete steps and cuts cleanly when step N+1 begins. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex and Antigravity (both run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it), `allowed_tools`/`disallowed_tools`/`system_prompt` on OpenCode (no CLI knob; warned at `start()`, not enforced), and **`agent.plugins[].path` depth** — claude-code REQUIRES a plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, while Codex and Antigravity scan both depths and accept either. That is the costly direction: the wrong depth produces no error, every positive row of an activation suite scores 0, and the suite reports recall 0.0, which reads exactly like a skill that never triggers. Held to the plugin-root shape (for `SKILL_SOURCE_PATH` only) by lint rule CE045. `plugins` on OpenCode is **honored for skills**: each local plugin root is mapped to the `skills` dir its `.claude-plugin/plugin.json` declares (default `/skills`, never the root itself — `skills.paths` is scanned recursively and a plugin root can hold a self-referential symlink) and injected via `OPENCODE_CONFIG_CONTENT`, which `--pure` does not suppress; a plugin's agents/hooks/commands/MCP servers are still dropped. **delegate-sdk** (UiPath Autopilot's Delegate agent, driven through the `@uipath/delegate-stdio` Node host) also keeps a native unit: `max_turns` is forwarded to the host as `maxSteps`, and the SDK stops a limit-hit tool call BEFORE emitting its `tool_call` event, so `max_turns: 1` routing rows are structurally unmeasurable there (`skill_triggered` never sees the engagement); it has NO cooperative stop (`supports_cooperative_stop` False — the host owns its inner loop), ignores `allowed_tools`/`disallowed_tools`/`system_prompt`/`setting_sources` with a one-time warning, and maps only the FIRST `plugins:` entry to `/skills` (`bundledSkillsPath`, the claude-code depth — a bare skills dir loads nothing). Full table + rationale: docs/agents/HARNESS_PARITY.md. - **sandbox isolation**: Tasks that don't need MCP servers should set `setting_sources: []` in their `agent:` block to isolate the sandbox from the host project's CLAUDE.md and settings. Without this, the host project's CLAUDE.md (often 20 KB+) is injected into every API call, inflating cache-creation tokens and cost significantly. - **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all *task-level* run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block. The one *per-criterion* cap, `stop_early.decide_within`, deliberately lives on `LiveSuccessCriterion` instead (see below) — the watcher must attribute a decision-step timeout to a specific criterion, which `RunLimits` (task-scoped, criterion-agnostic) cannot express. - **Early stop on criterion (opt-in, per-criterion arming)**: a `stop_early:` block (`StopEarlyPolicy`) on a criterion ends a single-shot run early once the run's **armed** criteria decide the outcome, so a raised `max_turns` isn't wasted on the smoke flavor. The block's PRESENCE is the arming and alone activates the watcher — there is **no run-level master switch**: `run_limits.stop_early: false` is the run-level KILL SWITCH that force-disarms every block (the one-line experiment-variant/`-D` override for an authoritative full run), and `run_limits.stop_early: true` (the removed master arm) is a hard `EarlyStopConfigError` at resolution. The block exists on `LiveSuccessCriterion` only (currently `skill_triggered`, `command_executed` — so arming an unobservable criterion is unrepresentable, a pydantic extra-forbid error). Arming carries one implicit trigger (a native live-fail may fail-stop the run); its keys refine it: `on_pass: stop` (pass-stop the moment the criterion live-passes; default `continue` just latches) and `decide_within: N` (still undecided after N tool-call steps latches an **effective fail**, fed through the same fail-stop rule, reported as `decision_budget_exceeded` — an ordinary weighted fail, NOT a gate-bypassing force-fail; cumulative across retry attempts of the same turn). A trigger whose polarity the instance can't decide (per the abstract, checker-independent `live_decidable_polarities()`, a pure function of the criterion's own fields, paired with the checker's `live_verdict` override by lint rule CE025, a registry-based whole-tree check) is **inert by design** — one dataset-fanned YAML line serves both positive rows (pass/timeout live) and distractor rows (fail live). Verdicts **latch**: once a criterion decides, its `live_verdict` is never polled again. Stop rule is weighted, not strict-boolean: `run_limits.stop_early_gate_threshold` (default `1.0`, reproducing strict-AND behavior exactly) is the minimum weighted score (`Σ weight·score / Σ weight` over the armed subset) required to pass; a fail-stop fires once the armed set's **ceiling** (best case for everything still undecided) can no longer reach the threshold — so a low-weight fail or timeout that can't doom the gate is absorbed and the run continues — and is **deferred while any pass-capable armed criterion is undecided** (a distractor misfire never truncates a positive row's recall signal); a pass-stop fires once the `on_pass: stop` subset's **floor** (worst case) already meets the threshold, and is symmetrically **deferred while any pass-capable armed criterion outside the `on_pass: stop` subset is undecided** (so an early pass never freezes a sibling `on_pass: continue` criterion's signal out of the trajectory). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a kill-switched (`stop_early: false`) run. Driven by `orchestration/early_stop.py::EarlyStopWatcher` (built when `early_stop_active(task)`: ≥1 armed criterion, kill switch not thrown) through the agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. Gating is **FIRED-ONLY**: a run the watcher actually cut gates on the **armed subset** via the weighted `EvaluationResult.armed_criteria_passed`; a run that completes naturally — armed or not — gates strict-AND via `all_criteria_passed`, so adding a block never changes the verdict of a run it didn't cut. Note the gate keys on the watcher having FIRED (`result.early_stop is not None`), not on confirmed truncation — an agent that ignores `should_stop`, or a stop firing on the final message, still gates armed-only. Every resolution-time guardrail violation is a hard error at resolution (plan *and* run); the one load-time case — a `stop_early:` block on a non-live criterion — is a pydantic schema error at task load, which the run surface reports as a skipped task like any other malformed task. A runtime verdict bug **fails open** to a full run. Surfaces: `EarlyStopInfo` (incl. `gate_threshold` at stop time), report notes/badges, `stopped_early` run.json rows, `EarlyStopped`/`EarlyStopReason` telemetry dims. Worked rationale: docs/TASK_DEFINITION_GUIDE.md § `stop_early`. No blocks anywhere ⇒ behavior byte-for-byte unchanged. diff --git a/README.md b/README.md index 1273b57c..94b65827 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ **evaluating and benchmarking AI coding agents and their skills** — built for CLI and skill builders — with sandboxing, reproducibility, and data-driven analysis. It runs a real agent (**Claude Code**, **Codex**, **Google Antigravity / -Gemini**, or **OpenCode**) in a sandbox against declarative YAML tasks, then scores the files and +Gemini**, **OpenCode**, or **UiPath Delegate**) in a sandbox against declarative YAML tasks, then scores the files and commands it actually produced. Not an "agentic coding" benchmark: it measures how effective your CLI and skills are when used by coding agents. @@ -33,7 +33,7 @@ telemetry. See [How it compares](https://coder-eval.com/docs/comparison). - **Sandboxed execution** in isolated environments with resource limits - **Weighted, continuous scoring** (0.0–1.0) with fractional credit and thresholds - **Many criterion types** — from file checks to code similarity and LLM-graded rubrics -- **Agent abstraction** — Claude Code, Codex, Antigravity (Gemini), and OpenCode today, extensible via a plugin SPI +- **Agent abstraction** — Claude Code, Codex, Antigravity (Gemini), OpenCode, and UiPath Delegate today, extensible via a plugin SPI - **Experiment layer** — A/B agent configs (models, tools, prompts) side-by-side - **Full telemetry** — every tool call, token counts, and cost, with real-time streaming @@ -216,6 +216,7 @@ The step's exit code is coder-eval's own: non-zero on any failed task. | [Codex](docs/agents/CODEX.md) | Running the OpenAI Codex agent | | [Antigravity (Gemini)](docs/agents/ANTIGRAVITY.md) | Running the Google Antigravity / Gemini agent | | [OpenCode](docs/agents/OPENCODE.md) | Running the OpenCode agent on open-weight models | +| [UiPath Delegate SDK](docs/agents/DELEGATE_SDK.md) | Running UiPath Autopilot's Delegate agent via the @uipath/delegate-stdio host | | [Run-Limit Parity](docs/agents/HARNESS_PARITY.md) | What each run_limits field means on every harness | | [A/B Experiments](docs/AB_EXPERIMENTS.md) | Compare models / tools / prompts across the same tasks | | [Bring Your Own Dataset](docs/DATASETS.md) | Fan a single task out over a dataset | diff --git a/docker/Dockerfile b/docker/Dockerfile index d9cdd586..94a2930d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -64,6 +64,10 @@ COPY experiments/default.yaml ./experiments/default.yaml # `--driver docker` does not support it. Adding it means a pinned version that # travels with the release tag (as CLAUDE_CODE_VERSION does) plus an # env_passthrough block; see docs/agents/OPENCODE.md "Running in Docker". +# The same holds for `delegate-sdk`: its `@uipath/delegate-stdio` Node host is +# not baked in, and its UiPath auth env is not in the default allowlist -- a +# docker run needs an overlay image + `-D sandbox.docker.env_passthrough_extra`; +# see docs/agents/DELEGATE_SDK.md "Running in Docker". # # CODER_EVAL_UV_EXTRAS carries ADDITIONAL opt-in extras on top of those; it # defaults to none. `make docker-image-full` passes `--extra uipath`, which diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md index 3befae19..50081e58 100644 --- a/docs/EXTENDING.md +++ b/docs/EXTENDING.md @@ -285,8 +285,9 @@ The base package ships **no** plugin rates; only the built-in table. ## See also - [Claude Code](agents/CLAUDE_CODE.md) · [Codex](agents/CODEX.md) · - [Antigravity](agents/ANTIGRAVITY.md) · [OpenCode](agents/OPENCODE.md) — the - built-in agents, each registered via this same SPI + [Antigravity](agents/ANTIGRAVITY.md) · [OpenCode](agents/OPENCODE.md) · + [UiPath Delegate SDK](agents/DELEGATE_SDK.md) — the built-in agents, each + registered via this same SPI - [Task Definition Guide](TASK_DEFINITION_GUIDE.md) — the criterion catalogue - [CLAUDE.md](https://github.com/UiPath/coder_eval/blob/main/CLAUDE.md) — architecture and extension points in depth diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 1676690d..d4c72b2e 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -39,7 +39,7 @@ coder-eval run tasks/hello_date.yaml --stream full # live LLM output | `-D path=value` / `--set` | Override any resolved task-config field (`agent`/`run_limits`/`sandbox` roots), e.g. `-D run_limits.max_turns=30 -D agent.permission_mode=plan -D agent.sdk_options.effort=high`. Repeatable; schema-validated. This is the way to set permission mode, turn/timeout limits, token/USD budget caps, tools, plugins, and SDK options. | | `--model, -m` | Shorthand alias for `-D agent.model=…` (e.g., `claude-sonnet-5`) | | `--driver` | Shorthand alias for `-D sandbox.driver=…` (`tempdir` or `docker`) | -| `--type, -T` | Override agent type for all tasks (`claude-code`, `codex`, `antigravity`, `opencode`, or a plugin kind). | +| `--type, -T` | Override agent type for all tasks (`claude-code`, `codex`, `antigravity`, `opencode`, `delegate-sdk`, or a plugin kind). | | `--repeats` | Run each `(task, variant)` N times (≥1); overrides experiment/variant `repeats:`. See [Replicates](#replicates). | | `--resume` | Resume an interrupted run: skip tasks already finalized in `--run-dir` and run the rest, folding prior results into `run.json`. Requires `--run-dir`. A task with *any* final status (incl. FAILED/ERROR) counts as finalized, so resume does **not** retry failures — delete a task's `task.json` to force a re-run. A config mismatch is warned, not refused. | | `--sample N` | For dataset-backed tasks, run a fixed-seed random N-row sample (reproducible; cheap smoke test). See [Bring Your Own Dataset](DATASETS.md). | diff --git a/docs/agents/DELEGATE_SDK.md b/docs/agents/DELEGATE_SDK.md new file mode 100644 index 00000000..e0a466dd --- /dev/null +++ b/docs/agents/DELEGATE_SDK.md @@ -0,0 +1,347 @@ +--- +description: >- + Run UiPath Autopilot's Delegate agent as the agent under evaluation in Coder + Eval — installing the @uipath/delegate-stdio host from npm, UiPath + authentication and token refresh, model selection, and how the host's stdio + event stream maps to sandboxed, weighted scoring. +--- + +# Running the UiPath Delegate SDK agent in Coder Eval + +## Overview + +The `delegate-sdk` agent evaluates **UiPath Autopilot's Delegate agent**. Coder Eval +is Python and the Delegate SDK is TypeScript, so `DelegateSdkAgent` drives it through +the [`@uipath/delegate-stdio`](https://www.npmjs.com/package/@uipath/delegate-stdio) +npm package — the Delegate agent exposed as a Node subprocess speaking +newline-delimited JSON over stdin/stdout (called "the host" below). The package is +**self-contained**: `@uipath/delegate-sdk` and its platform runtime are its npm +dependencies and install transitively, so one `npm install` is the complete install. +You install that package, point Coder Eval at it, supply UiPath auth, and run tasks as +usual. + +One thing to keep in mind: the Delegate agent's *reasoning* runs in the UiPath +backend, but its **tools execute locally** — the host `chdir`s into the task's sandbox +working directory at init, so file writes and commands land in the sandbox with no +"reconcile back" step. Ordinary file-based criteria (`file_exists`, `run_command`, +`pytest`, …) therefore work unchanged; `tasks/delegate_sdk_smoke_test.yaml` shows +this end to end. + +## Setup + +### 1. Install Node and the `@uipath/delegate-stdio` host + +The host is a **Node** package on the public npm registry, not a Python package: + +```bash +npm install @uipath/delegate-stdio +``` + +This drops the bundle at `node_modules/@uipath/delegate-stdio/dist/delegate_stdio.mjs` +and pulls in `@uipath/delegate-sdk` plus the `@uipath/delegate-runtime-*` interop +binaries for your platform (Linux x64, Windows x64, macOS arm64). There is **no** +separate SDK to set up. + +The `coder-eval[delegate-sdk]` extra exists for symmetry with the other harnesses and +carries **no Python dependencies** — the agent shells out to the bundle above: + +```bash +uv sync --extra delegate-sdk # documents the opt-in; installs no extra packages +``` + +If the bundle cannot be found, the task fails at `start()` with a non-retryable +`AgentConfigError` listing every path it probed and the install command, rather than +failing obscurely mid-run. + +> **npm's walk-up gotcha.** `npm install ` run in a directory with no +> `package.json` silently installs into the nearest *ancestor* that has one — often +> `~/node_modules`. Coder Eval's resolver walks the cwd's ancestors and `~` for exactly +> this reason, so the install is still found; `npm ls @uipath/delegate-stdio` shows +> where it landed. + +### 2. Point Coder Eval at the host + +**Usually nothing to set.** When neither variable below is set, the bundle is +auto-located by walking up from the current directory through its ancestors and `~`, +the way Node resolves modules. To override the search, set **one** of these: + +| Variable | Meaning | +|---|---| +| `DELEGATE_STDIO_NODE_MODULES` | The install root that holds `node_modules/@uipath/...` (probed exactly, no walk-up). | +| `DELEGATE_STDIO_PATH` | Absolute path straight to `dist/delegate_stdio.mjs`. The reliable seam for CI and container images, where the bundle lives outside any cwd ancestor. | + +### 3. Choose the cloud environment + +| Variable | Default | Purpose | +|---|---|---| +| `DELEGATE_SDK_ENV` | `alpha` | Cloud env slug (`alpha` / `staging` / `production`). The host composes the backend URL from your auth's org/tenant slugs plus this value, so there are normally no backend or interop URLs to configure. | +| `BACKEND_URL` / `INTEROP_URL` | unset | Advanced/local override only: pin the Delegate backend / interop endpoints directly (e.g. a localhost backend). Host precedence: `backendUrl` > `BACKEND_URL` > env slug. With `INTEROP_URL` set the SDK runs connect-only against an externally managed interop instead of spawning its own. | + +### 4. Authenticate + +Provide UiPath credentials one of two ways. All of them are consumed by the **host +process**, never read by Coder Eval itself: + +- **Environment token** — `AUTH_TOKEN`, `TENANT_ID`, `ORG_ID` (plus `USER_ID` for + service-to-service tokens, and `ORG_LOGICAL_NAME` / `TENANT_NAME` when the env slug + has to compose the backend URL from the org/tenant *slugs*). +- **Saved login** — a prior `npx @uipath/delegate-cli login --env ` (interactive + browser OAuth) that wrote `~/.aria/sdk-auth.json`. Saved tokens are short-lived + (about an hour) but refresh while their refresh token is valid. + +Your org/tenant must have autopilot-everywhere provisioned on the chosen environment. + +An auth failure at startup is **non-retryable**: the agent fails fast with an +`AgentConfigError` (instead of burning the API-error retry budget) and says whether the +saved login is *expired* (and how long ago) or *absent*, plus the exact login command. + +#### Staying authenticated past the token TTL + +An `AUTH_TOKEN` exported into the environment is frozen at that value, so a run longer +than its TTL starts failing with `401 Invalid token: Signature has expired`. Two +mechanisms cover that, both by keeping a **token file** fresh — the host re-reads it at +init, on every turn, and on its own refresh timer, and applies a newer token live: + +- **External refresher** — point `DELEGATE_AUTH_TOKEN_FILE` (or `AUTH_TOKEN_FILE`) at + a file some other process keeps current. It accepts a `PATH`-style list and takes the + first readable entry, so one value can name both a host path and the same file's + bind-mount path inside a container. This always wins. +- **Adapter-side S2S refresher** — when no token file is configured *and* the + `AUTH_TOKEN` was itself minted from the `LLMGW_CLIENT_ID` / `LLMGW_CLIENT_SECRET` / + `LLMGW_URL` client-credentials triple present in Coder Eval's environment, the agent + re-mints that token before each expiry and publishes it through a token file of its + own. The `LLMGW_*` secret is **stripped from the host environment** (the agent's own + shell tools inherit that environment, i.e. the code under test), so only the token + file — not the secret — is reachable from inside the sandbox. It activates only when + the inherited token's `client_id` claim matches, so it never forces a re-minted + service token onto a run that authenticated another way. Look for + `S2S token-file refresher active` (or the `client_id mismatch` / `already configured` + decline lines) in `task.log` to see which path a run took. + +## Usage + +### Command line + +```bash +uv run coder-eval run tasks/delegate_sdk_smoke_test.yaml +uv run coder-eval run tasks/hello_date.yaml -D agent.type=delegate-sdk -D agent.model=virtuoso-1-5 +``` + +### Task definition (YAML) + +```yaml +agent: + type: "delegate-sdk" + # Pin a model your tenant + env actually serves (see "Model selection"). + model: "virtuoso-1-5" + permission_mode: "acceptEdits" + sdk_options: + effort: "high" # optional: low | medium | high | xhigh + project_id: "project" # optional: route the local wiki under /projects//wiki + plugins: + - type: local + path: "$SKILLS_PLUGIN_PATH" + +success_criteria: + - type: file_exists + path: "string_utils.py" + description: "Solution file must exist" +``` + +### Model selection + +The Delegate backend serves a model list that is **specific to your tenant and +environment** — UiPath-native models (`virtuoso-1-5`, `gemini-3-5-flash`, …) and +gateway-routed ones (`gpt-5-6-terra`, `kimi-k2-7-code`, …). List exactly what yours +offers: + +```bash +npx @uipath/delegate-cli models --env alpha +``` + +Give Coder Eval the **hyphenated** id (`virtuoso-1-5`); the host converts it to the +backend's underscored form. Do not rely on the framework's inherited default +(`claude-sonnet-4-6` from `experiments/default.yaml`): the deployed alpha backend +rejects it (`Model 'claude_sonnet_4_6' is not available`), so an unpinned delegate run +fails. Pin a served model in the task or with `--model`. + +### `sdk_options.effort` — reasoning effort + +`sdk_options.effort` (`low` / `medium` / `high` / `xhigh`) sets the model's +reasoning-effort tier; the host forwards it as `user_config.effort` on every chat +request. Any other `sdk_options` keys are accepted and silently ignored, so one +experiment YAML can carry Claude-only options and still drive a `delegate-sdk` +variant. The layer-5 `-D agent.sdk_options.effort=` override works too: the +guard that restricts `sdk_options` overrides is registry-driven and admits every +agent whose config declares the field. + +### `project_id` / `session_id` — wiki routing + +Both are client-side routing keys forwarded to the SDK. `project_id` binds sessions to +a project so the agent's local wiki lands at `/projects//wiki` +instead of the per-session `/sessions//wiki`; `session_id` pins +the session id so that per-session directory is deterministic (a pinned id skips +`createSession`, so it must be one the backend accepts). Empty (the default) keeps the +SDK's own behavior. + +### `plugins` — skills + +A `plugins:` entry with a `path` is mounted as the agent's skills directory: the host +receives `/skills` as `bundledSkillsPath` (the Claude-plugin layout — +`/skills//SKILL.md`). Environment variables in the path (`$VAR`, +`${VAR}`) are expanded. **Only the first plugin is honored**; additional entries are +warned about and ignored. Point at the plugin *root*, never at the skills directory +itself — see [Run-Limit Parity](HARNESS_PARITY.md) for why the wrong depth fails +silently on this harness. + +## Telemetry + +The host streams one JSON object per line; `DelegateSdkAgent` reduces it into the +standardized event protocol and lets `EventCollector` build the `TurnRecord`: + +| Host message | Becomes | +|---|---| +| `event.thinking` / `event.message` | `TextChunkEvent`; a `message` tagged `isStepStart` opens a new generation | +| `event.tool_call` | `ToolStartEvent` + a `CommandTelemetry` (`toolName`, `toolArgs`, `toolId`) | +| `event.tool_result` | `ToolEndEvent` (`toolStatus: failed` → `error`); the untruncated `toolResult` is the `result_summary` | +| `result` | `TurnEndEvent` + `AgentEndEvent` with `response`, `assistantStepCount`, `usage`, `turnUsages`, `model`, `maxStepsReached` | +| `error` | `AgentCrashError` with the partial turn preserved on `pending_turn` | + +**Transcript.** `TurnRecord.messages` carries one `AssistantMessage` per backend +round-trip, reconstructed from stream order (a `tool_result` closes a round-trip; the +next generation activity opens the next one). Per-generation token buckets are zipped +from the `result` message's `turnUsages` list when it lines up 1:1 with the +reconstructed generations; otherwise the messages carry content and timing only and +the collector's reconciliation entry carries the turn total — the +transcript-sums-to-total invariant holds either way. + +**Tokens and cost.** `usage.input_tokens` from the host is the fresh (uncached) prompt +slice and maps onto `uncached_input_tokens`; cache reads/writes arrive separately. The +Delegate SDK exposes no pricing, so `total_cost_usd` is computed locally via +`calculate_cost` on the **hyphenated** model id the backend reports (normalized from +its underscored form) — the Delegate-routed rows in `src/coder_eval/pricing.py` +(`virtuoso-*`, `gemini-3-5-flash`, `gpt-5-6-*`, `kimi-k2-7-code`, …) are keyed that +way. A model your tenant serves that has no row reports no cost until one is added. + +**`max_turns`** is forwarded to the host as `maxSteps`; `maxStepsReached` on the +result becomes `max_turns_exhausted` and the run finalizes cleanly. + +**Environment info.** Every task records `delegate_env`, `delegate_model`, and — when +set — the *host* of `BACKEND_URL` / `INTEROP_URL` (never the full URL) so runs against +different environments stay distinguishable and no embedded credential leaks into the +run record. + +## Failure handling + +Some backend failures need a specific retry shape, so the agent classifies them before +the generic categorizer sees the message: + +- **First-response stall** (`DELEGATE_STALL_TIMEOUT_S`, opt-in). A turn that receives + *no* agent activity within this many seconds of the prompt is treated as a transient + backend stall: the wedged host is force-killed, respawned, and the prompt resent + once, inside the same turn. Only the wait for the first activity is guarded, so a + long-running tool call never trips it. Unset (the default) disables it — callers + pointing at a dedicated, healthy backend are unaffected; set it to a value *above* + the host's own SSE watchdog stack (a shared backend typically wants ~240s). +- **Backend session conflict** (`A reply is already being generated for this + conversation`). Resending into the same conversation can only re-conflict, so the + host is killed and the orchestrator's `AGENT_CRASH` retry gets a **fresh host**. +- **Cloudflare WAF content block** (the generic "not available in your country" 403 + page). Deterministic per payload — shell-like text in a prompt or tool result tripped + a managed rule — so the reason is rewritten to route the failure to the non-retryable + `AGENT_INVALID_OUTPUT` category instead of burning identical retries. +- **SSE connect watchdog** (`sse connect timeout`). The backend front door never sent + response headers; a transient availability window, so the reason is rewritten to the + *retryable* `AGENT_API_ERROR` signature and the live host is kept for the resend. +- **Host exit / oversized frame.** A host that dies mid-turn (or emits a single line + over the 64 MiB stream limit) crashes the turn with the last 20 stderr lines in the + message; retries do not respawn a dead host except in the session-conflict case. + +## Tracing the host + +`DELEGATE_STDIO_VERBOSE=1` makes the host trace its stdio protocol (every frame, +every agent event, its auth/init/token-refresh steps) to stderr, which the agent +forwards into the run log. `1`/`true` forces it on, `0`/`false` forces it off, and +unset follows Coder Eval's own `--verbose`. Set it when a turn looks hung: without it +the host is silent for the whole backend round-trip, so a long turn is +indistinguishable from a wedged one. `DELEGATE_STDIO_LOG_MAX_CHARS` (host-side, +default 50 000) caps each traced value. + +## Running in Docker + +**Not supported out of the box — use the default `tempdir` driver**, or build an +overlay image. Two things are missing from the stock `coder-eval-agent` image, both +deliberate: + +- **The host is not in the image.** Baking `@uipath/delegate-stdio` in means a pinned + version that travels with the release tag (as `CLAUDE_CODE_VERSION` does) — a + release-process decision. An overlay image can `npm install` it into a fixed + directory and set `DELEGATE_STDIO_PATH` to the bundle (the resolver never probes a + global npm prefix, so a fixed path is the reliable seam). +- **No UiPath auth reaches it.** The docker driver forwards host environment variables + through an explicit allowlist (`SandboxConfig.env_passthrough`) with no Delegate + block. Forward what the host needs with a layer-5 override, e.g.: + +```bash +uv run coder-eval run tasks/my_task.yaml --driver docker -D agent.type=delegate-sdk \ + -D 'sandbox.docker.env_passthrough_extra=[AUTH_TOKEN, DELEGATE_AUTH_TOKEN_FILE, TENANT_ID, ORG_ID, USER_ID, ORG_LOGICAL_NAME, TENANT_NAME, DELEGATE_SDK_ENV, DELEGATE_STALL_TIMEOUT_S, DELEGATE_STDIO_VERBOSE]' +``` + +Do **not** forward `DELEGATE_STDIO_PATH` (the image has its own copy) or the `LLMGW_*` +secret (the agent's shell tools have no use for it). + +Under any driver, running several tasks in parallel on one machine wants **one shared +interop** (`INTEROP_URL`): without it each task's SDK attach-or-spawns an interop via +`~/.delegate-sdk/interop.pid` and the task that spawned it kills it on finish while +its siblings are still attached, turning every later tool call into `ECONNREFUSED`. + +## Known limitations + +- **`allowed_tools` / `disallowed_tools` / `system_prompt` / `system_prompt_file` / + `setting_sources` have no Delegate SDK equivalent.** They are ignored, with a + one-time warning at `start()`. `permission_mode` is equally unsupported but ignored + silently — it always carries a value, so warning on it would be noise on every run. + The run marker `system_prompt_semantics` is therefore `"unknown"`. +- **One plugin only.** The first `plugins:` entry becomes `bundledSkillsPath`; the rest + are dropped with a warning. A plugin's agents, hooks, commands and MCP servers have + no Delegate equivalent. +- **No cooperative early stop.** The host drives its own inner loop with no + between-message poll point, so `run_limits.stop_early` is rejected for this harness. +- **`max_turns` counts Delegate steps**, and the SDK stops a limit-hit tool call + *before* emitting its `tool_call` event — under `max_turns: 1` a correct first-response + skill engagement never reaches `TurnRecord.commands`, so `skill_triggered` routing rows + are structurally unmeasurable here. See [Run-Limit Parity](HARNESS_PARITY.md). +- **Sandbox mock CLIs need a recent host.** `SandboxConfig.mock_path_dirs` is forwarded + as the `shellPathPrepend` init option, which the SDK injects into every shell command + it runs inside the interop; a host whose bundled SDK predates the option ignores it and + mock shadowing is lost. +- **No sub-agent attribution.** The host's stream carries no nested-agent boundaries. + +## Troubleshooting + +| Symptom | Fix | +|---|---| +| `AgentConfigError: delegate-stdio bundle not found …` | `npm install @uipath/delegate-stdio` (step 1), or set `DELEGATE_STDIO_NODE_MODULES` / `DELEGATE_STDIO_PATH` (step 2). | +| `node not on PATH` / spawn fails | Install Node.js and make sure `node` runs in your shell. | +| `Model '' is not available` | The pinned/inherited model is not in *your* tenant + environment's list. Run `npx @uipath/delegate-cli models --env ` and pick one it serves. | +| `AgentConfigError: Delegate SDK authentication failed during init …` | Auth missing/expired (non-retryable, fails immediately). The message says whether the saved login is expired or absent — run `npx @uipath/delegate-cli login --env `, or set `AUTH_TOKEN`/`TENANT_ID`/`ORG_ID`. Confirm the org/tenant has autopilot-everywhere provisioned on `DELEGATE_SDK_ENV`. | +| `401 Invalid token: Signature has expired` mid-run | The env `AUTH_TOKEN` outlived its TTL. Point `DELEGATE_AUTH_TOKEN_FILE` at a file you keep fresh, or let the adapter-side S2S refresher take over (see "Staying authenticated"). | +| Every tool call fails with `ECONNREFUSED` after one task finishes | Parallel tasks are sharing an auto-spawned interop. Start one interop yourself and export `INTEROP_URL`. | +| A turn looks hung | Re-run with `DELEGATE_STDIO_VERBOSE=1` (or `--verbose`) to see the host's own trace; consider `DELEGATE_STALL_TIMEOUT_S` for a shared backend. | + +## Bundled example and tests + +```bash +uv run coder-eval run tasks/delegate_sdk_smoke_test.yaml # needs the host + auth +uv run pytest tests/test_delegate_sdk_agent.py # offline: replays the stdio protocol in memory +uv run pytest -m live tests/test_delegate_sdk_agent_live.py # drives the real host; skips without prerequisites +``` + +## References + +- Host package: [`@uipath/delegate-stdio`](https://www.npmjs.com/package/@uipath/delegate-stdio) · + SDK: [`@uipath/delegate-sdk`](https://www.npmjs.com/package/@uipath/delegate-sdk) · + CLI: [`@uipath/delegate-cli`](https://www.npmjs.com/package/@uipath/delegate-cli) +- [Run-Limit Parity](HARNESS_PARITY.md) — what `run_limits` means on this harness +- [Extending Coder Eval](../EXTENDING.md) — the agent plugin SPI +- [Task Definition Guide](../TASK_DEFINITION_GUIDE.md) diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 8053c47b..1a47f10e 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -10,12 +10,12 @@ This page is the contract for what each run limit means per harness, plus the sh ## The table -| Limit | claude-code | codex | antigravity | opencode | -|---|---|---|---|---| -| `run_limits.max_turns` | native SDK cap (agent-loop turns) | visible-turn cap (resolved tool calls) | visible-turn cap (resolved tool calls) | native step cap (the CLI's own agent-loop steps) | -| `run_limits.turn_timeout` | watchdog, SIGKILL on the CLI subprocess | watchdog + cooperative interrupt | watchdog, plus an earlier internal poll deadline at 80% of it (see below) | deadline enforced in-loop and on the final reap; SIGTERM→SIGKILL on the CLI's whole process group | -| `run_limits.task_timeout` | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | -| `run_limits.stop_early` | cooperative `should_stop` | cooperative `should_stop` | cooperative `should_stop` | cooperative `should_stop` (event granularity) | +| Limit | claude-code | codex | antigravity | opencode | delegate-sdk | +|---|---|---|---|---|---| +| `run_limits.max_turns` | native SDK cap (agent-loop turns) | visible-turn cap (resolved tool calls) | visible-turn cap (resolved tool calls) | native step cap (the CLI's own agent-loop steps) | native step cap (forwarded to the host as `maxSteps`; see below) | +| `run_limits.turn_timeout` | watchdog, SIGKILL on the CLI subprocess | watchdog + cooperative interrupt | watchdog, plus an earlier internal poll deadline at 80% of it (see below) | deadline enforced in-loop and on the final reap; SIGTERM→SIGKILL on the CLI's whole process group | deadline enforced on every host read; SIGKILL on the Node host, partial turn preserved | +| `run_limits.task_timeout` | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | +| `run_limits.stop_early` | cooperative `should_stop` | cooperative `should_stop` | cooperative `should_stop` | cooperative `should_stop` (event granularity) | **not supported** — the host runs its own inner loop with no between-message poll point (`supports_cooperative_stop` is False, so arming it is rejected at resolution) | ## `max_turns` counts visible turns on Codex and Antigravity @@ -50,6 +50,17 @@ when step N+1 begins, with the completed steps' tokens intact. A step is one assistant generation and may carry several tool calls — so, as with claude-code, the same number is a looser tool-call budget than on the visible-turn backends. +**delegate-sdk also keeps a native unit — the Delegate SDK's own steps.** The cap +is forwarded to the `@uipath/delegate-stdio` host as `maxSteps`, and the host +reports back `maxStepsReached` on the terminal `result` message, which becomes +`max_turns_exhausted`. A step is one backend round-trip (one LLM generation), so, +as with claude-code and OpenCode, `max_turns: N` is a looser tool-call budget than on +the visible-turn backends. One consequence worth knowing before writing an +activation suite: the SDK stops a limit-hit tool call **before** it emits the +`tool_call` event, so under `max_turns: 1` a first-response skill engagement never +reaches `TurnRecord.commands` and `skill_triggered` reads false even when the +model did the right thing — do not run `max_turns: 1` routing rows on this harness. + **So holding `max_turns` constant across harnesses does not hold the budget constant.** If you are A/B-ing across backends and the cap is close to binding, that is the number to distrust. @@ -116,10 +127,10 @@ whose cap fires should not look like a task whose harness hung. Not a run limit, but the same promise: one task file, three harnesses, same meaning. This field breaks it silently. -| | claude-code | codex | antigravity | -|---|---|---|---| -| `/skills//SKILL.md` (plugin root) | **required** | accepted | accepted | -| `//SKILL.md` (bare skills dir) | **loads nothing** | accepted | accepted | +| | claude-code | codex | antigravity | delegate-sdk | +|---|---|---|---|---| +| `/skills//SKILL.md` (plugin root) | **required** | accepted | accepted | **required** | +| `//SKILL.md` (bare skills dir) | **loads nothing** | accepted | accepted | **loads nothing** | claude-code hands the value to the SDK as a *plugin directory*, and a plugin's skills live at `/skills//SKILL.md`. Point it at the directory that directly @@ -127,7 +138,12 @@ parents the skill directories and no skill loads. Codex (`codex_agent._setup_skills`) and Antigravity (`antigravity_agent._resolve_skills_paths`) both scan **both** layouts and take whichever actually holds a `/SKILL.md`. -So `.claude/skills` works on two backends out of three and fails on the third — and +delegate-sdk behaves like claude-code here: it forwards `/skills` to the +`@uipath/delegate-stdio` host as `bundledSkillsPath` (only the first `plugins:` entry; +a second one is warned about and ignored), so the same bare skills directory loads +nothing there too. + +So `.claude/skills` works on two backends out of four and fails on the other two — and fails without an error. The agent simply is not offered the skill, every positive row of an activation suite scores 0, and the suite reports recall 0.0. That is indistinguishable from a skill that never triggers, which is the finding such a suite @@ -147,7 +163,7 @@ claude --plugin-dir /path/to/root/skills # lists nothing <- loaded nothi A bare `` with no prefix is project discovery, not your plugin. -**Write the plugin root.** It is correct on all three, so there is never a reason to +**Write the plugin root.** It is correct on every harness, so there is never a reason to write the deeper form. For `.claude/skills/my-skill/SKILL.md` that is `.claude`. Note what else that pulls in: a plugin root loads the **whole** plugin, so an @@ -170,10 +186,10 @@ so the same requirement applies there and is unlinted. `tasks/run_limits/` holds one fixture per limit: `max_turns_cap.yaml` asks for more sequential work than its cap allows, and `turn_timeout.yaml` runs a command that outlives its watchdog. Run either with `--type claude-code` / `--type codex` / -`--type antigravity` / `--type opencode` to check a backend against the contract -above. +`--type antigravity` / `--type opencode` / `--type delegate-sdk` to check a backend +against the contract above. ## Related -- [Claude Code](CLAUDE_CODE.md) · [Codex](CODEX.md) · [Antigravity](ANTIGRAVITY.md) · [OpenCode](OPENCODE.md) +- [Claude Code](CLAUDE_CODE.md) · [Codex](CODEX.md) · [Antigravity](ANTIGRAVITY.md) · [OpenCode](OPENCODE.md) · [UiPath Delegate SDK](DELEGATE_SDK.md) - [Task Definition Guide](../TASK_DEFINITION_GUIDE.md) — the full `run_limits` schema diff --git a/docs/index.md b/docs/index.md index 5f22ce7e..807dae8c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,7 +4,7 @@ description: >- Coder Eval is an open-source framework to evaluate, benchmark, and A/B-test AI coding agents and Claude Code skills in a sandbox — declarative YAML tasks, weighted scoring, cost/token telemetry, and CI gates for Claude Code, Codex, - Gemini, and OpenCode. + Gemini, OpenCode, and UiPath Delegate. --- # Evaluate AI coding agents & Claude Code skills — Coder Eval @@ -14,7 +14,7 @@ description: >- builders — with sandboxing, reproducibility, and data-driven analysis. It is not an "agentic coding" benchmark: it measures how effective *your* CLI and skills are when used by coding agents such as **Claude Code**, **Codex**, **Google -Antigravity (Gemini)**, and **OpenCode**. +Antigravity (Gemini)**, **OpenCode**, and **UiPath Delegate**. If you have ever asked *"how do I test whether my Claude Code skill actually triggers?"*, *"how do I benchmark Claude Code vs. Codex on my own tasks?"*, or @@ -30,7 +30,7 @@ triggers?"*, *"how do I benchmark Claude Code vs. Codex on my own tasks?"*, or - **Sandboxed execution** in isolated environments with resource limits - **Weighted, continuous scoring** (0.0–1.0) with fractional credit and thresholds - **Many criterion types** — from file checks to code similarity and LLM-graded rubrics -- **Agent abstraction** — Claude Code, Codex, Antigravity (Gemini), and OpenCode today, extensible via a plugin SPI +- **Agent abstraction** — Claude Code, Codex, Antigravity (Gemini), OpenCode, and UiPath Delegate today, extensible via a plugin SPI - **Experiment layer** — A/B agent configs (models, tools, prompts) side-by-side - **Full telemetry** — every tool call, token counts, and cost, with real-time streaming @@ -82,6 +82,7 @@ New here? Start with **[Tutorial 01 — Your First Evaluation](tutorials/01-firs | [Codex](agents/CODEX.md) | Running the OpenAI Codex agent | | [Antigravity (Gemini)](agents/ANTIGRAVITY.md) | Running the Google Antigravity / Gemini agent | | [OpenCode](agents/OPENCODE.md) | Running the OpenCode agent on open-weight models | +| [UiPath Delegate SDK](agents/DELEGATE_SDK.md) | Running UiPath Autopilot's Delegate agent via the @uipath/delegate-stdio host | | [Run-Limit Parity](agents/HARNESS_PARITY.md) | What each run_limits field means on every harness | | [A/B Experiments](AB_EXPERIMENTS.md) | Compare models / tools / prompts across the same tasks | | [Bring Your Own Dataset](DATASETS.md) | Fan a single task out over a dataset | diff --git a/docs/llms.txt b/docs/llms.txt index 9dc3865f..c7630a3b 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -4,7 +4,7 @@ > benchmarking, and A/B-testing AI coding agents and their Claude Code skills in a > sandbox. It uses declarative YAML tasks with weighted, continuous scoring > (0.0–1.0), runs real agents (Claude Code, Codex, Google Antigravity/Gemini, -> OpenCode) with +> OpenCode, UiPath Delegate) with > full tool use, captures per-tool cost/token telemetry, and provides CI-ready > pass/fail gates. It is not a fixed benchmark or leaderboard — it scores your own > tasks, and can verify whether a Claude Code skill actually triggers. @@ -30,6 +30,7 @@ and A/B plumbing. - [Codex](https://coder-eval.com/docs/agents/codex): Running the OpenAI Codex agent - [Antigravity (Gemini)](https://coder-eval.com/docs/agents/antigravity): Running the Google Antigravity / Gemini agent - [OpenCode](https://coder-eval.com/docs/agents/opencode): Running the OpenCode agent on open-weight models +- [UiPath Delegate SDK](https://coder-eval.com/docs/agents/delegate-sdk): Running UiPath Autopilot's Delegate agent via the @uipath/delegate-stdio host - [Run-Limit Parity](https://coder-eval.com/docs/agents/harness-parity): What each run_limits field means on every harness - [A/B Experiments](https://coder-eval.com/docs/ab-experiments): Compare models / tools / prompts across the same tasks - [Bring Your Own Dataset](https://coder-eval.com/docs/datasets): Fan a single task out over a dataset diff --git a/evalboard/app/_components/harness-badge.tsx b/evalboard/app/_components/harness-badge.tsx index 02a7bbaa..ac1aa886 100644 --- a/evalboard/app/_components/harness-badge.tsx +++ b/evalboard/app/_components/harness-badge.tsx @@ -22,10 +22,10 @@ const HARNESS_LOGO: Record = { "deepseek.v3.2": p(0.74, 2.22, 0.74, 0), "zai.glm-5": p(1.2, 3.84, 1.2, 0), "moonshotai.kimi-k2.5": p(0.72, 3.6, 0.72, 0), + // UiPath Delegate SDK route (DelegateSdkAgent). The Delegate backend echoes + // UNDERSCORED ids (gpt_5_6_terra) which the agent normalizes to HYPHENATED + // keys (gpt-5-6-terra) — a different key space from the dotted OpenAI/Gemini + // rows above, so delegate runs need these rows or render "—" for cost. + // Mirror of pricing.py; the rates were carried over verbatim from the + // coder_eval_uipath plugin that used to register them, and several differ + // from the dotted rows for the same physical model (cache-write treatment) — + // see the pricing.py comment before "reconciling" either side. + "virtuoso-1-5": p(0.95, 4, 0, 0.16), + "virtuoso-2-0": p(0.95, 4, 0, 0.19), + "gemini-3-5-flash": p(1.5, 9, 0, 0.15), + "gemini-3-6-flash": p(1.5, 7.5, 0, 0.15), + "gemini-3-1-pro-preview": p(2, 12, 0, 0.2), + "gpt-5-4": p(2.5, 15, 15, 0.25), + "gpt-5-5": p(5, 30, 30, 0.5), + "gpt-5-6-sol": p(5, 30, 6.25, 0.5), + "gpt-5-6-terra": p(2, 12, 2.5, 0.2), + "gpt-5-6-luna": p(0.2, 1.2, 0.25, 0.02), + "kimi-k2-7-code": p(0.95, 4, 0, 0.19), }; function p( diff --git a/mkdocs.yml b/mkdocs.yml index db2d2728..a2fd7f55 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -84,6 +84,7 @@ extra: agents/CODEX.md: "Running the OpenAI Codex agent" agents/ANTIGRAVITY.md: "Running the Google Antigravity / Gemini agent" agents/OPENCODE.md: "Running the OpenCode agent on open-weight models" + agents/DELEGATE_SDK.md: "Running UiPath Autopilot's Delegate agent via the @uipath/delegate-stdio host" agents/HARNESS_PARITY.md: "What each run_limits field means on every harness" AB_EXPERIMENTS.md: "Compare models / tools / prompts across the same tasks" DATASETS.md: "Fan a single task out over a dataset" @@ -114,6 +115,7 @@ nav: - Codex: agents/CODEX.md - Antigravity (Gemini): agents/ANTIGRAVITY.md - OpenCode: agents/OPENCODE.md + - UiPath Delegate SDK: agents/DELEGATE_SDK.md - Run-Limit Parity: agents/HARNESS_PARITY.md - Advanced: - A/B Experiments: AB_EXPERIMENTS.md diff --git a/pyproject.toml b/pyproject.toml index 5eea61f9..bb08b119 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "coder-eval" version = "0.11.6" -description = "Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity) with sandboxed, reproducible YAML task suites." +description = "Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity, OpenCode, UiPath Delegate) with sandboxed, reproducible YAML task suites." readme = "README.md" license = "Apache-2.0" requires-python = ">=3.13" @@ -9,7 +9,8 @@ authors = [{ name = "UiPath", email = "coder-eval@uipath.com" }] keywords = [ "ai", "llm", "agent", "coding-agent", "evaluation", "eval", "evals", "benchmark", "swe-bench", "claude", "claude-code", "codex", "anthropic", - "gemini", "antigravity", "sandbox", "code-generation", "agent-evaluation", + "gemini", "antigravity", "opencode", "uipath", "delegate", + "sandbox", "code-generation", "agent-evaluation", "llm-evaluation", "llm-eval", "ai-evaluation", "skills-evaluation", "claude-skills", "claude-code-skills", "agent-skills", "skillsbench", "agent-testing", "llmops", @@ -145,6 +146,21 @@ antigravity = [ # fail at start() with a clear hint pointing back here. # See docs/agents/OPENCODE.md. opencode = [] +# Optional extra that enables UiPath Delegate SDK agent support. +# +# Deliberately EMPTY, for the same reason as `opencode` above. The Delegate +# agent is driven through the `@uipath/delegate-stdio` Node host +# (`npm install @uipath/delegate-stdio`, public npm registry), which +# DelegateSdkAgent spawns as a subprocess speaking JSON-Lines over stdio — it +# imports no third-party Python package. The extra keeps the opt-in surface +# uniform across harnesses and gives the prerequisite a single home in the +# packaging metadata. +# +# Without the host bundle on disk the framework still installs and runs; +# delegate-sdk tasks fail at start() with an AgentConfigError naming the +# install command and the DELEGATE_STDIO_PATH / DELEGATE_STDIO_NODE_MODULES seams. +# See docs/agents/DELEGATE_SDK.md. +delegate-sdk = [] [project.scripts] coder-eval = "coder_eval.cli:app" diff --git a/src/coder_eval/agents/__init__.py b/src/coder_eval/agents/__init__.py index 0bbf0dde..1d856b24 100644 --- a/src/coder_eval/agents/__init__.py +++ b/src/coder_eval/agents/__init__.py @@ -4,6 +4,7 @@ from coder_eval.agents.antigravity_agent import AntigravityAgent from coder_eval.agents.claude_code_agent import ClaudeCodeAgent from coder_eval.agents.codex_agent import CodexAgent +from coder_eval.agents.delegate_sdk_agent import DelegateSdkAgent from coder_eval.agents.noop_agent import NoOpAgent from coder_eval.agents.opencode_agent import OpenCodeAgent from coder_eval.agents.registry import AgentRegistry, create_agent @@ -11,7 +12,7 @@ def register_builtins(registry: type[AgentRegistry]) -> None: - """Register the built-in agents (Claude/Codex/Antigravity/OpenCode/NoOp) onto ``registry``. + """Register the built-in agents (Claude/Codex/Antigravity/OpenCode/Delegate/NoOp) onto ``registry``. This is the target of coder-eval's own ``coder_eval.plugins`` entry point, so the built-in agents travel the identical discovery path as any third-party @@ -21,7 +22,7 @@ def register_builtins(registry: type[AgentRegistry]) -> None: """ # Reference the imported classes so the registration side effect is explicit # and a future refactor that drops the top-level imports fails loudly here. - _ = (ClaudeCodeAgent, CodexAgent, AntigravityAgent, OpenCodeAgent, NoOpAgent) + _ = (ClaudeCodeAgent, CodexAgent, AntigravityAgent, OpenCodeAgent, DelegateSdkAgent, NoOpAgent) # Rot-protection: the decorators fire on import, but assert the built-ins are # actually registered so a future lazy-import refactor (which would leave the # import-cached modules' decorators un-run) fails loudly instead of silently @@ -31,6 +32,7 @@ def register_builtins(registry: type[AgentRegistry]) -> None: AgentKind.CODEX, AgentKind.ANTIGRAVITY, AgentKind.OPENCODE, + AgentKind.DELEGATE_SDK, AgentKind.NONE, ): if registry.get(kind) is None: @@ -42,6 +44,7 @@ def register_builtins(registry: type[AgentRegistry]) -> None: "AntigravityAgent", "ClaudeCodeAgent", "CodexAgent", + "DelegateSdkAgent", "NoOpAgent", "OpenCodeAgent", "create_agent", diff --git a/src/coder_eval/agents/_delegate_s2s_token_file.py b/src/coder_eval/agents/_delegate_s2s_token_file.py new file mode 100644 index 00000000..46a125d9 --- /dev/null +++ b/src/coder_eval/agents/_delegate_s2s_token_file.py @@ -0,0 +1,339 @@ +"""Keep a Delegate-host token file fresh by re-minting LLMGW S2S tokens. + +Closes the auth gap on runs that outlive the ~1h S2S token TTL: the adapter +strips the ``LLMGW_*`` client-credentials pair from the host env on purpose +(the host env is inherited by the shells its interop spawns — i.e. by the code +under test), which also disables the delegate-stdio host's own S2S +self-refresh. Without an external refresher writing ``DELEGATE_AUTH_TOKEN_FILE``, +the host keeps its start-up ``AUTH_TOKEN`` forever and every request past the +TTL dies with ``401 Invalid token: Signature has expired``. + +:class:`S2sTokenFileRefresher` is that external refresher, run inside the +adapter's own process: it holds the ``LLMGW_*`` pair privately (never in the +host env), re-mints a ``service.internal`` token before each expiry, and +publishes it through a token FILE the host already knows how to consume — the +host re-reads the file at init, at every turn, and on its own refresh timer. + +Guard rails: + +* It only activates when the run's inherited ``AUTH_TOKEN`` was itself minted + from the same LLMGW client (matching ``client_id`` claim). Some Delegate + backends reject ``client_credentials`` tokens outright ("Invalid user + token"), so a fresher S2S token must never be forced onto a run that + authenticated another way. +* An externally configured token file (``DELEGATE_AUTH_TOKEN_FILE`` / + ``AUTH_TOKEN_FILE``) always wins — e.g. the nightly keeps a USER token fresh + there, and this refresher must not fight it. Pointing + ``DELEGATE_AUTH_TOKEN_FILE`` at a file of your own is therefore also the way + to switch this refresher off. +* The token is minted over ``https`` only — the request body carries + ``LLMGW_CLIENT_SECRET``, so a plaintext ``LLMGW_URL`` is refused rather than + silently downgraded. +* The refresher is best-effort by construction: every mint/write failure + degrades to "keep serving the previous token and retry", never to an + exception escaping into the caller's ``start()``. + +Note that the token FILE (not the secret) is reachable by the code under test, +since its path is published on the host env the interop shells inherit. That is +a narrowing of the pre-existing ``AUTH_TOKEN`` exposure — the secret that mints +tokens stays adapter-side — but it does mean an exfiltrated token stays fresh +for the life of the run rather than dying at the first TTL boundary. +""" + +from __future__ import annotations + +import asyncio +import base64 +import contextlib +import http.client +import json +import logging +import os +import shutil +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path + + +#: The eval's LLM-Gateway S2S credentials — held by the adapter, never the host. +GATEWAY_S2S_ENV_VARS = ("LLMGW_CLIENT_ID", "LLMGW_CLIENT_SECRET", "LLMGW_URL") + +#: Env names the delegate-stdio host resolves a token file from. Both are set +#: on the host env: current bundles read the first, older ones only the second. +TOKEN_FILE_ENV_VARS = ("DELEGATE_AUTH_TOKEN_FILE", "AUTH_TOKEN_FILE") + +_MINT_TIMEOUT_SECONDS = 30.0 +#: UiPath's Cloudflare WAF bans urllib's default ``Python-urllib/3.x`` UA +#: (403, "error code: 1010") before the request reaches the IdP; any +#: descriptive UA passes. +_USER_AGENT = "coder-eval-uipath-s2s-refresher/1.0" +#: Mirror the host's refresh scheduling (delegate-stdio REFRESH_* constants) so +#: the file is rewritten just before the host itself would go looking for it. +_REFRESH_LEAD_SECONDS = 300.0 +_RETRY_SECONDS = 60.0 +_MIN_DELAY_SECONDS = 60.0 +#: Cadence when a token carries no decodable ``exp`` (matches the pipeline's +#: uip-CLI refresher interval — comfortably inside the observed 3600s TTL). +_FALLBACK_INTERVAL_SECONDS = 3000.0 + + +def decode_jwt_claims(token: str) -> dict[str, object] | None: + """Decode a JWT's payload segment without verifying the signature. + + Returns: + The claims dict, or None for malformed tokens. Claims are only + consulted for gating and refresh scheduling, never for auth. + """ + parts = token.split(".") + if len(parts) != 3: + return None + payload = parts[1] + try: + decoded = base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4)) + claims = json.loads(decoded) + except ValueError: # binascii.Error / UnicodeDecodeError / JSONDecodeError all subclass it + return None + return claims if isinstance(claims, dict) else None + + +def _token_exp_epoch_seconds(token: str) -> float | None: + claims = decode_jwt_claims(token) + exp = claims.get("exp") if claims else None + return float(exp) if isinstance(exp, (int, float)) else None + + +@dataclass(frozen=True, slots=True) +class _S2sCreds: + token_url: str + client_id: str + client_secret: str + + +def _read_creds(env: Mapping[str, str]) -> _S2sCreds | None: + """Build the credentials bundle from ``LLMGW_*``, or None when incomplete. + + ``LLMGW_URL`` may arrive as a bare host or with a gateway path appended; + the Identity Server is always mounted at the origin, so resolve against + the origin the way the host's own S2S source does. + + The scheme is pinned to ``https``: :func:`_mint_s2s_token` puts + ``LLMGW_CLIENT_SECRET`` in the request body, and a captured client_secret + mints ``service.internal`` tokens indefinitely — so a plaintext or + non-HTTP(S) ``LLMGW_URL`` declines the refresher rather than shipping the + secret in the clear. This also constrains the URL that reaches + ``urllib.request.urlopen`` (bandit B310). + """ + client_id = env.get("LLMGW_CLIENT_ID") + client_secret = env.get("LLMGW_CLIENT_SECRET") + base_url = env.get("LLMGW_URL") + if not client_id or not client_secret or not base_url: + return None + split = urllib.parse.urlsplit(base_url) + if split.scheme != "https" or not split.netloc: + return None + token_url = f"https://{split.netloc}/identity_/connect/token" + return _S2sCreds(token_url=token_url, client_id=client_id, client_secret=client_secret) + + +def _mint_s2s_token(creds: _S2sCreds) -> str: + """POST the client_credentials grant and return the access token. + + Blocking (urllib) on purpose — callers run it via ``asyncio.to_thread``. + + Raises: + RuntimeError: The IdP refused the grant or returned no access_token. + """ + body = urllib.parse.urlencode( + { + "grant_type": "client_credentials", + "client_id": creds.client_id, + "client_secret": creds.client_secret, + } + ).encode("ascii") + request = urllib.request.Request( + creds.token_url, + data=body, + headers={"Content-Type": "application/x-www-form-urlencoded", "User-Agent": _USER_AGENT}, + method="POST", + ) + try: + # B310: _read_creds pins the scheme to https, so no file:/custom scheme can reach here. + with urllib.request.urlopen(request, timeout=_MINT_TIMEOUT_SECONDS) as response: # nosec B310 + payload = json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as cause: + # The body distinguishes an IdP refusal ({"error":"invalid_client"}) + # from a WAF block ("error code: 1010") — never echo the request body. + cause_detail = "" + with contextlib.suppress(Exception): + cause_detail = cause.read().decode("utf-8", errors="replace")[:200].strip() + raise RuntimeError( + f"Token endpoint {creds.token_url} rejected client_credentials: {cause}" + + (f" | body: {cause_detail}" if cause_detail else "") + ) from cause + except (OSError, http.client.HTTPException, ValueError) as cause: + # Covers more than urllib.error.URLError on purpose: only ``h.request()`` + # is wrapped into a URLError inside urllib, so a front door that drops a + # keep-alive connection before the status line raises + # http.client.RemoteDisconnected straight out of urlopen, and + # response.read() can raise IncompleteRead / ssl.SSLError. Normalising + # them here is what makes the documented RuntimeError contract true. + raise RuntimeError(f"Token endpoint {creds.token_url} rejected client_credentials: {cause}") from cause + access_token = payload.get("access_token") + if not isinstance(access_token, str) or not access_token: + raise RuntimeError(f"Token endpoint {creds.token_url} returned no access_token") + return access_token + + +class S2sTokenFileRefresher: + """Keeps a raw-token file fresh for the Delegate host to re-read. + + Lifecycle: :meth:`maybe_create` gates activation, :meth:`start` mints the + first token, writes the file, and arms the background refresh task, + :meth:`stop` cancels the task and removes the file. One instance per + adapter ``start()``. + """ + + def __init__( + self, + creds: _S2sCreds, + inherited_token: str, + log: logging.Logger | logging.LoggerAdapter, # type: ignore[type-arg] + ) -> None: + self._creds = creds + self._inherited_token = inherited_token + self._log = log + self._dir: Path | None = None + self._task: asyncio.Task[None] | None = None + + @classmethod + def maybe_create( + cls, + env: Mapping[str, str], + log: logging.Logger | logging.LoggerAdapter, # type: ignore[type-arg] + ) -> S2sTokenFileRefresher | None: + """Create a refresher when this run can safely self-refresh, else None. + + Activation requires all of: + + * the ``LLMGW_*`` client-credentials triple in ``env``; + * no externally configured token file (that refresher owns freshness); + * an inherited ``AUTH_TOKEN`` whose ``client_id`` claim matches + ``LLMGW_CLIENT_ID`` — proof the run already authenticates with + tokens minted from this exact client, so a re-mint yields the same + kind of token the backend demonstrably accepts. + + ``env`` is the ADAPTER's own env: it is the only place the ``LLMGW_*`` + pair can still be read from, because the caller strips that pair off the + host env before getting here. + """ + creds = _read_creds(env) + if creds is None: + return None + if configured := [name for name in TOKEN_FILE_ENV_VARS if env.get(name)]: + log.debug( + "S2S token-file refresher: %s already configured — external refresher owns freshness", configured[0] + ) + return None + inherited = env.get("AUTH_TOKEN") + if not inherited: + log.debug("S2S token-file refresher: no AUTH_TOKEN in env — nothing to keep fresh") + return None + claims = decode_jwt_claims(inherited) + if claims is None or claims.get("client_id") != creds.client_id: + log.info( + "S2S token-file refresher: inherited AUTH_TOKEN was not minted by this LLMGW client " + + "(client_id mismatch) — leaving token freshness alone" + ) + return None + return cls(creds, inherited, log) + + @property + def token_file(self) -> str: + """Absolute path of the token file (available after :meth:`start`).""" + if self._dir is None: + raise RuntimeError("S2sTokenFileRefresher.start() has not run") + return str(self._dir / "delegate-auth-token") + + async def start(self) -> str: + """Mint the first token, write the file, arm the refresh task. + + Falls back to the inherited token when the initial mint fails — the + background task then keeps retrying, and the host re-reads the file + every turn, so a late first mint still lands. + + Returns: + The token-file path to publish via :data:`TOKEN_FILE_ENV_VARS`. + """ + self._dir = Path(tempfile.mkdtemp(prefix="delegate-s2s-token-")) + try: + token = await asyncio.to_thread(_mint_s2s_token, self._creds) + self._log.debug("S2S token-file refresher: initial mint OK (length=%d)", len(token)) + except Exception as error: + # Broad on purpose: the refresher is an enhancement, so ANY initial + # mint failure has to degrade to "serve the inherited token" rather + # than propagate into the caller's start(). + self._log.warning( + "S2S token-file refresher: initial mint failed (%s) — seeding the file with the inherited token", error + ) + token = self._inherited_token + self._write_token(token) + self._task = asyncio.create_task(self._refresh_loop(token)) + return self.token_file + + async def stop(self) -> None: + """Cancel the refresh task and remove the token file. Idempotent.""" + if self._task is not None: + task, self._task = self._task, None + task.cancel() + # asyncio.wait (unlike ``await task``) reports completion without + # re-raising, so the cancellation and a pre-existing failure can be + # told apart below instead of one masking the other. + await asyncio.wait({task}) + # A refresh loop that died on its own is the silent version of the + # 401-past-TTL failure this class exists to prevent, so surface it + # instead of letting the cancel/await swallow the traceback. + if not task.cancelled() and (error := task.exception()) is not None: + self._log.error("S2S token-file refresher: refresh loop had died — %r", error, exc_info=error) + if self._dir is not None: + await asyncio.to_thread(shutil.rmtree, self._dir, ignore_errors=True) + self._dir = None + + def _write_token(self, token: str) -> None: + """Atomically replace the token file (raw single-token format).""" + if self._dir is None: + raise RuntimeError("S2sTokenFileRefresher.start() has not run") + staging = self._dir / "delegate-auth-token.tmp" + staging.write_text(token, encoding="utf-8") + os.replace(staging, self.token_file) + + def _delay_until_refresh(self, token: str) -> float: + exp = _token_exp_epoch_seconds(token) + if exp is None: + return _FALLBACK_INTERVAL_SECONDS + return max(_MIN_DELAY_SECONDS, exp - time.time() - _REFRESH_LEAD_SECONDS) + + async def _refresh_loop(self, current_token: str) -> None: + delay = self._delay_until_refresh(current_token) + while True: + await asyncio.sleep(delay) + try: + token = await asyncio.to_thread(_mint_s2s_token, self._creds) + self._write_token(token) + except Exception as error: + # Broad, and covering the write as well as the mint: any escape + # here ends the task for good, and the run then dies at the TTL + # with the exact `401 Signature has expired` this class exists + # to prevent. The previous (stale but still valid) token stays + # on disk, so retrying is always better than giving up. + self._log.warning( + "S2S token-file refresher: refresh failed (%s) — retrying in %.0fs", error, _RETRY_SECONDS + ) + delay = _RETRY_SECONDS + continue + delay = self._delay_until_refresh(token) + self._log.debug("S2S token-file refresher: fresh token written; next refresh in %.0fs", delay) diff --git a/src/coder_eval/agents/delegate_sdk_agent.py b/src/coder_eval/agents/delegate_sdk_agent.py new file mode 100644 index 00000000..254a38fd --- /dev/null +++ b/src/coder_eval/agents/delegate_sdk_agent.py @@ -0,0 +1,2043 @@ +"""Delegate SDK agent — connects coder_eval to UiPath Autopilot's Delegate SDK. + +We drive the agent through the published ``@uipath/delegate-stdio`` npm +package — the Delegate agent exposed as a runnable subprocess speaking a +newline-delimited JSON protocol on stdin/stdout (referred to as "the host" +below). The package is self-contained: ``@uipath/delegate-sdk`` is its npm +dependency and installs transitively, so installing it is the COMPLETE +install — there is no separate SDK to set up. + +Prerequisites (runtime, documented — not enforced by coder_eval): + +* Install the host package (``npm install @uipath/delegate-stdio``, from the + public npm registry; the ``@uipath/delegate-sdk`` it pulls in is published + there too), then point coder_eval at it via either: + - ``DELEGATE_STDIO_PATH`` → absolute path to the bundle + (``.../@uipath/delegate-stdio/dist/delegate_stdio.mjs``), or + - ``DELEGATE_STDIO_NODE_MODULES`` → directory whose ``node_modules`` holds the + installed package. + When neither is set the bundle is auto-located by walking up from the cwd + through its ancestors (and ``~``), the way Node resolves modules — so an + ``npm install`` in the launch directory *or any ancestor* is found with zero + configuration (npm with no local ``package.json`` silently installs into the + nearest ancestor that has one, often ``~``). +* ``DELEGATE_SDK_ENV`` env var → cloud env slug (``alpha`` / ``staging`` / + ``production``); defaults to ``alpha`` when unset or blank. The host composes + the backend URL from the saved auth's org/tenant slugs plus this slug, and the + Delegate SDK spawns its own interop — so there are normally no backend or + interop URLs to configure. +* Advanced/local override only: ``BACKEND_URL`` / ``INTEROP_URL`` pin the backend + / interop endpoints directly (e.g. a localhost backend on + ``http://localhost:5002``). Host precedence: ``backendUrl`` > ``BACKEND_URL`` + env > ``env`` slug > localhost. +* ``DELEGATE_STALL_TIMEOUT_S`` env var (opt-in) → seconds to wait for the host's + first agent activity before treating a silent backend round-trip as a + transient stall and respawning+resending. Unset (default) disables it, so + callers pointing at a healthy dedicated backend are unaffected. See + :func:`_resolve_stall_timeout`. +* ``DELEGATE_STDIO_VERBOSE`` env var → trace the host's stdio protocol (every + frame it writes, every agent event, its auth/init/token-refresh steps) into + the run log. ``1``/``true`` forces it on, ``0``/``false`` forces it off, and + unset follows coder_eval's own verbosity (``--verbose``). Set it when a turn + looks hung: without it the host is silent for the whole round-trip, so a + long turn is indistinguishable from a wedged one. Companion knob + ``DELEGATE_STDIO_LOG_MAX_CHARS`` (host-side, default 50 000) caps each traced + value. See :func:`_resolve_stdio_verbose`. +* Auth: either ``AUTH_TOKEN``/``TENANT_ID``/``ORG_ID`` env vars (plus ``USER_ID`` + for S2S tokens — all consumed by the host process, never read by coder_eval + itself), or a previous ``delegate-cli login`` that wrote + ``~/.aria/sdk-auth.json``. For runs longer than the token's ~1h TTL, set + ``DELEGATE_AUTH_TOKEN_FILE`` to the token file an external process keeps + fresh: the host re-reads it at every turn and before each TTL boundary and + applies the newer token live, so a run outlives its start-up token without + this adapter touching credentials. It accepts a ``PATH``-style list and takes + the first readable entry, which is how one value covers both a host-run task + and a docker task that sees the same file at its bind-mount path. When no + such file is configured but ``AUTH_TOKEN`` was minted from the run's own + ``LLMGW_*`` client-credentials pair, the adapter maintains the token file + itself (re-minting adapter-side before each expiry) — see + :class:`coder_eval.agents._delegate_s2s_token_file.S2sTokenFileRefresher`. An auth + failure at init is non-retryable — the agent fails fast with an + :class:`AgentConfigError` (distinguishing an *expired* saved login from an + *absent* one) rather than burning the API-error retry budget. + +Notes / limitations: + +* ``token_usage`` is sourced from the framework's per-turn usage history, + summed and forwarded by the host on each ``result`` message. The Delegate + SDK does not expose pricing, so ``total_cost_usd`` is computed locally from + the reported model and token counts via + :func:`coder_eval.pricing.calculate_cost`; it stays ``None`` for + models not in the pricing table. +* Per-message transcript: ``TurnRecord.messages`` carries one + ``AssistantMessage`` per backend round-trip, reconstructed from the host's + event stream by :class:`_TranscriptBuilder`, with per-generation token + buckets zipped from the ``result`` message's ``turnUsages`` list (one entry + per round-trip; their sum IS ``usage``). When the host predates + ``turnUsages`` or the counts misalign, the messages carry content/timing but + no tokens — the ``EventCollector``'s reconciliation entry then carries the + turn total, preserving the transcript-sums-to-total invariant either way. +* ``allowed_tools``, ``disallowed_tools``, ``system_prompt``, ``system_prompt_file``, + and ``setting_sources`` from :class:`DelegateSdkAgentConfig` have no Delegate-SDK + equivalents; a warning is logged if any are set. ``permission_mode`` is equally + unsupported but silently ignored — it always carries a truthy value, so warning + on it would be noise on every run (see ``_UNSUPPORTED_FIELDS``). +* Plugins with a ``path`` field map to the SDK's ``bundledSkillsPath`` (assumed to be + ``/skills``). Multiple plugins: first wins; a warning is logged. +* Sandbox mock CLIs (``SandboxConfig.mock_path_dirs``, delivered to the agent ABC as + ``env_path_prepend``) are forwarded as the ``shellPathPrepend`` init option, which + the SDK injects into the environment of every shell command it runs inside the + interop service. Requires a ``@uipath/delegate-stdio`` build whose bundled SDK + understands ``shellPathPrepend``; older hosts ignore the option, in which case + mock-graded docker-sandbox tasks fall back to the overlay image's cwd-aware ``uip`` + shim (host/tempdir and Windows runs have no shim, so there mock shadowing is simply + lost). See :meth:`DelegateSdkAgent.start`. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import logging +import os +import re +import time +import uuid +from collections import deque +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, ClassVar, NoReturn +from urllib.parse import urlparse + +from coder_eval.agent import Agent, AgentState +from coder_eval.agents._delegate_s2s_token_file import ( + GATEWAY_S2S_ENV_VARS, + TOKEN_FILE_ENV_VARS, + S2sTokenFileRefresher, +) +from coder_eval.agents._logging import PrefixedAdapter +from coder_eval.agents.registry import AgentRegistry +from coder_eval.errors import AgentConfigError, AgentCrashError, TurnTimeoutError, truncate_crash_message +from coder_eval.models import ( + AgentKind, + ApiRoute, + AssistantMessage, + CommandTelemetry, + ContentBlock, + DelegateSdkAgentConfig, + DirectRoute, + SystemPromptSemantics, + TokenUsage, + TranscriptMessage, + TurnRecord, +) +from coder_eval.pricing import calculate_cost +from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback +from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.events import ( + AgentEndEvent, + AgentEndStatus, + AgentStartEvent, + TextChunkEvent, + ToolEndEvent, + ToolEndStatus, + ToolStartEvent, + TurnEndEvent, + TurnEndStatus, + TurnStartEvent, +) +from coder_eval.utils import process_plugins + + +logger = logging.getLogger(__name__) + + +# ---- Constants -------------------------------------------------------------- + +_STDIO_PACKAGE = "@uipath/delegate-stdio" +"""npm package (public npm registry) that ships the bundled host.""" + +_STDIO_BUNDLE_REL_PATH = Path("node_modules") / "@uipath" / "delegate-stdio" / "dist" / "delegate_stdio.mjs" +"""Path to the bundle inside an install dir's ``node_modules``.""" + + +def _candidate_install_roots() -> list[Path]: + """Install roots to probe for ``node_modules/@uipath/...`` when nothing is configured. + + Mirrors Node's own module resolution: start at the cwd and walk up through + every ancestor directory. ``npm install `` run in a directory with no + ``package.json`` silently installs into the nearest *ancestor* that has one + (frequently the user's home directory), so the bundle often lands above the + cwd rather than in it. Home (``~``) is appended explicitly because it is not + always an ancestor of the cwd (e.g. on Windows the cwd lives under + ``C:\\source\\...`` while home is ``C:\\Users\\...``). + """ + cwd = Path(os.getcwd()).resolve() + roots: list[Path] = [cwd, *cwd.parents] + home = Path.home().resolve() + if home not in roots: + roots.append(home) + return roots + + +def _resolve_stdio_bundle() -> Path: + """Locate the installed host bundle (``dist/delegate_stdio.mjs``). + + Resolution order: + 1. ``DELEGATE_STDIO_PATH`` — explicit absolute path to the bundle. CI and + advanced setups set this directly (e.g. to the package's installed bin). + 2. ``DELEGATE_STDIO_NODE_MODULES`` — explicit install root; probed exactly + (no walk-up — the operator told us where it is). + 3. Otherwise, walk up from the cwd through its ancestors (plus ``~``), the + way Node resolves modules, so an ``npm install`` in any ancestor — or in + home, where npm lands it when the cwd has no ``package.json`` — is found + with zero configuration. + + The agent invokes Node directly on the bundle, getting ~1s cold start and + avoiding any runtime CJS/ESM interop hazards from the dependency graph. + """ + explicit = os.environ.get("DELEGATE_STDIO_PATH") + if explicit: + path = Path(explicit).expanduser().resolve() + if not path.is_file(): + raise AgentConfigError( + f"DELEGATE_STDIO_PATH={path} does not point to a file. Point it at the " + + f"{_STDIO_PACKAGE} bundle (dist/delegate_stdio.mjs).", + ) + return path + + base_override = os.environ.get("DELEGATE_STDIO_NODE_MODULES") + if base_override: + path = (Path(base_override).expanduser().resolve() / _STDIO_BUNDLE_REL_PATH).resolve() + if not path.is_file(): + raise AgentConfigError( + f"DELEGATE_STDIO_NODE_MODULES={base_override}: delegate-stdio bundle not found at {path}. " + + f"Run `npm install {_STDIO_PACKAGE}` there, or set DELEGATE_STDIO_PATH to the bundle directly.", + ) + return path + + searched: list[Path] = [] + for root in _candidate_install_roots(): + candidate = (root / _STDIO_BUNDLE_REL_PATH).resolve() + searched.append(candidate) + if candidate.is_file(): + return candidate + + searched_block = "\n ".join(str(p) for p in searched) + raise AgentConfigError( + f"delegate-stdio bundle not found. Searched the cwd, its ancestors, and home:\n {searched_block}\n" + + f"Run `npm install {_STDIO_PACKAGE}` in {os.getcwd()} (installs into ./node_modules), " + + "or set DELEGATE_STDIO_NODE_MODULES to the install root that holds node_modules/@uipath/..., " + + "or set DELEGATE_STDIO_PATH to the bundle directly.", + ) + + +def _maybe_pin_npm_globalconfig(host_env: dict[str, str], plugin_tools_dir: str | None) -> Path | None: + """Pin ``NPM_CONFIG_GLOBALCONFIG`` for the host so shells keep the global npmrc. + + The Delegate runtime's interop injects ``npm_config_prefix=~/.aria/npm/prefix`` + into every shell command it spawns (Aria's desktop guard against EACCES on + read-only install dirs). Both npm and the uip CLI derive the global config as + ``/etc/npmrc``, so that injection silently relocates the lookup away + from the real global npmrc — losing the ``@uipath:registry`` + auth token an + image or operator baked there, and breaking on-demand ``uip tools install`` + ("No compatible version ... on npm or GitHub Packages"). Both npm and uip + consult ``NPM_CONFIG_GLOBALCONFIG`` *before* the prefix-derived path, and the + interop forwards inherited env untouched, so pinning it here restores the + baked registry config for every shell in the host's process tree. + + Strictly additive: an operator-set value always wins, and when no global + npmrc exists the env is left unchanged (there is nothing to pin — behavior + identical to today). ``plugin_tools_dir`` is ``/@uipath`` of + the npm installation that owns ``uip``; npm's global ``node_modules`` sits at + ``/lib/node_modules`` (POSIX) or ``/node_modules`` (Windows), + so both shapes are probed and the existence check disambiguates. + + Returns the pinned path, or None when nothing was pinned. + """ + if "NPM_CONFIG_GLOBALCONFIG" in host_env or not plugin_tools_dir: + return None + node_modules = Path(plugin_tools_dir).resolve().parent + for prefix in (node_modules.parent.parent, node_modules.parent): + candidate = prefix / "etc" / "npmrc" + if candidate.is_file(): + host_env["NPM_CONFIG_GLOBALCONFIG"] = str(candidate) + return candidate + return None + + +def _strip_gateway_creds(host_env: dict[str, str]) -> tuple[str, ...]: + """Remove the eval's LLMGW_* gateway S2S credentials from the host env. + + Everything in the host env is inherited by the shells its interop spawns + for the agent's own Bash / PowerShell tool calls, i.e. by the code under + test. Withholding a live client secret from that surface is the + least-privilege default; the docker driver already does the same by keeping + LLMGW_* off run.py's env allowlist, so this gives host-run (tempdir) tasks + the same treatment. When the pair is needed for token freshness, the + adapter consumes it itself (see :class:`S2sTokenFileRefresher`) and hands + the host a token FILE instead. + + Returns the names actually removed. + """ + return tuple(name for name in GATEWAY_S2S_ENV_VARS if host_env.pop(name, None) is not None) + + +def _resolve_stall_timeout() -> float | None: + """First-response stall ceiling in seconds, or ``None`` when disabled. + + Read from ``DELEGATE_STALL_TIMEOUT_S``. When set to a positive number, a + turn that receives NO agent activity from the host within this many seconds + of sending the prompt is treated as a transient backend stall: the wedged + host is respawned and the prompt resent (see :meth:`DelegateSdkAgent.communicate`). + Left unset (the default) the detection is off and ``communicate`` behaves + exactly as before — so callers pointing the host at a dedicated, healthy + backend (e.g. the Autopilot RPA eval's per-build ACI) are unaffected. The + guard covers only the wait for the FIRST activity, so an in-flight tool + execution (a long ``tool_call`` with no interim events) never trips it. + """ + raw = os.environ.get("DELEGATE_STALL_TIMEOUT_S") + if not raw: + return None + try: + seconds = float(raw) + except ValueError: + logger.warning("Ignoring non-numeric DELEGATE_STALL_TIMEOUT_S=%r; stall detection stays off", raw) + return None + return seconds if seconds > 0 else None + + +_VERBOSE_ENV_VAR = "DELEGATE_STDIO_VERBOSE" +"""Host tracing switch — read here AND by the host itself (it parses '1'/'true').""" + +_VERBOSE_TRUTHY = frozenset({"1", "true"}) +_VERBOSE_FALSY = frozenset({"0", "false"}) + + +def _resolve_stdio_verbose() -> bool: + """Whether the Delegate host should trace its stdio protocol to stderr. + + ``DELEGATE_STDIO_VERBOSE`` is the operator's switch and wins in BOTH + directions: ``1``/``true`` forces tracing on even on a quiet run, and + ``0``/``false`` forces it off even under ``--verbose``. Unset (the default) + follows coder_eval's own verbosity, so ``--verbose`` traces and a normal run + stays quiet. + + The vocabulary deliberately mirrors what the host's own ``DEBUG_LOGGING`` + gate parses, so the two sides can never disagree about what a value means; + anything else warns and falls back to the ``--verbose`` default rather than + guessing. Verbosity is read off this module's logger, a child of the + ``coder_eval`` app logger that ``setup_logging`` configures, so ``--verbose`` + is exactly what flips it. + """ + raw = os.environ.get(_VERBOSE_ENV_VAR, "").strip().lower() + if raw in _VERBOSE_TRUTHY: + return True + if raw in _VERBOSE_FALSY: + return False + if raw: + logger.warning( + "Ignoring unrecognised %s=%r (expected one of %s); falling back to coder_eval's verbosity", + _VERBOSE_ENV_VAR, + raw, + ", ".join(sorted(_VERBOSE_TRUTHY | _VERBOSE_FALSY)), + ) + return logger.isEnabledFor(logging.DEBUG) + + +_STOP_TIMEOUT_SEC = 60.0 +"""How long to wait for the host process to exit after `destroy` before killing it.""" + +_DEFAULT_STALL_RESENDS = 1 +"""In-turn resend attempts when the host produces no output within the stall +window (see :func:`_resolve_stall_timeout`). One resend absorbs a transient +backend blip while staying inside the ``task_timeout`` budget; a persistent +stall then rides the normal turn/task timeout, surfacing a sustained backend +outage rather than masking it.""" + +_ACTIVITY_EVENT_TYPES = frozenset({"thinking", "message", "tool_call", "tool_result"}) +"""Host event types that prove the backend round-trip is progressing. Receiving +any of these (or a ``result``) clears the first-response stall guard. Excludes +the informational ``session_start`` — the host can emit it *before* the backend +call, so it must not reset the guard.""" + +_STREAM_READER_LIMIT_BYTES = 64 * 1024 * 1024 +"""Per-line buffer for the host's stdout/stderr StreamReaders (one ``limit=`` +kwarg on ``create_subprocess_exec`` sizes both). asyncio's 64 KiB default is far +too small — a single ``result`` frame carries the agent's full reply, and tool +payloads reach multiple MB. 64 MiB keeps the protocol JSON intact while still +bounding runaway-output memory; the drain tasks catch ``LimitOverrunError`` +beyond it (stdout treats the host as crashed, stderr drops the line).""" + +_UNSUPPORTED_FIELDS = ( + "allowed_tools", + "disallowed_tools", + "system_prompt", + "system_prompt_file", + "setting_sources", +) +"""AgentConfig fields that have no Delegate SDK equivalent. + +``permission_mode`` is intentionally absent: the Delegate SDK has no +permission-prompt concept, so silently ignoring it (any value) is fine. Its +default ``"acceptEdits"`` is truthy, so listing it here would emit a noisy +WARNING on every run.""" + + +# ---- Auth-failure diagnostics ---------------------------------------------- + +_AUTH_ERROR_MARKERS = ( + "auth required", + "authentication", + "unauthorized", + "invalid credentials", + "invalid token", + "invalid api key", + "no auth", + "expired", + "401", + "403", +) +"""Substrings (matched case-insensitively) that mark an init failure as an auth +problem. Auth failures never succeed on retry, so they short-circuit to a +non-retryable :class:`AgentConfigError` instead of the retryable AGENT_API_ERROR +the host's generic ``RuntimeError`` would otherwise be categorized as.""" + + +def _is_auth_init_error(message: str) -> bool: + """True when a host init-error message looks like an auth failure.""" + lowered = message.lower() + return any(marker in lowered for marker in _AUTH_ERROR_MARKERS) + + +_SESSION_CONFLICT_MARKER = "already being generated" +"""Backend-409 fingerprint (matched case-insensitively): the SDK's own SSE +idle-watchdog / network-retry layer re-POSTed a message into a conversation +whose previous generation is still running, and the backend rejected it with +"A reply is already being generated for this conversation.". Resending into +the SAME conversation can only re-conflict, so every AGENT_CRASH retry burns +against the wedged conversation (build 12679882: 14 tasks died 3/3 attempts). + +Dropping the adapter's ``_session_id`` alone is NOT enough (build 12685191: +4/7 Windows tasks still died): the failures hit on the task's FIRST turn, so +there is no session id to drop — and even with ``sessionId: null`` the live +host resumes the wedged conversation anyway, because the SDK's +``sendMessage(prompt, undefined)`` falls back to its in-memory +``currentSessionId``. The only clean recovery is a FRESH HOST: +:meth:`DelegateSdkAgent._crash` kills the wedged host on this marker and +:meth:`DelegateSdkAgent.communicate`'s entry guard respawns one for the +retry, whose new ``DelegateAgent`` starts with no current session.""" + + +_WAF_BLOCK_PAGE_MARKERS = ("continue with uipath platform", "not available in your country") +"""Fingerprints (matched case-insensitively) of UiPath's generic Cloudflare +block page — ``Continue with UiPath Platform`` over "We are +sorry. UiPath platform is not available in your country." — served as an HTTP +403 when a managed WAF rule matches the REQUEST BODY. Despite the wording it is +NOT a geo/IP block: shell-like text in a tool result or prompt (e.g. a skill +doc's ``python -c "...open(...,'w')..."`` one-liner echoed back by ReadFile) +trips Cloudflare's command-injection rules in front of alpha.uipath.com, and +the request never reaches the backend (run adhoc-2026-07-24_16-51-27: 61 tasks, +47/47 uipath-maestro-case). + +Two markers because the host truncates its error message at ~50 KB while the +page is ~68 KB: the visible text sits AFTER ~48 KB of base64 web-font CSS, so +the mid-turn tool-result PATCH shape — whose message is additionally prefixed +by the request URL — is cut before the country sentence ever appears. The +```` lands in the first ~300 bytes and survives both shapes; in that run +it matched 61/61 crash messages against the country text's 48/61.""" + + +def _describe_waf_block(reason: str) -> str: + """Concise, correctly-categorized replacement for a WAF-block crash reason. + + The raw reason embeds the whole block-page HTML, which is noise and — worse + — reads as a geo/auth problem. The block is deterministic per payload (the + retried turn re-sends the same tripping content), so the message stamps the + framework categorizer's "content filter" signature, routing the failure to + the non-retryable ``AGENT_INVALID_OUTPUT`` instead of burning AGENT_CRASH + retries that fail identically. + """ + prefix = reason.split("<", 1)[0].strip().rstrip(":").strip() + return ( + "Backend request blocked by the Cloudflare WAF content filter in front of the UiPath backend " + "(the generic 'not available in your country' 403 page — not a geo or auth problem): shell-like " + "text in the prompt or a tool result matched a command-injection managed rule, and the same " + "payload would be blocked again on retry. Fix by defanging shell one-liners in the skill docs " + f"the task reads, or exempting the delegate_ route from the WAF managed rules. [{prefix}]" + ) + + +_SSE_CONNECT_TIMEOUT_MARKER = "sse connect timeout" +"""Fingerprint (matched case-insensitively) of the delegate host's SSE +connect/first-byte watchdog: a turn's POST got no HTTP response headers within +``SSE_CONNECT_TIMEOUT_MS`` (30s, agenticApi.ts), and the SDK's agentic loop has +already burned its 3 back-to-back retries — the error only surfaces after ~90s +of continuous backend front-door silence (build 12874194: both Windows failures +show the exact 3x30s signature, 90.0s from last tool result to error). + +The raw message's "timeout" wording makes the framework categorizer route the +crash to AGENT_TIMEOUT — non-retryable, because for genuine task/turn *budget* +breaches a retry just re-burns the budget. This failure is neither: no headers +means the backend never started answering (streaming resets a separate 180s +idle watchdog once headers arrive, and the backend deliberately keeps the +pre-header phase minimal), so it marks a transient availability window that an +orchestrator retry with backoff usually lands outside of. +:meth:`DelegateSdkAgent._crash` rewrites the reason via +:func:`_describe_sse_connect_timeout` to stamp the "connection" signature +(→ retryable AGENT_API_ERROR, 5s/10s/20s backoff). The live host is kept — its +conversation context makes the resend a genuine continuation; if the resend +races a half-released backend turn claim, the resulting 409 lands in +:data:`_SESSION_CONFLICT_MARKER`'s fresh-host recovery.""" + + +def _describe_sse_connect_timeout(reason: str) -> str: + """Concise, correctly-categorized replacement for an SSE connect-watchdog crash reason. + + Stamps the framework categorizer's "connection" signature (→ retryable + ``AGENT_API_ERROR``) and defangs every "timeout" occurrence so the earlier, + non-retryable AGENT_TIMEOUT arm cannot match. The original watchdog wording + (with its window length) is preserved defanged in brackets. + """ + marker_at = reason.lower().find(_SSE_CONNECT_TIMEOUT_MARKER) + original = reason[marker_at:] if marker_at >= 0 else reason + defanged = re.sub("timeout", "time-out", original, flags=re.IGNORECASE) + return ( + "Delegate backend connection failure: the turn's POST received no HTTP response headers within " + "the host's SSE connect watchdog window, and the SDK's three back-to-back internal attempts all " + "hit the same wall (~90s of continuous front-door silence) — a transient backend availability " + "window, not a task-budget breach. The live host and its conversation are kept, so a delayed " + f"resend continues where the turn left off. [{defanged}]" + ) + + +def _saved_auth_file() -> Path: + """Path to the Delegate SDK's saved-login file (``~/.aria/sdk-auth.json``).""" + return Path.home() / ".aria" / "sdk-auth.json" + + +def _format_duration(seconds: float) -> str: + """Render a non-negative duration coarsely (days / hours / minutes).""" + seconds = max(0.0, seconds) + for unit_seconds, label in ((86_400.0, "day"), (3_600.0, "hour"), (60.0, "minute")): + value = round(seconds / unit_seconds) + if seconds >= unit_seconds: + return f"~{value} {label}{'s' if value != 1 else ''}" + return "less than a minute" + + +def _describe_saved_auth() -> str: + """Describe the saved-login state for an auth-failure diagnostic. + + Distinguishes *absent* from *expired* — which the host's generic + "Auth required" message cannot. Reads ``~/.aria/sdk-auth.json`` and, when its + ``expiresAt`` (epoch seconds) is in the past, reports how long ago. The + host has already attempted a token refresh (``loadAndRefreshAuth``) by the + time we see the failure, so an expired file means the refresh failed too. + Best-effort: never raises. + """ + auth_file = _saved_auth_file() + if not auth_file.is_file(): + return f"No saved login found at {auth_file}." + try: + data = json.loads(auth_file.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + return f"Saved login at {auth_file} could not be read ({exc})." + expires_at = data.get("expiresAt") if isinstance(data, dict) else None + if not isinstance(expires_at, (int, float)) or isinstance(expires_at, bool): + return f"Saved login found at {auth_file} (no expiry recorded)." + age_seconds = time.time() - expires_at + if age_seconds > 0: + return f"Saved login at {auth_file} expired {_format_duration(age_seconds)} ago and could not be refreshed." + return f"Saved login at {auth_file} is unexpired (expires in {_format_duration(-age_seconds)})." + + +# ---- Plugin / skills resolution -------------------------------------------- + + +def _resolve_bundled_skills_path(plugins: list[dict[str, Any]] | None) -> str | None: + """Translate coder_eval's plugin list to the SDK's ``bundledSkillsPath`` option. + + The Delegate SDK expects ``bundledSkillsPath`` to point at a directory whose + direct children are skill folders (each containing a ``SKILL.md``). coder_eval + plugins wrap that layout under ``<plugin>/skills/<skill>/SKILL.md``, so we append + ``/skills`` to the plugin root. + + If multiple plugins are provided we use the first and log a warning listing the + others — merging multiple plugin skill directories is out of scope for now. + """ + expanded = process_plugins(plugins or [], log=logger) + if not expanded: + return None + + first = expanded[0] + if "path" not in first: + logger.warning("Delegate SDK plugin missing 'path' field — skipping: %r", first) + return None + + if len(expanded) > 1: + others = [p.get("path", repr(p)) for p in expanded[1:]] + logger.warning( + "Delegate SDK supports only one plugin; using %s and ignoring: %s", + first["path"], + ", ".join(others), + ) + + skills_path = Path(first["path"]) / "skills" + return str(skills_path) + + +# ---- Transcript reconstruction ---------------------------------------------- + + +def _usage_bucket_ints(raw: Any) -> tuple[int, int, int, int]: + """Read the four token buckets off one host ``turnUsages`` entry. + + Returns ``(input, output, cache_creation, cache_read)``, zero for missing + or invalid values — same tolerance as :meth:`DelegateSdkAgent._parse_usage`. + """ + + def _int(value: Any) -> int: + return value if isinstance(value, int) and value >= 0 else 0 + + if not isinstance(raw, dict): + return 0, 0, 0, 0 + return ( + _int(raw.get("input_tokens")), + _int(raw.get("output_tokens")), + _int(raw.get("cache_creation_input_tokens")), + _int(raw.get("cache_read_input_tokens")), + ) + + +class _GenerationSegment: + """One LLM round-trip (generation) reconstructed from the host's event stream.""" + + def __init__(self, started_at: datetime) -> None: + self.started_at = started_at + self.completed_at = started_at + # (kind, payload) in emission order; kind is "thinking"/"text" (payload + # is the accumulated text) or "tool_use" (payload is the tool_use_id). + # Consecutive same-kind text payloads merge so streaming deltas form + # one block instead of one block per token. + self.blocks: list[list[str]] = [] + self.tool_use_ids: list[str] = [] + self.saw_tool_result = False + + def append_text(self, kind: str, text: str, now: datetime) -> None: + if self.blocks and self.blocks[-1][0] == kind: + self.blocks[-1][1] += text + else: + self.blocks.append([kind, text]) + self.completed_at = now + + def append_tool_use(self, tool_id: str, now: datetime) -> None: + self.blocks.append(["tool_use", tool_id]) + self.tool_use_ids.append(tool_id) + self.completed_at = now + + @property + def has_text_or_tools(self) -> bool: + return any(kind != "thinking" for kind, _ in self.blocks) + + +class _TranscriptBuilder: + """Reconstruct per-generation ``AssistantMessage`` entries from host events. + + The Delegate host streams ``thinking`` / ``message`` / ``tool_call`` / + ``tool_result`` events but carries no explicit generation-boundary marker, + so boundaries are derived from stream order (the host forwards events on + one ordered stdout pipe, preserving emission order): + + * a ``message`` event tagged ``isStepStart`` starts a new generation — + unless the current one holds only thinking content, in which case the + text belongs to the same round-trip (thinking streams before text); + * any generation activity (thinking / message / tool_call) arriving after a + ``tool_result`` belongs to the NEXT round-trip: the agentic loop is + client-driven, so tool results are sent back in a fresh backend request. + + The reconstructed segment list lines up 1:1 with the host's + ``turnUsages`` array (one entry per backend round-trip) in the normal case, + which is what lets :meth:`build_messages` stamp real per-generation token + buckets onto the transcript. When the counts disagree, stamping is skipped + entirely — the ``EventCollector``'s reconciliation entry then carries the + whole turn total, exactly as before — rather than risk misattributing + buckets to the wrong generation. + + Timing follows :class:`ClaudeCodeAgent` semantics: a generation *starts* at + the wall-clock arrival of the previous host event (which for round-trip + N+1 is round-trip N's last ``tool_result`` — i.e. when the next backend + request goes out) and *completes* at the arrival of its own last + generation event. Tool execution time is therefore excluded. + """ + + def __init__(self) -> None: + self._segments: list[_GenerationSegment] = [] + self._prev_event_at: datetime | None = None + + def _open_segment(self, now: datetime) -> _GenerationSegment: + segment = _GenerationSegment(self._prev_event_at or now) + self._segments.append(segment) + return segment + + def _segment_for_generation(self, now: datetime) -> _GenerationSegment: + """Current segment, or a fresh one when none is open / a round ended.""" + current = self._segments[-1] if self._segments else None + if current is None or current.saw_tool_result: + current = self._open_segment(now) + return current + + def on_thinking(self, text: str) -> None: + now = datetime.now() + self._segment_for_generation(now).append_text("thinking", text, now) + self._prev_event_at = now + + def on_message(self, text: str, *, is_step_start: bool) -> None: + now = datetime.now() + current = self._segments[-1] if self._segments else None + # isStepStart only splits when the current segment already carries this + # round's text or tools — a thinking-only segment means the round's + # first text just arrived and joins it. Missing/False (streaming delta, + # or an un-rebuilt host that never tags) appends to the current + # segment so an old host degrades to merged text, not one segment + # per streamed token. + if current is None or current.saw_tool_result or (is_step_start and current.has_text_or_tools): + current = self._open_segment(now) + current.append_text("text", text, now) + self._prev_event_at = now + + def on_tool_call(self, tool_id: str) -> None: + now = datetime.now() + self._segment_for_generation(now).append_tool_use(tool_id, now) + self._prev_event_at = now + + def on_tool_result(self) -> None: + now = datetime.now() + if self._segments: + self._segments[-1].saw_tool_result = True + self._prev_event_at = now + + def final_text(self) -> str: + """The assistant's final answer, assembled from the merged ``text`` blocks + of the last text-bearing generation. + + Used as the ``agent_output`` fallback when the host's ``result`` message + carries no authoritative ``response`` — e.g. a build that streams the + final answer as deltas (each delta replaces, so the running scalar would + keep only the last fragment), or a crash/timeout partial with no result + message. Returns ``""`` when no text was produced. + """ + for segment in reversed(self._segments): + texts = [payload for kind, payload in segment.blocks if kind == "text"] + if texts: + return "".join(texts) + return "" + + def build_messages( + self, + *, + turn_id: str, + model: str | None, + turn_usages: Any, + log: logging.Logger | logging.LoggerAdapter, # type: ignore[type-arg] + ) -> list[TranscriptMessage]: + """Materialize the segments as ``AssistantMessage`` transcript entries. + + ``turn_usages`` is the host's per-round-trip usage list from the + ``result`` message (``None`` for crash partials and older hosts). + Token buckets are stamped only when it zips 1:1 onto the segments; the + bucket sum then equals the turn total by construction (the host's + ``usage`` is the sum of the same entries), so the collector's + reconciliation entry vanishes instead of carrying the whole bill. + + Each message gets a synthetic per-generation ``message_id`` + (``{turn_id}-msg-{index}``, mirroring CodexAgent) so the evalboard + groups exactly one row per generation instead of falling back to its + wall-clock gap heuristic. + """ + usages: list[Any] | None = turn_usages if isinstance(turn_usages, list) else None + if usages is not None and len(usages) != len(self._segments): + # Misattributing buckets is worse than not attributing: skip the + # stamping wholesale and let the reconciliation entry carry the total. + log.warning( + "Host sent %d turnUsages entries but %d generations were reconstructed; skipping per-message tokens", + len(usages), + len(self._segments), + ) + usages = None + + messages: list[TranscriptMessage] = [] + for index, segment in enumerate(self._segments): + blocks: list[ContentBlock] = [] + for sequence, (kind, payload) in enumerate(segment.blocks): + if kind == "thinking": + blocks.append(ContentBlock(block_type="thinking", sequence=sequence, thinking=payload)) + elif kind == "text": + blocks.append(ContentBlock(block_type="text", sequence=sequence, text=payload)) + else: + blocks.append(ContentBlock(block_type="tool_use", sequence=sequence, tool_use_id=payload)) + input_tokens, output_tokens, cache_creation, cache_read = _usage_bucket_ints( + usages[index] if usages is not None else None + ) + duration_ms = max(0.0, (segment.completed_at - segment.started_at).total_seconds() * 1000.0) + messages.append( + AssistantMessage( + started_at=segment.started_at, + completed_at=segment.completed_at, + generation_duration_ms=duration_ms, + content_blocks=blocks, + tool_use_ids=list(segment.tool_use_ids), + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_creation_tokens=cache_creation, + cache_read_tokens=cache_read, + model=model, + message_id=f"{turn_id}-msg-{index}", + ) + ) + return messages + + +# ---- Per-turn mutable state ------------------------------------------------ + + +@dataclass +class _TurnState: + """Mutable accumulators + per-turn context for one ``communicate()`` call. + + Bundling these (previously ~12 ``nonlocal`` closures inside ``communicate``) + into one object lets the per-message-type handling live in small methods that + take the state explicitly, instead of one ~390-line god-method. + """ + + # Per-turn context (set once at construction). + task_id: str + turn_id: str + turn_start: float + user_input: str + iteration: int + emit: CompositeStreamCallback + collector: EventCollector + transcript: _TranscriptBuilder + + # Accumulators mutated while draining the host event stream. + commands: dict[str, dict[str, Any]] = field(default_factory=dict) # tool_id -> {telemetry, start_time} + ended_tool_ids: set[str] = field(default_factory=set) + sequence_number: int = 0 + # Incremented per ``isStepStart: true`` message event (one per LLM round-trip), + # matching ClaudeCodeAgent's "one turn per AssistantMessage" semantics; the host + # leaves streaming-delta events untagged so naive counting would inflate this. + # Overwritten by the result message's authoritative ``assistantStepCount``. + assistant_turn_count: int = 0 + model_used: str | None = None + token_usage: TokenUsage | None = None + max_turns_exhausted: bool = False + error_message: str | None = None + final_response: str = "" + turn_usages: Any = None + finalized: bool = False + + +# ---- The agent ------------------------------------------------------------- + + +@AgentRegistry.register(AgentKind.DELEGATE_SDK, DelegateSdkAgentConfig) +class DelegateSdkAgent(Agent[DelegateSdkAgentConfig]): + """Agent adapter that drives the UiPath Delegate SDK via a Node subprocess. + + See module docstring for prerequisites and limitations. + """ + + # The Delegate SDK has no system-prompt knob — ``system_prompt`` sits in + # ``_UNSUPPORTED_FIELDS`` and is warned about at start() — so the honest + # regime is ``"unknown"`` (also the base default). Declared explicitly so + # the run marker is deliberate rather than an unset oversight. + system_prompt_semantics: ClassVar[SystemPromptSemantics] = "unknown" + + def __init__( + self, + config: DelegateSdkAgentConfig, + route: ApiRoute | None = None, + *, + instance_name: str = "delegate", + ) -> None: + """Initialise the agent. + + Args: + config: Resolved task agent configuration. + route: Accepted for API symmetry with :class:`ClaudeCodeAgent`; ignored. + The Delegate SDK has its own UiPath auth path (not Anthropic-style routing). + instance_name: Short label used to prefix this instance's log records. + """ + self.config = config + self.route = route or DirectRoute() # stored for diagnostics only + self.working_directory: Path | None = None + + # Turn-lifecycle state (_state / _iteration / _iteration_was_incremented / + # pending_turn) lives on the Agent base class — managed via _begin_turn() + # / _end_turn_ok() / discard_pending_turn() / _mark_stopped(). + self._session_id: str | None = None + + self._process: asyncio.subprocess.Process | None = None + self._stderr_task: asyncio.Task[None] | None = None + self._stdout_task: asyncio.Task[None] | None = None + self._stdout_queue: asyncio.Queue[dict[str, Any] | None] = asyncio.Queue() + # Bounded ring buffer: the verbose writeLine echo can stream many large + # lines to stderr; only the last entries feed the crash-message tail. + self._stderr_lines: deque[str] = deque(maxlen=200) + self._init_options: dict[str, Any] | None = None + self._stdio_bundle: Path | None = None + # Cached so the host can be re-established mid-turn without re-running + # start() (the orchestrator retries communicate() only). Populated by + # start(); consumed by _spawn_and_init()/_respawn_host(). + self._host_env: dict[str, str] | None = None + self._stall_timeout: float | None = None + # Set by _crash() on a backend session conflict (the wedged host was + # killed); tells communicate()'s entry guard to respawn a fresh host + # for the AGENT_CRASH retry instead of failing fast. + self._respawn_before_retry = False + # Keeps the host's token file fresh past the ~1h S2S TTL (None when the + # run cannot safely self-refresh — see S2sTokenFileRefresher.maybe_create). + self._token_refresher: S2sTokenFileRefresher | None = None + self._log = PrefixedAdapter(logger, {"prefix": instance_name}) + # Level at which _drain_stderr forwards the host's stderr lines. start() + # raises it to INFO when DELEGATE_STDIO_VERBOSE forces tracing on a quiet + # run, where DEBUG records would be dropped by the app's INFO handlers. + self._host_stderr_log_level = logging.DEBUG + + # -- Agent ABC ----------------------------------------------------------- + + async def start( + self, + working_directory: str, + *, + env_path_prepend: list[str] | None = None, + plugin_tools_dir: str | None = None, + ) -> None: + """Spawn the host subprocess and initialise the Delegate agent. + + ``env_path_prepend`` and ``plugin_tools_dir`` are part of the + :class:`Agent` ABC for agents that shell out (sandbox PATH injection + and pinning the UiPath CLI's plugin discovery via ``PLUGIN_TOOLS_DIR``, + respectively). ``env_path_prepend`` (the sandbox's mock_path_dirs — + e.g. a ``mocks/uip`` wrapper that must shadow the real CLI) is + forwarded to the host as the ``shellPathPrepend`` init option rather + than applied to this process: shell tools execute inside the interop + service, whose own PATH was fixed at spawn (and which is deliberately + reused across runs), so a prepend here could never reach them. The SDK + delivers it through the per-command environment interop applies to each + shell child — see the Autopilot-side ``runtime/shellPathEnv.ts``. Hosts + whose bundled SDK predates the option ignore it; docker-sandbox tasks + then fall back to the overlay image's cwd-aware ``uip`` shim + (uip-mock-shim.sh), which dispatches to the nearest ``mocks/uip`` above + the command's cwd — host/tempdir and Windows runs don't install that + shim, so on those paths mock shadowing is lost entirely. + ``plugin_tools_dir`` is used only as the anchor for the + ``NPM_CONFIG_GLOBALCONFIG`` pin (see + :func:`_maybe_pin_npm_globalconfig`). + """ + # The orchestrator retries start() on a retryable init failure without an + # intervening stop(); reclaim any subprocess/drain tasks from a prior + # attempt first so a retry can't orphan the previous Node host. + await self._teardown_host() + self.working_directory = Path(working_directory) + self._state = AgentState.WORKING + + self._warn_unsupported_fields() + + bundle_path = _resolve_stdio_bundle() + self._stdio_bundle = bundle_path + + # Host tracing + our own log visibility are one decision: the host writes + # its trace to stderr and _drain_stderr forwards it line by line. Under + # --verbose this module's logger (a child of the ``coder_eval`` app + # logger) is already at DEBUG, so the forwarded lines reach the run log + # as DEBUG records. When the operator forces DELEGATE_STDIO_VERBOSE on a + # QUIET run, DEBUG records would be dropped by the app's INFO-level + # handlers — so the trace is forwarded at INFO instead, which is the only + # way the switch can deliver what it promises. + stdio_verbose = _resolve_stdio_verbose() + self._host_stderr_log_level = ( + logging.INFO if stdio_verbose and not logger.isEnabledFor(logging.DEBUG) else logging.DEBUG + ) + + self._init_options = self._build_init_options(env_path_prepend=list(env_path_prepend or [])) + + host_env = {**os.environ} + # Normalise to the literal the host's own gate parses, or remove the var + # outright. The host reads it from this inherited env, so forwarding an + # operator's raw value verbatim would let the two sides disagree, and an + # explicit opt-out has to actually erase an inherited truthy value. + if stdio_verbose: + host_env[_VERBOSE_ENV_VAR] = "1" + self._log.debug("Delegate host stdio tracing ON (%s): one log line per host frame", _VERBOSE_ENV_VAR) + else: + host_env.pop(_VERBOSE_ENV_VAR, None) + # Keep uip/npm registry auth reachable from the shells the Delegate + # runtime spawns (its interop overrides npm_config_prefix per command, + # which would otherwise orphan the global npmrc — see the helper). + if pinned := _maybe_pin_npm_globalconfig(host_env, plugin_tools_dir): + self._log.debug("Pinned NPM_CONFIG_GLOBALCONFIG=%s for the Delegate host", pinned) + # Keep the eval's gateway S2S secret out of the agent's own shell tools. + if stripped := _strip_gateway_creds(host_env): + self._log.debug("Stripped %s from the Delegate host env", stripped) + # Token freshness: the host owns applying it via DELEGATE_AUTH_TOKEN_FILE + # (re-read at init, at every turn, and before each TTL boundary). When an + # external refresher already maintains that file, nothing to do here. + # Otherwise — runs whose AUTH_TOKEN was minted from the very LLMGW pair + # stripped above (the rpa-eval pipeline) — the host's own S2S refresh is + # disabled BY the strip, so the adapter takes over: it re-mints + # adapter-side and publishes through a token file, keeping the secret out + # of the host env while restoring freshness. + # Gated against the adapter's own env, not host_env: the strip above just + # removed the LLMGW_* pair the gate needs to read. + if refresher := S2sTokenFileRefresher.maybe_create(os.environ, self._log): + try: + token_file = await refresher.start() + except Exception as error: + # Freshness is an enhancement over the inherited AUTH_TOKEN, so a + # refresher that cannot even start (a full or read-only tempdir, + # say) must not fail `Agent start` — that would turn runs which + # finish inside the ~1h TTL into ERRORs the orchestrator retries + # against the same broken condition. + self._log.warning( + "S2S token-file refresher unavailable (%s) — continuing with the inherited AUTH_TOKEN", error + ) + await refresher.stop() + else: + self._token_refresher = refresher + for name in TOKEN_FILE_ENV_VARS: + host_env[name] = token_file + self._log.info("S2S token-file refresher active — host token file: %s", token_file) + # Cached so a mid-turn respawn can rebuild an identical host without + # re-running start() (the env is stable across a task). + self._host_env = host_env + self._stall_timeout = _resolve_stall_timeout() + + await self._spawn_and_init() + + async def _spawn_and_init(self) -> None: + """Spawn the Node host subprocess, wire the drain tasks, and run the init handshake. + + The spawn tail shared by :meth:`start` (first launch) and + :meth:`_respawn_host` (mid-turn stall recovery). Reuses the cached + ``_stdio_bundle`` / ``_host_env`` / ``_init_options`` from ``start``. + A fresh ``_stdout_queue`` is installed first so a stale EOF sentinel + left by a previously-crashed host can't be read as this host's output. + + @raises AgentConfigError: init failed for an auth reason (non-retryable). + @raises RuntimeError: init failed for any other reason. + """ + assert self._stdio_bundle is not None, "_spawn_and_init requires a resolved bundle (call start first)" + assert self._host_env is not None, "_spawn_and_init requires a built host env (call start first)" + bundle_path = self._stdio_bundle + # Drop any EOF sentinel a prior (crashed) host's drain task queued. + self._stdout_queue = asyncio.Queue() + + self._process = await asyncio.create_subprocess_exec( + "node", + str(bundle_path), + # cwd is host-adjacent (not the eval sandbox): Node resolves + # @uipath/delegate-sdk relative to the bundle file, not cwd. The + # host itself chdir()s into options["workingDirectory"] during + # init, so tools still operate from the sandbox. + cwd=str(bundle_path.parent), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=self._host_env, + limit=_STREAM_READER_LIMIT_BYTES, + ) + assert self._process.stderr is not None + assert self._process.stdout is not None + self._stderr_task = asyncio.create_task(self._drain_stderr(self._process.stderr)) + self._stdout_task = asyncio.create_task(self._drain_stdout(self._process.stdout)) + + await self._send_command({"cmd": "init", "options": self._init_options}) + ack = await self._read_until(("init_ok", "error")) + if ack.get("type") == "error": + message = str(ack.get("message", "unknown error")) + if _is_auth_init_error(message): + # Auth failures never succeed on retry. Raise the non-retryable + # AgentConfigError (the categorizer routes it to + # AGENT_CONFIG_ERROR) so the run fails fast instead of burning + # ~40s on AGENT_API_ERROR's exponential backoff — mirroring the + # host-not-found path — and enrich it with the saved-login's + # expiry so "expired" is distinguishable from "absent". + raise AgentConfigError(self._format_auth_init_error(message)) + raise RuntimeError(f"Delegate SDK init failed: {message}") + + async def _respawn_host(self) -> None: + """SIGKILL the wedged host and start a fresh one for in-turn stall recovery. + + A host stuck on a backend round-trip cannot be unblocked, so it is + force-killed (a graceful ``destroy`` would hang too) and re-initialised + via :meth:`_spawn_and_init`. The delegate ``sessionId`` is left intact so + the resent prompt resumes the conversation on backends that persist + sessions. Propagates :meth:`_spawn_and_init`'s init failures. + """ + await self._force_kill_host() + await self._cancel_drain_tasks() + self._process = None + await self._spawn_and_init() + + async def communicate( + self, + user_input: str, + *, + stream_callback: StreamCallback | None = None, + timeout: float | None = None, + max_turns: int | None = None, + should_stop: Callable[[], bool] | None = None, + ) -> TurnRecord: + """Send one turn to the Delegate agent and return a :class:`TurnRecord`. + + Args: + user_input: The message/prompt to send. + stream_callback: Optional callback for real-time event streaming. + timeout: Hard wall-clock deadline in seconds. When exceeded, the + host subprocess is force-killed and :class:`TurnTimeoutError` + is raised; a ``crashed=True`` partial :class:`TurnRecord` is + stashed on ``self.pending_turn`` for the orchestrator to drain. + max_turns: Per-call cap on inner-loop turns, forwarded to the + host as ``maxSteps``. ``None`` defers to the SDK default. + should_stop: Accepted for ``Agent.communicate`` override + compatibility and ignored — the Delegate host drives its own + inner loop, so there is no between-message poll point here + (``supports_cooperative_stop`` stays False). + """ + if not self.working_directory: + raise RuntimeError("Agent not started. Call start() first.") + if self._process is None: + if self._respawn_before_retry and self._host_env is not None and self._stdio_bundle is not None: + # The previous attempt crashed on a backend session conflict + # and _crash() killed the wedged host (its in-memory + # currentSessionId would resume the wedged conversation on any + # resend). Respawn a fresh host so this AGENT_CRASH retry runs + # in a genuinely fresh conversation — the only attempt shape + # that can succeed (build 12685191). Init failures propagate + # typed (AgentConfigError for auth, RuntimeError otherwise). + self._respawn_before_retry = False + self._log.warning( + "Respawning a fresh Delegate host for this retry — the previous host was killed " + + "after a backend session conflict (its conversation is wedged behind a " + + "still-running generation)" + ) + await self._respawn_host() + else: + # The host died in a previous turn (the crash/timeout branches + # below drop the handle). AGENT_CRASH retries re-enter + # communicate() without re-running start() — the only place the + # host is spawned — so fail fast with the typed crash error (the + # orchestrator's drain/discard hook still fires) instead of + # writing into a broken pipe (mis-categorized as a retryable + # AGENT_API_ERROR) or blocking forever on the already-consumed + # EOF sentinel. + raise AgentCrashError( + "Delegate SDK host subprocess is not running — it crashed or timed out " + + "in a previous turn, and retries do not respawn it." + ) + + assert self.config.type is not None, "DelegateSdkAgent requires AgentConfig.type to be set before communicate()" + + # Reset the pending slot + bump the iteration counter (shared lifecycle). + self._begin_turn() + turn_start = time.monotonic() + deadline = turn_start + timeout if timeout is not None else None + + # Event emission: the agent is the SOLE emitter; events fan out to an + # internal EventCollector (which assembles the returned TurnRecord — the + # single, agent-agnostic capture path) and the caller's stream_callback. + task_id = self.config.type + collector = EventCollector() + # Rebuilds the per-generation AssistantMessage transcript from the + # host's event stream; the messages ride the terminal AgentEndEvent's + # finalization payload (the collector reads them back verbatim). + transcript = _TranscriptBuilder() + emit = CompositeStreamCallback([c for c in (collector, stream_callback) if c is not None]) + # The host runs an inner step loop, but the cross-agent TurnRecord + # treats one communicate() as one turn (assistant_turn_count carries the + # step count authoritatively), so a single turn_id covers the call. + turn_id = f"delegate-{self._iteration}" + + # All per-turn accumulators + context live on one mutable state object so + # the per-message-type handling can live in small methods (below) instead + # of one ~390-line god-method of nonlocal closures. model_used starts as + # the task-configured model so reports always have something to display; + # the host overwrites it from the ``result`` message. + st = _TurnState( + task_id=task_id, + turn_id=turn_id, + turn_start=turn_start, + user_input=user_input, + iteration=self._iteration, + emit=emit, + collector=collector, + transcript=transcript, + model_used=self.config.model, + ) + + # Everything from the opening events onward runs inside the try so that + # EVERY exit path — including a pre-loop send failure — flows through + # _finalize (terminal AgentEndEvent + pending_turn stash) per the Agent + # contract, mirroring ClaudeCodeAgent's finally-finalize. + try: + emit.on_event( + AgentStartEvent(task_id=task_id, prompt=user_input, iteration=self._iteration, model=st.model_used) + ) + emit.on_event(TurnStartEvent(task_id=task_id, turn_id=turn_id, model=st.model_used)) + + send_payload: dict[str, Any] = { + "cmd": "send", + "prompt": user_input, + "sessionId": self._session_id, + } + if max_turns is not None: + send_payload["maxSteps"] = max_turns + await self._send_command(send_payload) + + # First-response stall recovery (opt-in via DELEGATE_STALL_TIMEOUT_S). + # A wedged backend round-trip emits no events; without this it rides + # the non-retryable task/turn timeout to a score-0 loss. Bounded + # in-turn respawn+resend turns a transient stall into a recovered + # turn; a persistent one still falls through to the timeout. + resends_left = _DEFAULT_STALL_RESENDS if self._stall_timeout is not None else 0 + first_activity_seen = False + while True: + read_deadline, stall_capped = self._first_activity_read_deadline( + deadline, first_activity_seen=first_activity_seen, resends_left=resends_left + ) + try: + msg = await self._read_next_message(read_deadline) + except TimeoutError: + if not stall_capped: + raise # genuine turn deadline — handled by the outer timeout branch + resends_left -= 1 + self._log.warning( + "Delegate host produced no output within %.0fs of the prompt; respawning the " + + "host and resending (transient backend stall; %d resend(s) left)", + self._stall_timeout, + resends_left, + ) + await self._respawn_host() + await self._send_command(send_payload) + continue + + if msg is None: + # Host subprocess exited mid-turn (EOF sentinel from drain task). + # Drop the dead handle so a retried communicate() fails fast in + # the entry guard (retries never respawn the host). + self._crash(st, self._build_crash_message(), drop_process=True) + mtype = msg.get("type") + + if mtype == "event": + event = msg.get("event") or {} + if event.get("type") in _ACTIVITY_EVENT_TYPES: + first_activity_seen = True + self._handle_event(st, event) + elif mtype == "result": + first_activity_seen = True + self._handle_result(st, msg) + break + elif mtype == "error": + self._crash( + st, + f"Delegate SDK reported error during turn: {msg.get('message', 'unknown error')}", + from_host_error=True, + ) + else: + self._log.debug("Ignoring unknown host message: %r", msg) + + # (loop exits via ``break`` on the result message) + if st.error_message is not None: + # The host process is still alive after reporting a turn error, so + # the handle is kept — an AGENT_CRASH retry can legitimately reuse + # the live host (transient backend errors do recover on resend). + # Exception: a session-conflict error makes _crash kill the host, + # because the live host would resume the wedged conversation. + self._crash(st, f"Communication with Delegate SDK failed: {st.error_message}", from_host_error=True) + + # A clean turn returns the agent to WORKING — including after a prior + # attempt set ERROR and was retried (AGENT_CRASH / AGENT_API_ERROR are + # retryable). The orchestrator's discard_pending_turn() rolls back the + # iteration but does not touch _state, so without this reset a retried + # success would still report ERROR via get_state(). Mirrors + # ClaudeCodeAgent (_update_state_from_messages) and CodexAgent's + # success-path reset. + self._state = AgentState.WORKING + + if st.max_turns_exhausted: + self._log.warning( + "Delegate SDK halted after %d step(s) — max_turns cap reached before the agent completed", + st.assistant_turn_count, + ) + + # _finalize_turn emits the terminal TurnEnd + AgentEnd boundary (the + # finalization payload the collector reads back) and force-closes any + # tool calls that never received a result. + self._finalize_turn(st, AgentEndStatus.COMPLETED, crashed=False, crash_reason=None) + + except (AgentCrashError, TurnTimeoutError): + raise # already finalized at the raise site (via _crash / the timeout branch) + except TimeoutError: + # Wall-clock deadline hit. Kill the host, stash a partial crashed=True + # TurnRecord on pending_turn so the orchestrator can drain it, then raise + # TurnTimeoutError per the Agent contract. + assert timeout is not None # only enters this branch when deadline is set + await self._force_kill_host() + self._state = AgentState.ERROR + self._finalize_turn(st, AgentEndStatus.TIMEOUT, crashed=True, crash_reason=f"turn timeout after {timeout}s") + # Drop the killed handle so a retried communicate() fails fast in the + # entry guard (retries never respawn the host). + self._process = None + raise TurnTimeoutError(timeout, iteration=self._iteration) from None + except Exception as exc: + # Anything else mid-turn — e.g. the stdin pipe breaking during the + # pre-loop send. Without this wrap a bare BrokenPipeError would be + # mis-categorized as a retryable AGENT_API_ERROR whose retry never + # drains the partial or rolls the iteration back. + self._crash(st, f"Communication with Delegate SDK failed: {exc}", cause=exc) + finally: + # Terminal-event guarantee (Agent contract: AgentEndEvent on EVERY exit + # path). The handlers above finalize every Exception route, so this fires + # only on non-Exception exits (e.g. an external CancelledError) — still + # close the event tree before propagating. + if not st.finalized: + self._finalize_turn( + st, AgentEndStatus.CRASHED, crashed=True, crash_reason="turn aborted before completion" + ) + + # Turn completed cleanly — the iteration bump stands. + self._end_turn_ok() + + # The returned TurnRecord is the EventCollector's reduction of the emitted + # events — single, agent-agnostic capture path. + return collector.build_turn_record() + + # -- communicate() helpers (per-message-type handling) ------------------- + + async def _read_next_message(self, deadline: float | None) -> dict[str, Any] | None: + """Read one host message, honouring the wall-clock ``deadline``. + + Raises :class:`TimeoutError` if the deadline has already passed (or + ``wait_for`` trips it mid-read); returns ``None`` on the EOF sentinel. + """ + if deadline is None: + return await self._read_line() + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError + return await asyncio.wait_for(self._read_line(), timeout=remaining) + + def _first_activity_read_deadline( + self, turn_deadline: float | None, *, first_activity_seen: bool, resends_left: int + ) -> tuple[float | None, bool]: + """Deadline for the next host read, and whether it is the stall cap. + + While still awaiting the first agent activity of a turn (and a resend is + still budgeted), the wait is capped at ``_stall_timeout`` so a wedged + backend round-trip is caught quickly. Returns ``(deadline, stall_capped)`` + where ``stall_capped`` is True only when the returned deadline is the + stall cap and strictly tighter than the real turn deadline — the signal + that a :class:`TimeoutError` should trigger respawn+resend rather than a + turn timeout. When stall detection is off, activity has been seen, or the + resend budget is spent, the real ``turn_deadline`` is returned unchanged. + """ + stall = self._stall_timeout + if stall is None or first_activity_seen or resends_left <= 0: + return turn_deadline, False + stall_deadline = time.monotonic() + stall + if turn_deadline is not None and turn_deadline <= stall_deadline: + return turn_deadline, False + return stall_deadline, True + + def _handle_event(self, st: _TurnState, event: dict[str, Any]) -> None: + """Dispatch one host ``event`` message to the right per-type handler. + + ``session_start`` / ``done`` are informational and ignored. + """ + etype = event.get("type") + if etype in ("thinking", "message"): + self._handle_text_event(st, event) + elif etype == "tool_call": + self._handle_tool_call(st, event) + elif etype == "tool_result": + self._handle_tool_result(st, event) + elif etype == "error": + st.error_message = str(event.get("error") or "unknown error") + + def _handle_text_event(self, st: _TurnState, event: dict[str, Any]) -> None: + """Thinking / assistant-message text: feed the transcript + stream it out.""" + text = str(event.get("content") or "") + if event.get("type") == "message": + # The host tags new LLM round-trips with isStepStart=True; streaming-delta + # events for the same round-trip arrive with isStepStart=False. Older host + # builds don't emit the flag at all, so a missing field falls back to + # "count it as a turn" to preserve behaviour against an un-rebuilt host. + is_step_start = event.get("isStepStart") + if is_step_start is None or bool(is_step_start): + st.assistant_turn_count += 1 + if text: + st.transcript.on_message(text, is_step_start=bool(is_step_start)) + elif text: + st.transcript.on_thinking(text) + if text: + st.emit.on_event(TextChunkEvent(task_id=st.task_id, turn_id=st.turn_id, text=text)) + + def _handle_tool_call(self, st: _TurnState, event: dict[str, Any]) -> None: + """Open a tool call: record telemetry + emit ToolStart.""" + tool_name = str(event.get("toolName") or "") + tool_args = event.get("toolArgs") or {} + tool_id = str(event.get("toolId") or uuid.uuid4()) + telemetry = CommandTelemetry( + tool_name=tool_name, + tool_id=tool_id, + timestamp=datetime.now(), + parameters=tool_args if isinstance(tool_args, dict) else {"raw": tool_args}, + sequence_number=st.sequence_number, + result_status=None, + duration_ms=None, + ) + st.commands[tool_id] = {"telemetry": telemetry, "start_time": time.monotonic()} + st.transcript.on_tool_call(tool_id) + st.emit.on_event(ToolStartEvent(task_id=st.task_id, turn_id=st.turn_id, tool=telemetry)) + st.sequence_number += 1 + + def _handle_tool_result(self, st: _TurnState, event: dict[str, Any]) -> None: + """Close a tool call: resolve the pending telemetry + emit ToolEnd.""" + # Round-trip boundary marker: any generation activity after a tool result + # belongs to the next round-trip. + st.transcript.on_tool_result() + tool_name = str(event.get("toolName") or "") + tool_id = str(event.get("toolId") or "") + result_raw = event.get("toolResult") + is_error = (event.get("toolStatus") or "") == "failed" + matched_id = tool_id if tool_id in st.commands else self._find_pending_by_name(st.commands, tool_name) + if not matched_id: + self._log.debug("tool_result for unknown tool %s (id=%s); ignoring", tool_name, tool_id) + return + telem = st.commands[matched_id]["telemetry"] + telem.result_status = "error" if is_error else "success" + telem.duration_ms = (time.monotonic() - st.commands[matched_id]["start_time"]) * 1000 + result_text = str(result_raw) if result_raw is not None else "" + # Stored untruncated to match ClaudeCodeAgent and the documented invariant + # (CLAUDE.md): result_summary is kept whole so sub-agent returns are + # preserved. The 64 MiB StreamReader limit already bounds a pathological + # payload upstream. + telem.result_summary = result_text or None + if is_error: + telem.error_message = result_text + st.emit.on_event( + ToolEndEvent( + task_id=st.task_id, + turn_id=st.turn_id, + tool=telem, + status=ToolEndStatus.ERROR if is_error else ToolEndStatus.OK, + ) + ) + st.ended_tool_ids.add(matched_id) + + def _handle_result(self, st: _TurnState, msg: dict[str, Any]) -> None: + """Terminal ``result`` message: fold in the authoritative turn totals.""" + response = msg.get("response") + if isinstance(response, str) and response: + st.final_response = response + session_id = msg.get("sessionId") + if isinstance(session_id, str): + self._session_id = session_id + # Authoritative per-turn step count from the host — supersedes the running + # counter, which is only kept current so crash/timeout partials have a value. + step_count = msg.get("assistantStepCount") + if isinstance(step_count, int) and step_count >= 0: + st.assistant_turn_count = step_count + model_value = msg.get("model") + if isinstance(model_value, str) and model_value: + st.model_used = model_value + st.token_usage = self._parse_usage(msg.get("usage"), st.model_used) or st.token_usage + # Per-round-trip usage entries (sum == `usage`); zipped onto the transcript + # in _finalize_turn. Absent on older hosts, which fall back to token-less + # messages. + st.turn_usages = msg.get("turnUsages") + st.max_turns_exhausted = bool(msg.get("maxStepsReached")) + + def _crash( + self, + st: _TurnState, + reason: str, + *, + drop_process: bool = False, + cause: BaseException | None = None, + from_host_error: bool = False, + ) -> NoReturn: + """Mark ERROR, finalize a crashed=True partial, optionally drop the host + handle, and raise :class:`AgentCrashError`. + + Consolidates the four mid-turn crash sites (EOF sentinel, host ``error`` + message, post-loop ``error_message``, generic exception) so they can't + drift — e.g. one site forgetting ``truncate_crash_message`` or the ERROR + state set. ``drop_process`` nulls the (dead) handle so a retried + communicate() fails fast in the entry guard; ``cause`` chains the + original exception for the generic-exception site. + + A backend session-conflict crash (see :data:`_SESSION_CONFLICT_MARKER`) + additionally kills the host and drops ``_session_id``: the conversation + is wedged behind its own still-running generation, and the live host + would resume it even on a ``sessionId: null`` resend (the SDK falls + back to its in-memory ``currentSessionId``), so a fresh host — which + the entry guard respawns via ``_respawn_before_retry`` — is the only + attempt shape that can succeed. + + A Cloudflare WAF block (see :data:`_WAF_BLOCK_PAGE_MARKERS`) instead + gets its reason rewritten via :func:`_describe_waf_block`: the block is + deterministic per payload, so no retry shape can succeed — the rewrite + stamps the "content filter" signature that routes the failure to the + non-retryable ``AGENT_INVALID_OUTPUT`` category and explains the real + cause instead of the misleading country/auth wording. + + An SSE connect-watchdog failure (see :data:`_SSE_CONNECT_TIMEOUT_MARKER`) + gets the inverse treatment via :func:`_describe_sse_connect_timeout`: + the raw "timeout" wording would route it to the non-retryable + ``AGENT_TIMEOUT``, but it marks a transient backend availability window, + so the rewrite stamps the "connection" signature (→ retryable + ``AGENT_API_ERROR``) while keeping the live host for the resend. That + rewrite requires ``from_host_error``: unlike the HTML-page WAF markers, + the SSE fingerprint is a log-line-shaped string, so it also appears in + the 20-line host stderr tail that :meth:`_build_crash_message` embeds — + and a dead-host crash whose stderr merely *mentions* the watchdog (even + one the SDK recovered from) must keep its own reason. Rewriting it there + would drop the exit code, assert a live host that the same call nulls, + and flip a terminal crash into retries that the entry guard can only + fail. + """ + lowered = reason.lower() + if any(marker in lowered for marker in _WAF_BLOCK_PAGE_MARKERS): + reason = _describe_waf_block(reason) + self._log.warning(reason) + elif from_host_error and _SSE_CONNECT_TIMEOUT_MARKER in lowered and _SESSION_CONFLICT_MARKER not in lowered: + # Session-conflict takes precedence when a reason carries both + # fingerprints: only the fresh-host shape below can recover it. + reason = _describe_sse_connect_timeout(reason) + self._log.warning(reason) + elif _SESSION_CONFLICT_MARKER in lowered: + self._log.warning( + "Backend reports the conversation (session %s) is still generating a reply; killing the " + + "wedged host so a retry starts a fresh one — a live host would resume the same wedged " + + "conversation and re-conflict", + self._session_id or "<first turn — no id yet>", + ) + self._session_id = None + self.kill_sync() + drop_process = True + self._respawn_before_retry = True + self._state = AgentState.ERROR + self._finalize_turn(st, AgentEndStatus.CRASHED, crashed=True, crash_reason=truncate_crash_message(reason)) + if drop_process: + self._process = None + raise AgentCrashError(reason) from cause + + def _finalize_turn( + self, st: _TurnState, status: AgentEndStatus, *, crashed: bool, crash_reason: str | None + ) -> None: + """Emit the terminal TurnEnd + AgentEnd boundary exactly once. + + On crash/timeout it also stashes a crashed=True TurnRecord (the + EventCollector's reduction of the events seen so far) on + ``self.pending_turn`` for the orchestrator to drain. Best-effort: if the + reduction itself fails we log and leave ``pending_turn=None`` so the typed + exception's category still routes correctly. Iteration rollback happens in + ``discard_pending_turn()``, per the Agent contract. + """ + if st.finalized: + return + st.finalized = True + + # Force-close any tool calls that never received a result so they appear in + # the record (and the event tree stays balanced). + for tool_id, pending in st.commands.items(): + if tool_id in st.ended_tool_ids: + continue + telem = pending["telemetry"] + if telem.result_status is None: + telem.result_status = "unknown" + self._log.warning("Tool %s (id=%s) ended without result", telem.tool_name, tool_id) + if telem.duration_ms is None: + telem.duration_ms = 0.0 + st.emit.on_event( + ToolEndEvent(task_id=st.task_id, turn_id=st.turn_id, tool=telem, status=ToolEndStatus.UNRESOLVED) + ) + st.ended_tool_ids.add(tool_id) + + # AgentEndStatus and TurnEndStatus share identical members; map by value. + st.emit.on_event( + TurnEndEvent( + task_id=st.task_id, turn_id=st.turn_id, status=TurnEndStatus(status.value), tokens=st.token_usage + ) + ) + + st.emit.on_event( + AgentEndEvent( + task_id=st.task_id, + status=status, + usage=st.token_usage or TokenUsage(), + iteration=st.iteration, + user_input=st.user_input, + # Prefer the host's authoritative result.response; fall back to the + # transcript's merged final text so a delta-streamed answer (or a + # crash/timeout partial) isn't truncated to its last fragment. + agent_output=st.final_response or st.transcript.final_text(), + model_used=st.model_used, + assistant_turn_count=st.assistant_turn_count, + # Per-generation transcript reconstructed from the event stream, with + # token buckets zipped from the host's turnUsages (when present and + # aligned). The collector reads this back verbatim into + # TurnRecord.messages. On crash/timeout partials turn_usages is still + # None, so messages carry content and timing but no token attribution. + messages=st.transcript.build_messages( + turn_id=st.turn_id, model=st.model_used, turn_usages=st.turn_usages, log=self._log + ), + # num_turns is the cross-agent inner-loop turn count reports sum. The + # delegate analog is the host's authoritative assistantStepCount, + # already folded into assistant_turn_count. Left None on crash/timeout + # partials (no result message arrived), matching ClaudeCodeAgent. + num_turns=st.assistant_turn_count if not crashed else None, + max_turns_exhausted=st.max_turns_exhausted, + crashed=crashed, + crash_reason=crash_reason, + duration_seconds=time.monotonic() - st.turn_start, + ) + ) + + if crashed: + try: + self.pending_turn = st.collector.build_turn_record() + except Exception: + logger.exception("Failed to build partial turn record; continuing without partial") + self.pending_turn = None + + async def kill(self) -> None: + """Force-terminate the host subprocess. Fire-and-forget; safe at any time.""" + self.kill_sync() + + def kill_sync(self) -> None: + """Synchronously SIGKILL the host subprocess. + + Invoked by :class:`ThreadedWatchdog` from its timer thread on task-level + timeout, so this must not touch the event loop or await anything. + ``asyncio.subprocess.Process.kill()`` ultimately calls + ``os.kill``/``TerminateProcess``, which are thread-safe. + """ + proc = self._process + if proc is None or proc.returncode is not None: + return + with contextlib.suppress(ProcessLookupError, OSError): + proc.kill() + + async def _force_kill_host(self) -> None: + """Kill the host subprocess and await its exit. Idempotent.""" + self.kill_sync() + if self._process is None: + return + with contextlib.suppress(Exception): + await asyncio.wait_for(self._process.wait(), timeout=5.0) + + @staticmethod + def _parse_usage(raw: Any, model: str | None) -> TokenUsage | None: + """Map the host's ``usage`` payload onto :class:`TokenUsage`. + + The host emits keys in coder_eval's pre-split naming + (``input_tokens`` / ``output_tokens`` / ``cache_creation_input_tokens`` + / ``cache_read_input_tokens``) sourced from the Delegate framework's + per-turn ``lastTurnUsage``. The host's ``input_tokens`` has always + carried the fresh (uncached) prompt slice, so it maps onto + ``uncached_input_tokens``; ``TokenUsage.input_tokens`` is now the + derived total of all three prompt buckets and is never set directly. + The Delegate SDK doesn't expose pricing, so + ``total_cost_usd`` is computed locally from ``model`` via + :func:`coder_eval.pricing.calculate_cost`; the backend's + underscored id form is normalized to the table's hyphenated keys + first. It stays ``None`` when the model is unknown or absent from the + pricing table. Returns ``None`` + for missing, non-dict, or all-zero payloads so the caller can decide + whether to keep any prior value (e.g. cached from an earlier turn). + """ + if not isinstance(raw, dict): + return None + + def _int(value: Any) -> int: + return value if isinstance(value, int) and value >= 0 else 0 + + usage = TokenUsage( + uncached_input_tokens=_int(raw.get("input_tokens")), + output_tokens=_int(raw.get("output_tokens")), + cache_creation_input_tokens=_int(raw.get("cache_creation_input_tokens")), + cache_read_input_tokens=_int(raw.get("cache_read_input_tokens")), + ) + # An all-zero usage is indistinguishable from "framework didn't record + # any usage for this turn yet" — surface it as None so the caller can + # preserve a previous non-zero value rather than overwriting it. + if usage.is_empty(): + return None + if model: + # The backend echoes underscored ids (``claude_sonnet_4_6``) while + # the pricing table is keyed on the hyphenated form — normalize so + # a priced model doesn't silently report total_cost_usd=None. + usage.total_cost_usd = calculate_cost( + model.replace("_", "-"), + usage.uncached_input_tokens, + usage.output_tokens, + usage.cache_creation_input_tokens, + usage.cache_read_input_tokens, + ) + return usage + + async def _teardown_host(self) -> None: + """Terminate the host subprocess and its drain tasks, then null them out. + + Idempotent and state-neutral (does NOT mark the agent stopped), so it is + safe to call both from :meth:`stop` and at the top of :meth:`start` — the + orchestrator re-invokes ``start`` on a retryable init failure WITHOUT an + intervening ``stop``, so without this a retry would orphan the previous + Node subprocess and leak both drain tasks. + """ + try: + if self._process is not None and self._process.returncode is None: + try: + await self._send_command({"cmd": "destroy"}) + except Exception as e: + self._log.debug("Sending destroy to host failed: %s", e) + + try: + await asyncio.wait_for(self._process.wait(), timeout=_STOP_TIMEOUT_SEC) + except TimeoutError: + self._log.warning("Delegate SDK host did not exit within %.1fs — killing", _STOP_TIMEOUT_SEC) + # Same guard as _force_kill_host: the host can exit on its own + # in the window between the wait_for timing out and this kill, + # and killing an already-reaped process raises. + with contextlib.suppress(ProcessLookupError, OSError): + self._process.kill() + await self._process.wait() + finally: + # Unconditional: the refresher owns a background task and a tempdir + # holding a live bearer token, so a raise in the host-kill path above + # must not leak them for the rest of the process. + await self._cancel_drain_tasks() + self._process = None + # The refresher is per-start() (unlike _respawn_host, which keeps it so + # the replacement host reads the same fresh file); a start() retry + # builds a new one against the then-current env. + if self._token_refresher is not None: + refresher, self._token_refresher = self._token_refresher, None + await refresher.stop() + + async def _cancel_drain_tasks(self) -> None: + """Cancel and clear the stdout/stderr drain tasks. Idempotent.""" + for task_attr in ("_stderr_task", "_stdout_task"): + task = getattr(self, task_attr, None) + if task is not None: + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + setattr(self, task_attr, None) + + async def stop(self) -> None: + """Tear down the host subprocess cleanly.""" + await self._teardown_host() + self._mark_stopped() + + def get_environment_info(self) -> dict[str, Any]: + """Record the resolved Delegate routing so runs against different cloud + envs (alpha/staging/production) — or a pinned localhost backend — are + distinguishable in ``EvaluationResult.environment_info``. + + Only the *host* of any explicit ``BACKEND_URL``/``INTEROP_URL`` override + is recorded (never the full URL), mirroring :class:`CodexAgent`, so an + embedded credential can't leak into the run record. + """ + # Spread the base first so the ``system_prompt_semantics`` run marker is + # always present (dashboards read an absent marker as a pre-marker run). + info: dict[str, Any] = { + **super().get_environment_info(), + "delegate_env": os.environ.get("DELEGATE_SDK_ENV") or "alpha", + } + if self.config.model: + info["delegate_model"] = self.config.model + if backend := os.environ.get("BACKEND_URL"): + info["delegate_backend_url_host"] = urlparse(backend).hostname or "" + if interop := os.environ.get("INTEROP_URL"): + info["delegate_interop_url_host"] = urlparse(interop).hostname or "" + return info + + def get_sdk_options(self) -> dict[str, Any] | None: + """Return the options dict sent to the host on init (or ``None`` if not started).""" + return self._init_options + + # -- Internals ----------------------------------------------------------- + + def _build_init_options(self, *, env_path_prepend: list[str] | None = None) -> dict[str, Any]: + """Translate :class:`DelegateSdkAgentConfig` fields to Delegate SDK init options. + + ``env_path_prepend`` is start()'s sandbox mock_path_dirs (see there). + """ + options: dict[str, Any] = {} + if self.config.model: + options["model"] = self.config.model + + # Cloud env slug (alpha/staging/production) the host uses to compose + # the backend URL from the saved auth's org/tenant slugs — matches + # `npm start -- --env alpha` in basic.ts. Sourced from DELEGATE_SDK_ENV, + # defaulting to "alpha". Host precedence: backendUrl > BACKEND_URL env + # > env > localhost, so this is a no-op when an explicit backend URL is + # also configured. + options["env"] = os.environ.get("DELEGATE_SDK_ENV") or "alpha" + + # Task sandbox path. The host chdir()s into it and the SDK seeds the + # shell tools' per-session default cwd from it, so commands run in the + # sandbox without a per-call workingDirectory argument (and without a + # working-directory prompt prefix — the orchestrator already injects + # "Your working directory is: ..." into every prompt). + if self.working_directory is not None: + options["workingDirectory"] = str(self.working_directory) + + # Sandbox mock CLIs (SandboxConfig.mock_path_dirs, resolved by the + # orchestrator). Forwarded only when non-empty so hosts that predate the + # option — and the option-merge on the SDK side — see a clean config. + # The host validates each dir exists and the SDK injects the composed + # PATH into every shell command's environment. + if env_path_prepend: + options["shellPathPrepend"] = env_path_prepend + self._log.debug("Forwarding shellPathPrepend=%s to the Delegate host", env_path_prepend) + + # Local wiki routing under the sandbox (workingDirectory). Forward only when + # set: empty project_id keeps the per-session wiki dir; empty session_id lets + # the SDK generate a fresh session (a pinned id skips createSession). + if self.config.project_id: + options["projectId"] = self.config.project_id + if self.config.session_id: + options["sessionId"] = self.config.session_id + + # config.plugins is list[SdkPluginConfig] (a TypedDict); the helper takes + # list[dict[str, Any]] — identical at runtime, but list invariance trips + # the checker. Same pattern as ClaudeCodeAgent's process_plugins call. + bundled_skills = _resolve_bundled_skills_path(self.config.plugins) # type: ignore[arg-type] + if bundled_skills: + options["enableSkills"] = True + options["bundledSkillsPath"] = bundled_skills + else: + options["enableSkills"] = False + + # Reuse the cross-agent sdk_options surface to carry the reasoning-effort + # tier (low/medium/high/xhigh). The host maps this onto useAppStore.effort, + # which the ChatFramework already serialises as user_config.effort on every + # chat request; backend/llm/effort.py then translates per provider — for + # Kimi / Virtuoso (Fireworks) into the top-level reasoning_effort field + # (low maps to a hard-disable-thinking extra_body payload upstream). + # + # DelegateSdkAgentConfig.sdk_options is a permissive dict[str, Any]; the + # host only consumes `effort`. Other keys (Claude-only SDK options + # carried by shared experiment YAMLs) are silently ignored. + if (effort := self.config.sdk_options.get("effort")) is not None: + options["effort"] = effort + + # Mirror the DELEGATE_STDIO_VERBOSE forwarding in start(): when + # coder_eval runs at DEBUG (--verbose), turn on the SDK's own verbose so + # DelegateAgent logs its setup (loaded skills, tool count, interop URL). + if logger.isEnabledFor(logging.DEBUG): + options["verbose"] = True + + # Runtime endpoints (env vars win, defaults match the SDK's own host). + if backend := os.environ.get("BACKEND_URL"): + options["backendUrl"] = backend + if interop := os.environ.get("INTEROP_URL"): + options["interopUrl"] = interop + return options + + def _warn_unsupported_fields(self) -> None: + """Emit a single warning listing :class:`DelegateSdkAgentConfig` fields the SDK can't honour.""" + unsupported: list[str] = [] + for field_name in _UNSUPPORTED_FIELDS: + value = getattr(self.config, field_name, None) + if value: # falsy values (None, empty list/str) count as unset + unsupported.append(field_name) + if unsupported: + self._log.warning( + "DelegateSdkAgent does not support these AgentConfig fields — they will be ignored: %s", + ", ".join(unsupported), + ) + + def _format_auth_init_error(self, host_message: str) -> str: + """Build a fail-fast auth diagnostic from the host's init error. + + Distinguishes an *expired* saved login from an *absent* one and points at + the env-specific login command, turning a confusing 40s retry-then-fail + into a single actionable message. Raised as a non-retryable + :class:`AgentConfigError` by ``start()``. + """ + env = (self._init_options or {}).get("env") or os.environ.get("DELEGATE_SDK_ENV") or "alpha" + return ( + f"Delegate SDK authentication failed during init: {host_message} " + f"{_describe_saved_auth()} " + f"Fix it by setting AUTH_TOKEN/TENANT_ID/ORG_ID, or run " + f"`npx @uipath/delegate-cli login --env {env}` (writes ~/.aria/sdk-auth.json). " + f"This error is non-retryable — failing fast instead of retrying." + ) + + async def _send_command(self, cmd: dict[str, Any]) -> None: + """Write one JSON-Lines command to the host's stdin.""" + if self._process is None or self._process.stdin is None: + raise RuntimeError("Host subprocess has no stdin") + line = (json.dumps(cmd) + "\n").encode("utf-8") + self._process.stdin.write(line) + await self._process.stdin.drain() + + async def _read_line(self) -> dict[str, Any] | None: + """Return the next JSON protocol message from the stdout drain queue, + or ``None`` if the host has exited (EOF sentinel posted by + :meth:`_drain_stdout`). + + Callers must handle ``None``: ``communicate()`` stashes a partial + TurnRecord and raises :class:`AgentCrashError`; ``start()`` / + ``_read_until()`` raise the bare exception. + """ + return await self._stdout_queue.get() + + async def _drain_stdout(self, stdout: asyncio.StreamReader) -> None: + """Continuously read the host's stdout, logging non-JSON lines and queuing protocol messages.""" + try: + while True: + try: + raw = await stdout.readline() + except asyncio.LimitOverrunError as e: + # A single line exceeded _STREAM_READER_LIMIT_BYTES — the + # bytes are still buffered. Drain them so readline() can + # advance, log it, and treat the host as crashed (we + # can't reassemble the dropped JSON line, and silently + # continuing would leave communicate() blocked on the + # next result message). Posting the EOF sentinel routes + # this through the same AgentCrashError path as a real + # subprocess exit. + with contextlib.suppress(Exception): + await stdout.readexactly(e.consumed) + self._log.error( + "[delegate-sdk stdout] line exceeded %d-byte StreamReader limit; treating host as crashed", + _STREAM_READER_LIMIT_BYTES, + ) + self._stderr_lines.append( + f"[coder_eval] stdout line exceeded {_STREAM_READER_LIMIT_BYTES}-byte limit", + ) + raw = b"" # fall through to the EOF branch + if not raw: + # EOF — wake the queue consumer FIRST, then settle the process / + # stderr drain. Posting after a multi-second wait would block any + # in-flight ``_read_line()`` for up to 3s, which under timeout + # pressure can flip a clean ``AgentCrashError`` into a + # ``TurnTimeoutError`` with worse diagnostics. The crash-message + # builder reads ``self._stderr_lines`` directly, so a slightly + # stale stderr tail is fine — better than a slow sentinel. + await self._stdout_queue.put(None) # sentinel + if self._process is not None and self._process.returncode is None: + with contextlib.suppress(TimeoutError): + await asyncio.wait_for(self._process.wait(), timeout=2.0) + if self._stderr_task is not None and not self._stderr_task.done(): + with contextlib.suppress(TimeoutError, asyncio.CancelledError): + await asyncio.wait_for(self._stderr_task, timeout=1.0) + return + text = raw.decode("utf-8", errors="replace").rstrip() + try: + parsed = json.loads(text) + except json.JSONDecodeError: + self._log.debug("[delegate-sdk stdout] %s", text) + continue + if not isinstance(parsed, dict): + self._log.debug("[delegate-sdk stdout] non-object JSON skipped: %r", parsed) + continue + await self._stdout_queue.put(parsed) + except asyncio.CancelledError: + raise + except BaseException: + # Any other unexpected failure in the drain loop would leave + # communicate() blocked on _stdout_queue.get() forever. Post the + # EOF sentinel so the consumer fails fast with AgentCrashError + # instead of hanging until the threaded watchdog hard-kills the + # subprocess (the original 3000s silent-hang bug). + self._log.exception("[delegate-sdk] _drain_stdout crashed; posting EOF sentinel") + with contextlib.suppress(Exception): + await self._stdout_queue.put(None) + raise + + def _build_crash_message(self) -> str: + """Build a descriptive error message when the host process exits unexpectedly.""" + parts = ["Delegate SDK host crashed"] + if self._process is not None and self._process.returncode is not None: + parts.append(f"(exit code {self._process.returncode})") + if self._stderr_lines: + stderr_tail = "\n".join(list(self._stderr_lines)[-20:]) + parts.append(f"— stderr:\n{stderr_tail}") + else: + parts.append("— no stderr captured") + return " ".join(parts) + + async def _read_until(self, accepted_types: tuple[str, ...]) -> dict[str, Any]: + """Read host messages until one of the accepted types is seen. + + Raises :class:`AgentCrashError` if the host exits before producing + an accepted message. Used during ``start()`` where there is no + partial TurnRecord to stash. + """ + while True: + msg = await self._read_line() + if msg is None: + raise AgentCrashError(self._build_crash_message()) + if msg.get("type") in accepted_types: + return msg + self._log.debug("Ignoring host message while waiting for %s: %r", accepted_types, msg) + + @staticmethod + def _find_pending_by_name(commands: dict[str, dict[str, Any]], tool_name: str) -> str | None: + """Fall back to matching a tool result by name when no tool_id is supplied.""" + if not tool_name: + return None + for tool_id, pending in commands.items(): + if pending["telemetry"].tool_name == tool_name and pending["telemetry"].result_status is None: + return tool_id + return None + + async def _drain_stderr(self, stderr: asyncio.StreamReader) -> None: + """Forward the host's stderr to our logger and store for error reporting. + + Mirrors ``_drain_stdout``'s resilience: a single verbose ``writeLine`` + echo can now be large, so an over-limit line is drained and dropped with a + warning rather than letting ``LimitOverrunError`` kill this task. A dead + ``_stderr_task`` would strand the EOF await in ``_drain_stdout`` (which + joins it) and blank the crash-message stderr tail. + """ + try: + while True: + try: + line = await stderr.readline() + except asyncio.LimitOverrunError as e: + # Bytes are still buffered; drain them so readline() can + # advance, then drop the over-limit line and keep reading. + with contextlib.suppress(Exception): + await stderr.readexactly(e.consumed) + self._log.warning( + "[delegate-sdk stderr] line exceeded %d-byte StreamReader limit; dropped", + _STREAM_READER_LIMIT_BYTES, + ) + continue + if not line: + return + text = line.decode("utf-8", errors="replace").rstrip() + if text: + self._stderr_lines.append(text) + self._log.log(self._host_stderr_log_level, "[delegate-sdk] %s", text) + except asyncio.CancelledError: + raise + except Exception: + # Never let an unexpected drain failure kill the task silently — the + # stdout EOF path joins this task and the crash tail reads its buffer. + # Exception (not BaseException): KeyboardInterrupt/SystemExit propagate. + self._log.exception("[delegate-sdk] _drain_stderr crashed") + return diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py index 97d4f64c..408f7325 100644 --- a/src/coder_eval/models/__init__.py +++ b/src/coder_eval/models/__init__.py @@ -11,6 +11,7 @@ BaseAgentConfig, ClaudeCodeAgentConfig, CodexAgentConfig, + DelegateSdkAgentConfig, LocalPluginConfig, NoneAgentConfig, OpenCodeAgentConfig, @@ -224,6 +225,7 @@ "BaseAgentConfig", "ClaudeCodeAgentConfig", "CodexAgentConfig", + "DelegateSdkAgentConfig", "LocalPluginConfig", "NoneAgentConfig", "OpenCodeAgentConfig", diff --git a/src/coder_eval/models/agent_config.py b/src/coder_eval/models/agent_config.py index a22756f7..c77d60ba 100644 --- a/src/coder_eval/models/agent_config.py +++ b/src/coder_eval/models/agent_config.py @@ -381,6 +381,52 @@ class OpenCodeAgentConfig(BaseAgentConfig): ) +class DelegateSdkAgentConfig(BaseAgentConfig): + """Delegate SDK agent configuration (UiPath Autopilot's Delegate agent). + + Drives the Delegate agent through the published ``@uipath/delegate-stdio`` + Node host, a subprocess speaking JSON-Lines over stdio. The host package is + self-contained: it pulls ``@uipath/delegate-sdk`` in as its npm dependency. + Runtime prerequisites (host install + ``DELEGATE_STDIO_PATH`` / + ``DELEGATE_STDIO_NODE_MODULES``, auth) are documented in + ``coder_eval.agents.delegate_sdk_agent`` and ``docs/agents/DELEGATE_SDK.md``. + """ + + type: Literal[AgentKind.DELEGATE_SDK] # type: ignore[assignment] + + sdk_options: dict[str, Any] = Field( + default_factory=dict, + description=( + "Pass-through options for the delegate-stdio host. The host currently " + "consumes ``effort`` (low/medium/high/xhigh) and forwards it as " + "user_config.effort on every chat request. Other keys are accepted and " + "silently ignored so a shared experiment YAML can carry Claude-only " + "sdk_options keys (e.g. ``max_thinking_tokens``) without breaking the " + "delegate-sdk variant." + ), + ) + + project_id: str = Field( + default="", + description=( + "Bind SDK-created sessions to this project id for local wiki routing " + "(forwarded as the SDK ``projectId`` option). When set, the wiki lands at " + "``<workingDirectory>/projects/<project_id>/wiki`` instead of the per-session " + "``<workingDirectory>/sessions/<sessionId>/wiki``. Client-side routing key " + "only; empty keeps the session-scoped default." + ), + ) + session_id: str = Field( + default="", + description=( + "Pin the SDK session id (forwarded as the SDK ``sessionId`` option) so the " + "session-scoped wiki dir is deterministic. Empty lets the SDK generate a " + "fresh session per run. A pinned id skips ``createSession``, so it must be " + "one the backend accepts." + ), + ) + + class NoneAgentConfig(BaseAgentConfig): """No-op ("agentless") agent configuration. @@ -405,7 +451,12 @@ class NoneAgentConfig(BaseAgentConfig): # Only includes the concrete subclasses (not BaseAgentConfig) since the discriminator # must be a Literal type. BaseAgentConfig is returned by parse_agent_config when type=None. type AgentConfig = Annotated[ - ClaudeCodeAgentConfig | CodexAgentConfig | AntigravityAgentConfig | OpenCodeAgentConfig | NoneAgentConfig, + ClaudeCodeAgentConfig + | CodexAgentConfig + | AntigravityAgentConfig + | OpenCodeAgentConfig + | DelegateSdkAgentConfig + | NoneAgentConfig, Field(discriminator="type"), ] diff --git a/src/coder_eval/models/enums.py b/src/coder_eval/models/enums.py index 0cba3650..195b0e13 100644 --- a/src/coder_eval/models/enums.py +++ b/src/coder_eval/models/enums.py @@ -109,6 +109,7 @@ class AgentKind(StrEnum): CODEX = "codex" ANTIGRAVITY = "antigravity" OPENCODE = "opencode" + DELEGATE_SDK = "delegate-sdk" # UiPath Autopilot's Delegate agent, driven via the @uipath/delegate-stdio host. NONE = "none" # Agentless / system task — no coding agent runs; success criteria do all the work. UNKNOWN = "unknown" # Used when agent type cannot be determined (e.g., task loading failure) diff --git a/src/coder_eval/orchestration/overrides.py b/src/coder_eval/orchestration/overrides.py index de1f55d9..f39dfce6 100644 --- a/src/coder_eval/orchestration/overrides.py +++ b/src/coder_eval/orchestration/overrides.py @@ -72,6 +72,29 @@ def parse_override(raw: str) -> tuple[str, Any]: return path, parse_scalar(value) +def _kind_declares_sdk_options(kind: str | None) -> bool: + """True when ``kind`` is registered with a config class that has an ``sdk_options`` field. + + Looked up on the live registry (plugins included) rather than hardcoded, so + the layer-5 guard stays in step with whatever config classes actually + exist. An unregistered or missing kind is False — the resolver's own error + then reports it. + """ + from ..agents.registry import AgentRegistry + + if kind is None: + return False + registration = AgentRegistry.get(kind) + return registration is not None and "sdk_options" in registration.config_class.model_fields + + +def _kinds_declaring_sdk_options() -> list[str]: + """Sorted registered kinds whose config declares ``sdk_options`` (for the error hint).""" + from ..agents.registry import AgentRegistry + + return [kind for kind in AgentRegistry.list_kinds() if _kind_declares_sdk_options(kind)] + + def _assign_nested(patch: dict[str, Any], segments: list[str], value: Any) -> None: """Set ``patch[seg0][seg1]... = value``, creating intermediate dicts. @@ -132,15 +155,22 @@ def apply_overrides( if agent_patch: assert task.agent is not None, f"Task '{task.task_id}' has no agent config" - # Preserve the friendly "sdk_options only for claude-code" message before - # reconstruction, keyed on the type the agent is *becoming*. + # Preserve the friendly "sdk_options is not a field of this agent" message + # before reconstruction, keyed on the type the agent is *becoming*. The + # gate is REGISTRY-DRIVEN: a kind qualifies iff the config class it is + # registered with declares an ``sdk_options`` field (claude-code and + # delegate-sdk today, plus any plugin that models the same knob) — so a + # new harness never has to be hand-listed here, and a kind without the + # field still gets this hint instead of the resolver's generic typo error. if "sdk_options" in agent_patch: becoming = agent_patch.get("type", task.agent.type) type_value = becoming.value if isinstance(becoming, AgentKind) else becoming - if type_value != AgentKind.CLAUDE_CODE.value: + if not _kind_declares_sdk_options(type_value): where = "no agent type is set" if type_value is None else f"agent type {type_value}" raise OverrideError( - f"sdk_options cannot be used with {where}. This option is only supported for claude-code agents." + f"sdk_options cannot be used with {where}. This option is only supported for claude-code " + + "and other agents whose config declares an sdk_options field " + + f"(registered: {', '.join(_kinds_declaring_sdk_options())})." ) # Seed with only the explicitly-set fields (exclude_unset) so switching the # agent subclass via --type doesn't drag subclass-only defaults into a model diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py index 88af853a..3630e030 100644 --- a/src/coder_eval/pricing.py +++ b/src/coder_eval/pricing.py @@ -117,10 +117,41 @@ class ModelPricing: "moonshotai/kimi-k3": ModelPricing(3.0, 15.0, 3.0, 0.30), "z-ai/glm-5.2": ModelPricing(0.7168, 2.2528, 0.7168, 0.13312), "deepseek/deepseek-v4-pro": ModelPricing(0.435, 0.87, 0.435, 0.003625), + # UiPath Delegate SDK route (DelegateSdkAgent). The Delegate backend echoes + # UNDERSCORED model ids (``gpt_5_6_terra``) and the agent normalizes them to + # HYPHENATED keys (``gpt-5-6-terra``) before pricing — nothing maps them onto + # the dotted ids above (``gpt-5.6-terra``), so every delegate-routed model + # needs its own hyphenated row here or it silently reports total_cost_usd=None. + # Only UiPath-reachable routes (Delegate SDK / Fireworks / Vertex / Gateway) + # belong in this block. Rates were carried over VERBATIM from the + # coder_eval_uipath plugin that used to register them (behavior-preserving + # migration): note that several differ from the dotted rows for the same + # physical model — the plugin priced GPT-5.4/5.5 cache WRITES at the output + # rate where the rows above use the input rate, and prices no cache writes + # for Gemini — so a cross-harness cost comparison mixes rate tables until the + # two blocks are reconciled. + # Virtuoso 1.5 / 2.0 (Fireworks): implicit caching — no cache writes. + "virtuoso-1-5": ModelPricing(0.95, 4.0, 0.0, 0.16), + "virtuoso-2-0": ModelPricing(0.95, 4.0, 0.0, 0.19), + # Gemini 3.5 / 3.6 Flash + 3.1 Pro Preview (Vertex via Delegate): no cache writes. + "gemini-3-5-flash": ModelPricing(1.50, 9.0, 0.0, 0.15), + "gemini-3-6-flash": ModelPricing(1.50, 7.50, 0.0, 0.15), + "gemini-3-1-pro-preview": ModelPricing(2.0, 12.0, 0.0, 0.20), + # GPT-5.4 / 5.5 (OpenAI via the LLM Gateway): cache writes billed at the output rate. + "gpt-5-4": ModelPricing(2.50, 15.0, 15.0, 0.25), + "gpt-5-5": ModelPricing(5.0, 30.0, 30.0, 0.50), + # GPT-5.6 (Azure OpenAI via the LLM Gateway): sol flagship / terra balanced / + # luna economy. From 5.6 on, cache writes bill at 1.25x input and cached reads + # at 10% of input; post-2026-07-30 terra/luna cut, like the dotted rows. + "gpt-5-6-sol": ModelPricing(5.0, 30.0, 6.25, 0.50), + "gpt-5-6-terra": ModelPricing(2.0, 12.0, 2.50, 0.20), + "gpt-5-6-luna": ModelPricing(0.20, 1.20, 0.25, 0.02), + # Kimi K2.7 Code (Fireworks, gateway-routed only): implicit caching — no cache writes. + "kimi-k2-7-code": ModelPricing(0.95, 4.0, 0.0, 0.19), } -# Plugin-contributed rates (e.g. coder_eval_uipath registers UiPath models). +# Plugin-contributed rates (a third-party plugin registering its own models). # Merged over the built-in table at lookup time. _REGISTERED_PRICING: dict[str, ModelPricing] = {} diff --git a/tasks/delegate_sdk_smoke_test.yaml b/tasks/delegate_sdk_smoke_test.yaml new file mode 100644 index 00000000..bac9672f --- /dev/null +++ b/tasks/delegate_sdk_smoke_test.yaml @@ -0,0 +1,42 @@ +task_id: "delegate_sdk_smoke_test" +description: "Smoke-test the UiPath Delegate SDK agent harness: create and run a small Python script." +initial_prompt: "Create a Python file named app.py in the current working directory that prints 'Hello, Delegate!' on one line, and today's date in YYYY-MM-DD format on the next line. Use the datetime module. Then run the script with: python app.py" +# No `smoke-pass`: that tag routes a task into the CI E2E bucket, which runs on +# Bedrock runners with no @uipath/delegate-stdio host and no UiPath auth. Run +# this task locally (or in a job that installs both) instead — see +# docs/agents/DELEGATE_SDK.md. +tags: [smoke, basic, pure-python, delegate-sdk] + +run_limits: + expected_turns: 5 + # The host drives a full agent loop per turn against the UiPath backend; give + # it room but keep the smoke bounded so a stalled backend fails fast. + task_timeout: 600 + +agent: + type: "delegate-sdk" + permission_mode: "acceptEdits" + # Cloud env slug (alpha/staging/production) is not a task field — it comes + # from the DELEGATE_SDK_ENV environment variable (defaults to "alpha"). + # Pin a UiPath-native model: the inherited default (claude-sonnet-4-6 from + # experiments/default.yaml) is rejected by the deployed alpha Delegate backend + # ("Model 'claude_sonnet_4_6' is not available"). virtuoso-1-5 is the Delegate + # SDK default and is priced in src/coder_eval/pricing.py. + model: "virtuoso-1-5" + +sandbox: + driver: "tempdir" + python: {} + +success_criteria: + - type: "file_exists" + path: "app.py" + description: "The file app.py must be created." + - type: "file_contains" + path: "app.py" + includes: ["Hello, Delegate!", "datetime"] + description: "The script must contain the required string and import." + - type: "run_command" + command: "python app.py" + timeout: 10 + description: "The script must execute successfully." diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 64508fe7..85867518 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -2087,7 +2087,9 @@ def test_drift_is_detected(self, tmp_path: Path): write(tmp_path) target = tmp_path / "plugins/coder-eval/reference/criteria.md" - target.write_text(target.read_text(encoding="utf-8").replace("### `file_exists`", "### `tampered`")) + target.write_text( + target.read_text(encoding="utf-8").replace("### `file_exists`", "### `tampered`"), encoding="utf-8" + ) assert str(target) in check(tmp_path) diff --git a/tests/test_delegate_s2s_token_file.py b/tests/test_delegate_s2s_token_file.py new file mode 100644 index 00000000..6c042506 --- /dev/null +++ b/tests/test_delegate_s2s_token_file.py @@ -0,0 +1,455 @@ +"""Tests for :mod:`coder_eval.agents._delegate_s2s_token_file`. + +Mint calls are monkeypatched — no real IdP round-trips. +""" + +from __future__ import annotations + +import asyncio +import base64 +import http.client +import json +import logging +import ssl +import time +from pathlib import Path +from typing import Any + +import pytest + +from coder_eval.agents import _delegate_s2s_token_file +from coder_eval.agents._delegate_s2s_token_file import ( + S2sTokenFileRefresher, + _read_creds, + _S2sCreds, + decode_jwt_claims, +) + + +_LOG = logging.LoggerAdapter(logging.getLogger("test"), {}) + + +def _fake_jwt(claims: dict[str, Any]) -> str: + payload = base64.urlsafe_b64encode(json.dumps(claims).encode()).decode().rstrip("=") + return f"eyJhbGciOiJub25lIn0.{payload}.sig" + + +def _refresher_env(**overrides: str) -> dict[str, str]: + env = { + "LLMGW_CLIENT_ID": "eval-client", + "LLMGW_CLIENT_SECRET": "s3cret", + "LLMGW_URL": "https://alpha.uipath.com", + "AUTH_TOKEN": _fake_jwt({"client_id": "eval-client", "exp": int(time.time()) + 3600}), + } + env.update(overrides) + return {k: v for k, v in env.items() if v} + + +def _maybe_create(env: dict[str, str] | None = None) -> S2sTokenFileRefresher | None: + """``maybe_create`` against the activating env by default.""" + return S2sTokenFileRefresher.maybe_create(_refresher_env() if env is None else env, _LOG) + + +# ---- decode_jwt_claims ------------------------------------------------------ + + +class TestDecodeJwtClaims: + def test_decodes_payload_claims(self) -> None: + token = _fake_jwt({"client_id": "abc", "exp": 123}) + + assert decode_jwt_claims(token) == {"client_id": "abc", "exp": 123} + + @pytest.mark.parametrize("token", ["", "not-a-jwt", "a.b", "a.!!!.c", "a." + "b" * 10 + ".c"]) + def test_malformed_tokens_return_none(self, token: str) -> None: + assert decode_jwt_claims(token) is None + + +# ---- _read_creds ------------------------------------------------------------ + + +class TestReadCreds: + def test_resolves_token_url_at_origin(self) -> None: + creds = _read_creds(_refresher_env()) + + assert creds is not None + assert creds.token_url == "https://alpha.uipath.com/identity_/connect/token" + + def test_gateway_path_suffix_is_discarded(self) -> None: + creds = _read_creds(_refresher_env(LLMGW_URL="https://alpha.uipath.com/llmgw")) + + assert creds is not None + assert creds.token_url == "https://alpha.uipath.com/identity_/connect/token" + + @pytest.mark.parametrize("missing", ["LLMGW_CLIENT_ID", "LLMGW_CLIENT_SECRET", "LLMGW_URL"]) + def test_incomplete_triple_returns_none(self, missing: str) -> None: + assert _read_creds(_refresher_env(**{missing: ""})) is None + + def test_invalid_url_returns_none(self) -> None: + assert _read_creds(_refresher_env(LLMGW_URL="not a url")) is None + + @pytest.mark.parametrize("url", ["http://alpha.uipath.com", "file:///etc/passwd", "ftp://alpha.uipath.com"]) + def test_non_https_scheme_returns_none(self, url: str) -> None: + """The mint body carries LLMGW_CLIENT_SECRET — never over a non-TLS scheme.""" + assert _read_creds(_refresher_env(LLMGW_URL=url)) is None + + +# ---- _mint_s2s_token --------------------------------------------------------- + + +class TestMintS2sToken: + @staticmethod + def _creds() -> _S2sCreds: + creds = _read_creds(_refresher_env()) + assert creds is not None + return creds + + def test_request_carries_a_real_user_agent(self, monkeypatch: pytest.MonkeyPatch) -> None: + """UiPath's Cloudflare WAF 403s urllib's default UA (error code 1010).""" + seen: dict[str, Any] = {} + + class _FakeResponse: + def read(self) -> bytes: + return json.dumps({"access_token": "tok"}).encode() + + def __enter__(self) -> _FakeResponse: + return self + + def __exit__(self, *args: Any) -> None: + return None + + def _fake_urlopen(request: Any, timeout: float) -> _FakeResponse: + seen["user_agent"] = request.get_header("User-agent") + return _FakeResponse() + + monkeypatch.setattr(_delegate_s2s_token_file.urllib.request, "urlopen", _fake_urlopen) + + assert _delegate_s2s_token_file._mint_s2s_token(self._creds()) == "tok" + assert seen["user_agent"] + assert "python-urllib" not in seen["user_agent"].lower() + + def test_http_error_surfaces_response_body(self, monkeypatch: pytest.MonkeyPatch) -> None: + import io + import urllib.error + + def _fake_urlopen(request: Any, timeout: float) -> Any: + raise urllib.error.HTTPError( + request.full_url, 403, "Forbidden", hdrs=None, fp=io.BytesIO(b"error code: 1010\n") + ) + + monkeypatch.setattr(_delegate_s2s_token_file.urllib.request, "urlopen", _fake_urlopen) + + with pytest.raises(RuntimeError, match=r"HTTP Error 403.*error code: 1010"): + _delegate_s2s_token_file._mint_s2s_token(self._creds()) + + @pytest.mark.parametrize( + "raised", + [ + pytest.param(http.client.RemoteDisconnected("closed early"), id="remote-disconnected"), + pytest.param(http.client.IncompleteRead(b"half"), id="incomplete-read"), + pytest.param(ssl.SSLError("handshake"), id="ssl-error"), + ], + ) + def test_transport_failures_are_normalized_to_runtimeerror( + self, monkeypatch: pytest.MonkeyPatch, raised: Exception + ) -> None: + """These escape urllib un-wrapped (not URLError), so the contract must still hold.""" + + def _fake_urlopen(request: Any, timeout: float) -> Any: + raise raised + + monkeypatch.setattr(_delegate_s2s_token_file.urllib.request, "urlopen", _fake_urlopen) + + with pytest.raises(RuntimeError, match="rejected client_credentials"): + _delegate_s2s_token_file._mint_s2s_token(self._creds()) + + def test_missing_access_token_raises(self, monkeypatch: pytest.MonkeyPatch) -> None: + class _FakeResponse: + def read(self) -> bytes: + return json.dumps({"token_type": "Bearer"}).encode() + + def __enter__(self) -> _FakeResponse: + return self + + def __exit__(self, *args: Any) -> None: + return None + + monkeypatch.setattr( + _delegate_s2s_token_file.urllib.request, "urlopen", lambda request, timeout: _FakeResponse() + ) + + with pytest.raises(RuntimeError, match="no access_token"): + _delegate_s2s_token_file._mint_s2s_token(self._creds()) + + +# ---- S2sTokenFileRefresher.maybe_create ------------------------------------- + + +class TestMaybeCreate: + def test_creates_when_auth_token_minted_by_same_client(self) -> None: + assert _maybe_create() is not None + + def test_none_without_gateway_creds(self) -> None: + assert _maybe_create(_refresher_env(LLMGW_CLIENT_SECRET="")) is None + + @pytest.mark.parametrize("name", ["DELEGATE_AUTH_TOKEN_FILE", "AUTH_TOKEN_FILE"]) + def test_none_when_external_token_file_configured(self, name: str) -> None: + env = _refresher_env(**{name: "/some/token/file"}) + + assert _maybe_create(env) is None + + def test_none_without_auth_token(self) -> None: + assert _maybe_create(_refresher_env(AUTH_TOKEN="")) is None + + def test_none_when_auth_token_from_other_client(self) -> None: + env = _refresher_env(AUTH_TOKEN=_fake_jwt({"client_id": "someone-else"})) + + assert _maybe_create(env) is None + + def test_none_when_auth_token_is_opaque(self) -> None: + env = _refresher_env(AUTH_TOKEN="opaque-token-with-no-claims") + + assert _maybe_create(env) is None + + +# ---- start / stop lifecycle -------------------------------------------------- + + +class TestStartStop: + @pytest.mark.asyncio + async def test_start_writes_minted_token_and_stop_removes_it(self, monkeypatch: pytest.MonkeyPatch) -> None: + minted = _fake_jwt({"client_id": "eval-client", "exp": int(time.time()) + 3600}) + monkeypatch.setattr(_delegate_s2s_token_file, "_mint_s2s_token", lambda creds: minted) + refresher = _maybe_create() + assert refresher is not None + + path = await refresher.start() + + try: + assert await asyncio.to_thread(Path(path).read_text, encoding="utf-8") == minted + finally: + await refresher.stop() + assert not Path(path).exists() + + @pytest.mark.asyncio + async def test_initial_mint_failure_seeds_inherited_token(self, monkeypatch: pytest.MonkeyPatch) -> None: + def _boom(creds: Any) -> str: + raise RuntimeError("IdP down") + + monkeypatch.setattr(_delegate_s2s_token_file, "_mint_s2s_token", _boom) + env = _refresher_env() + refresher = _maybe_create(env) + assert refresher is not None + + path = await refresher.start() + + try: + assert await asyncio.to_thread(Path(path).read_text, encoding="utf-8") == env["AUTH_TOKEN"] + finally: + await refresher.stop() + + @pytest.mark.asyncio + async def test_initial_mint_transport_failure_does_not_propagate(self, monkeypatch: pytest.MonkeyPatch) -> None: + """start() is best-effort for non-RuntimeError failures too.""" + + def _boom(creds: Any) -> str: + raise http.client.RemoteDisconnected("closed early") + + monkeypatch.setattr(_delegate_s2s_token_file, "_mint_s2s_token", _boom) + env = _refresher_env() + refresher = _maybe_create(env) + assert refresher is not None + + path = await refresher.start() + + try: + assert await asyncio.to_thread(Path(path).read_text, encoding="utf-8") == env["AUTH_TOKEN"] + finally: + await refresher.stop() + + @pytest.mark.asyncio + async def test_refresh_loop_rewrites_file_with_new_mint(self, monkeypatch: pytest.MonkeyPatch) -> None: + # An already-inside-the-lead-window exp forces the loop's first sleep to + # the 60s floor; shrinking the floor makes the rewrite observable fast. + first = _fake_jwt({"client_id": "eval-client", "exp": int(time.time())}) + second = _fake_jwt({"client_id": "eval-client", "exp": int(time.time()) + 7200}) + mints = iter([first, second]) + monkeypatch.setattr(_delegate_s2s_token_file, "_mint_s2s_token", lambda creds: next(mints)) + monkeypatch.setattr(_delegate_s2s_token_file, "_MIN_DELAY_SECONDS", 0.01) + refresher = _maybe_create() + assert refresher is not None + + path = await refresher.start() + + try: + await _await_token(path, second) + finally: + await refresher.stop() + + def test_token_file_before_start_raises(self) -> None: + refresher = _maybe_create() + assert refresher is not None + + with pytest.raises(RuntimeError, match="start"): + _ = refresher.token_file + + def test_write_token_before_start_raises(self) -> None: + refresher = _maybe_create() + assert refresher is not None + + with pytest.raises(RuntimeError, match="start"): + refresher._write_token("tok") + + @pytest.mark.asyncio + async def test_stop_is_idempotent(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(_delegate_s2s_token_file, "_mint_s2s_token", lambda creds: "tok") + refresher = _maybe_create() + assert refresher is not None + path = await refresher.start() + + await refresher.stop() + await refresher.stop() + + assert not Path(path).exists() + + +# ---- refresh-loop resilience ------------------------------------------------- + + +class TestRefreshLoopResilience: + """A loop that dies silently resurrects the very 401 this module prevents.""" + + @staticmethod + def _fast(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(_delegate_s2s_token_file, "_MIN_DELAY_SECONDS", 0.01) + monkeypatch.setattr(_delegate_s2s_token_file, "_RETRY_SECONDS", 0.01) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "error", + [ + pytest.param(RuntimeError("IdP down"), id="idp-refusal"), + pytest.param(http.client.RemoteDisconnected("closed early"), id="non-runtimeerror-transport"), + ], + ) + async def test_loop_survives_a_mint_failure_and_lands_the_next_token( + self, monkeypatch: pytest.MonkeyPatch, error: Exception + ) -> None: + # exp already inside the refresh lead window ⇒ the loop's first sleep is + # the (shrunk) floor, so both the failure and the recovery are observable. + first = _fake_jwt({"client_id": "eval-client", "exp": int(time.time())}) + third = _fake_jwt({"client_id": "eval-client", "exp": int(time.time()) + 7200}) + outcomes: list[Any] = [first, error, third] + + def _mint(creds: Any) -> str: + outcome = outcomes.pop(0) + if isinstance(outcome, Exception): + raise outcome + return outcome + + self._fast(monkeypatch) + monkeypatch.setattr(_delegate_s2s_token_file, "_mint_s2s_token", _mint) + refresher = _maybe_create() + assert refresher is not None + + path = await refresher.start() + + try: + # The failed mint must leave the previous (stale but still valid) + # token in place rather than clobbering the file. + assert await asyncio.to_thread(Path(path).read_text, encoding="utf-8") == first + await _await_token(path, third) + assert refresher._task is not None and not refresher._task.done() + finally: + await refresher.stop() + + @pytest.mark.asyncio + async def test_loop_survives_a_write_failure(self, monkeypatch: pytest.MonkeyPatch) -> None: + first = _fake_jwt({"client_id": "eval-client", "exp": int(time.time())}) + second = _fake_jwt({"client_id": "eval-client", "exp": int(time.time()) + 7200}) + mints = iter([first, second, second]) + self._fast(monkeypatch) + monkeypatch.setattr(_delegate_s2s_token_file, "_mint_s2s_token", lambda creds: next(mints)) + refresher = _maybe_create() + assert refresher is not None + path = await refresher.start() + real_write = refresher._write_token + writes = {"n": 0} + + def _flaky_write(token: str) -> None: + writes["n"] += 1 + if writes["n"] == 2: # the loop's first write, after start()'s + raise OSError("no space left on device") + real_write(token) + + monkeypatch.setattr(refresher, "_write_token", _flaky_write) + + try: + await _await_token(path, second) + assert refresher._task is not None and not refresher._task.done() + finally: + await refresher.stop() + + @pytest.mark.asyncio + async def test_stop_logs_a_loop_that_died( + self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + ) -> None: + monkeypatch.setattr(_delegate_s2s_token_file, "_mint_s2s_token", lambda creds: "tok") + refresher = _maybe_create() + assert refresher is not None + await refresher.start() + + async def _die() -> None: + raise ValueError("loop blew up") + + assert refresher._task is not None + refresher._task.cancel() + with pytest.raises(asyncio.CancelledError): + await refresher._task + refresher._task = asyncio.create_task(_die()) + await asyncio.sleep(0) + + with caplog.at_level(logging.ERROR, logger="test"): + await refresher.stop() + + assert "refresh loop had died" in caplog.text + + +# ---- _delay_until_refresh ---------------------------------------------------- + + +class TestDelayUntilRefresh: + def test_schedules_ahead_of_expiry(self) -> None: + refresher = _maybe_create() + assert refresher is not None + token = _fake_jwt({"exp": int(time.time()) + 3600}) + + delay = refresher._delay_until_refresh(token) + + assert ( + _delegate_s2s_token_file._MIN_DELAY_SECONDS + <= delay + <= 3600 - _delegate_s2s_token_file._REFRESH_LEAD_SECONDS + ) + + def test_floors_the_delay_for_an_already_expiring_token(self) -> None: + refresher = _maybe_create() + assert refresher is not None + + assert refresher._delay_until_refresh(_fake_jwt({"exp": 1})) == _delegate_s2s_token_file._MIN_DELAY_SECONDS + + @pytest.mark.parametrize("token", ["opaque-token", _fake_jwt({"client_id": "x"})]) + def test_falls_back_to_a_fixed_interval_without_exp(self, token: str) -> None: + refresher = _maybe_create() + assert refresher is not None + + assert refresher._delay_until_refresh(token) == _delegate_s2s_token_file._FALLBACK_INTERVAL_SECONDS + + +async def _await_token(path: str, expected: str, timeout: float = 5.0) -> None: + """Poll the token file until it holds ``expected``, or fail the test.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if await asyncio.to_thread(Path(path).read_text, encoding="utf-8") == expected: + return + await asyncio.sleep(0.02) + pytest.fail(f"token file never reached the expected value within {timeout}s") diff --git a/tests/test_delegate_sdk_agent.py b/tests/test_delegate_sdk_agent.py new file mode 100644 index 00000000..cd9735f0 --- /dev/null +++ b/tests/test_delegate_sdk_agent.py @@ -0,0 +1,2885 @@ +"""Tests for :mod:`coder_eval.agents.delegate_sdk_agent`. + +Subprocess-level tests use an in-memory fake that mimics the host's JSON-lines +protocol — no real Node process is spawned. +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import logging +import time +from collections.abc import Callable, Iterator +from datetime import datetime +from pathlib import Path +from typing import Any + +import pytest + +from coder_eval.agents import _delegate_s2s_token_file, delegate_sdk_agent +from coder_eval.agents.delegate_sdk_agent import ( + DelegateSdkAgent, + _describe_saved_auth, + _is_auth_init_error, + _maybe_pin_npm_globalconfig, + _resolve_bundled_skills_path, + _resolve_stall_timeout, + _resolve_stdio_bundle, + _resolve_stdio_verbose, + _strip_gateway_creds, +) +from coder_eval.errors import AgentConfigError, AgentCrashError, TurnTimeoutError +from coder_eval.errors.categories import ErrorCategory +from coder_eval.errors.categorization import categorize_error +from coder_eval.models import ( + AgentState, + AssistantMessage, + DelegateSdkAgentConfig, + DirectRoute, + ReconciliationMessage, + parse_agent_config, +) +from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.events import ( + AgentEndEvent, + AgentStartEvent, + StreamEvent, + TextChunkEvent, + ToolEndEvent, + ToolEndStatus, + ToolStartEvent, +) + + +# ---- helpers --------------------------------------------------------------- + + +def _make_config(**overrides: Any) -> DelegateSdkAgentConfig: + """Build a DelegateSdkAgentConfig for delegate-sdk with any field overrides.""" + defaults: dict[str, Any] = { + "type": "delegate-sdk", + "permission_mode": "default", # AgentConfig requires a value + } + defaults.update(overrides) + cfg = parse_agent_config(**defaults) + assert isinstance(cfg, DelegateSdkAgentConfig) + return cfg + + +class _CapturingCallback: + """Simple StreamCallback that records every event for later assertion.""" + + def __init__(self) -> None: + self.events: list[StreamEvent] = [] + + def on_event(self, event: StreamEvent) -> None: + self.events.append(event) + + +# ---- _resolve_bundled_skills_path ----------------------------------------- + + +class TestResolveBundledSkillsPath: + def test_empty_plugins_returns_none(self) -> None: + assert _resolve_bundled_skills_path(None) is None + assert _resolve_bundled_skills_path([]) is None + + def test_single_plugin_appends_skills_suffix(self, tmp_path: Path) -> None: + plugin_root = tmp_path / "my-plugin" + plugin_root.mkdir() + result = _resolve_bundled_skills_path([{"type": "local", "path": str(plugin_root)}]) + assert result is not None + assert Path(result) == (plugin_root / "skills").resolve() + + def test_expands_env_var_in_path(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + plugin_root = tmp_path / "envvar-plugin" + plugin_root.mkdir() + monkeypatch.setenv("DELEGATE_TEST_PLUGIN_DIR", str(plugin_root)) + result = _resolve_bundled_skills_path([{"type": "local", "path": "$DELEGATE_TEST_PLUGIN_DIR"}]) + assert result is not None + assert Path(result) == (plugin_root / "skills").resolve() + + def test_multiple_plugins_first_wins_with_warning(self, tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + first = tmp_path / "first" + second = tmp_path / "second" + first.mkdir() + second.mkdir() + with caplog.at_level(logging.WARNING, logger="coder_eval.agents.delegate_sdk_agent"): + result = _resolve_bundled_skills_path( + [ + {"type": "local", "path": str(first)}, + {"type": "local", "path": str(second)}, + ] + ) + assert result is not None + assert Path(result) == (first / "skills").resolve() + assert any("only one plugin" in rec.message for rec in caplog.records) + + def test_plugin_without_path_returns_none(self, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger="coder_eval.agents.delegate_sdk_agent"): + result = _resolve_bundled_skills_path([{"type": "local"}]) + assert result is None + assert any("missing 'path'" in rec.message for rec in caplog.records) + + +# ---- host bundle resolution ---------------------------------------------- + + +def _make_bundle(install_root: Path) -> Path: + """Create a stub host bundle under ``install_root/node_modules/@uipath/...``.""" + bundle = install_root / "node_modules" / "@uipath" / "delegate-stdio" / "dist" / "delegate_stdio.mjs" + bundle.parent.mkdir(parents=True, exist_ok=True) + bundle.write_text("// stub", encoding="utf-8") + return bundle + + +class TestStdioBundleResolution: + """Pin the host discovery contract: DELEGATE_STDIO_PATH wins; then an + explicit DELEGATE_STDIO_NODE_MODULES is probed exactly; otherwise the cwd's + ancestors (and ``~``) are walked the way Node resolves modules. AgentConfigError + (RuntimeError subclass) is typed so the categorizer routes it to the + non-retryable AGENT_CONFIG_ERROR category by isinstance. + """ + + def test_missing_host_raises(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("DELEGATE_STDIO_PATH", raising=False) + monkeypatch.setenv("DELEGATE_STDIO_NODE_MODULES", str(tmp_path)) + with pytest.raises(AgentConfigError, match="not found"): + _resolve_stdio_bundle() + + def test_explicit_path_not_a_file_raises(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("DELEGATE_STDIO_PATH", str(tmp_path / "nope.mjs")) + with pytest.raises(AgentConfigError, match="does not point to a file"): + _resolve_stdio_bundle() + + def test_explicit_path_resolves(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + bundle = tmp_path / "delegate_stdio.mjs" + bundle.write_text("// stub", encoding="utf-8") + monkeypatch.setenv("DELEGATE_STDIO_PATH", str(bundle)) + assert _resolve_stdio_bundle() == bundle.resolve() + + def test_node_modules_resolution(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + bundle = _make_bundle(tmp_path) + monkeypatch.delenv("DELEGATE_STDIO_PATH", raising=False) + monkeypatch.setenv("DELEGATE_STDIO_NODE_MODULES", str(tmp_path)) + assert _resolve_stdio_bundle() == bundle.resolve() + + def test_walks_up_ancestors_to_find_bundle(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """No env config: an install in an *ancestor* of the cwd is auto-located. + + This is the exact onboarding failure the walk-up fixes — `npm install` in + a directory with no package.json lands the bundle in an ancestor, and the + old cwd-only probe missed it. + """ + bundle = _make_bundle(tmp_path) # installed at the ancestor root + deep = tmp_path / "a" / "b" / "c" + deep.mkdir(parents=True) + home = tmp_path / "home" # a home WITHOUT the bundle, so the ancestor walk must find it + home.mkdir() + monkeypatch.delenv("DELEGATE_STDIO_PATH", raising=False) + monkeypatch.delenv("DELEGATE_STDIO_NODE_MODULES", raising=False) + monkeypatch.chdir(deep) + monkeypatch.setattr(Path, "home", lambda: home) + assert _resolve_stdio_bundle() == bundle.resolve() + + def test_walks_up_to_home_when_not_an_ancestor(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Home is probed even when it is not an ancestor of the cwd (the Windows case).""" + home = tmp_path / "home" + home.mkdir() + bundle = _make_bundle(home) # installed in ~ (npm's fallback target) + workdir = tmp_path / "work" / "deep" # cwd in a sibling subtree, home is NOT above it + workdir.mkdir(parents=True) + monkeypatch.delenv("DELEGATE_STDIO_PATH", raising=False) + monkeypatch.delenv("DELEGATE_STDIO_NODE_MODULES", raising=False) + monkeypatch.chdir(workdir) + monkeypatch.setattr(Path, "home", lambda: home) + assert _resolve_stdio_bundle() == bundle.resolve() + + def test_no_config_not_found_error_lists_searched_paths( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The diagnostic error names the locations searched (cwd, ancestors, home).""" + workdir = tmp_path / "work" + workdir.mkdir() + home = tmp_path / "home" + home.mkdir() + monkeypatch.delenv("DELEGATE_STDIO_PATH", raising=False) + monkeypatch.delenv("DELEGATE_STDIO_NODE_MODULES", raising=False) + monkeypatch.chdir(workdir) + monkeypatch.setattr(Path, "home", lambda: home) + with pytest.raises(AgentConfigError, match="Searched the cwd, its ancestors, and home"): + _resolve_stdio_bundle() + + +# ---- _maybe_pin_npm_globalconfig --------------------------------------------- + + +class TestMaybePinNpmGlobalconfig: + """The NPM_CONFIG_GLOBALCONFIG pin keeps the global npmrc (registry + token) + reachable from Delegate-runtime shells despite the interop's per-command + npm_config_prefix override. It must be strictly additive: operator value wins, + and no npmrc on disk means no env change.""" + + def test_pins_posix_layout(self, tmp_path: Path) -> None: + """<prefix>/lib/node_modules/@uipath ⇒ <prefix>/etc/npmrc (nodesource/deb layout).""" + tools_dir = tmp_path / "usr" / "lib" / "node_modules" / "@uipath" + tools_dir.mkdir(parents=True) + npmrc = tmp_path / "usr" / "etc" / "npmrc" + npmrc.parent.mkdir(parents=True) + npmrc.write_text("@uipath:registry=https://npm.pkg.github.com/\n", encoding="utf-8") + + env: dict[str, str] = {} + assert _maybe_pin_npm_globalconfig(env, str(tools_dir)) == npmrc + assert env["NPM_CONFIG_GLOBALCONFIG"] == str(npmrc) + + def test_pins_windows_layout(self, tmp_path: Path) -> None: + """<prefix>/node_modules/@uipath ⇒ <prefix>/etc/npmrc (Windows global layout).""" + tools_dir = tmp_path / "nodejs" / "node_modules" / "@uipath" + tools_dir.mkdir(parents=True) + npmrc = tmp_path / "nodejs" / "etc" / "npmrc" + npmrc.parent.mkdir(parents=True) + npmrc.write_text("@uipath:registry=https://npm.pkg.github.com/\n", encoding="utf-8") + + env: dict[str, str] = {} + assert _maybe_pin_npm_globalconfig(env, str(tools_dir)) == npmrc + assert env["NPM_CONFIG_GLOBALCONFIG"] == str(npmrc) + + def test_operator_value_wins(self, tmp_path: Path) -> None: + """An already-set NPM_CONFIG_GLOBALCONFIG is never overwritten (additive-only).""" + tools_dir = tmp_path / "usr" / "lib" / "node_modules" / "@uipath" + tools_dir.mkdir(parents=True) + npmrc = tmp_path / "usr" / "etc" / "npmrc" + npmrc.parent.mkdir(parents=True) + npmrc.write_text("", encoding="utf-8") + + env = {"NPM_CONFIG_GLOBALCONFIG": "/operator/npmrc"} + assert _maybe_pin_npm_globalconfig(env, str(tools_dir)) is None + assert env["NPM_CONFIG_GLOBALCONFIG"] == "/operator/npmrc" + + def test_no_npmrc_leaves_env_unchanged(self, tmp_path: Path) -> None: + """No global npmrc on disk ⇒ nothing to pin ⇒ env untouched.""" + tools_dir = tmp_path / "usr" / "lib" / "node_modules" / "@uipath" + tools_dir.mkdir(parents=True) + + env: dict[str, str] = {} + assert _maybe_pin_npm_globalconfig(env, str(tools_dir)) is None + assert "NPM_CONFIG_GLOBALCONFIG" not in env + + def test_no_plugin_tools_dir_is_noop(self) -> None: + """Without a discovered @uipath tools dir there is no anchor — env untouched.""" + env: dict[str, str] = {} + assert _maybe_pin_npm_globalconfig(env, None) is None + assert env == {} + + +# ---- _strip_gateway_creds ---------------------------------------------------- + + +class TestStripGatewayCreds: + """The eval's LLMGW_* gateway S2S credentials are for the judge/proxy. The + Delegate host authenticates with its own UiPath user token, and everything in + its env reaches the shells its interop spawns for the agent's Bash/PowerShell + tool calls — i.e. the code under test. run.py already withholds LLMGW_* from + docker; this covers tempdir tasks, which inherit the full eval env.""" + + def test_removes_llmgw_triple_and_keeps_the_rest(self) -> None: + env = { + "LLMGW_CLIENT_ID": "id", + "LLMGW_CLIENT_SECRET": "secret", + "LLMGW_URL": "https://gw", + "AUTH_TOKEN": "tok", + "DELEGATE_AUTH_TOKEN_FILE": "/live/.auth", + } + assert _strip_gateway_creds(env) == ("LLMGW_CLIENT_ID", "LLMGW_CLIENT_SECRET", "LLMGW_URL") + assert not any(k.startswith("LLMGW_") for k in env) + assert env == {"AUTH_TOKEN": "tok", "DELEGATE_AUTH_TOKEN_FILE": "/live/.auth"} + + def test_partial_set_still_stripped(self) -> None: + """Strip whatever is present — the secret is the sensitive part, and it + does not become harmless just because its siblings are absent.""" + env = {"LLMGW_CLIENT_SECRET": "secret", "AUTH_TOKEN": "tok"} + assert _strip_gateway_creds(env) == ("LLMGW_CLIENT_SECRET",) + assert "LLMGW_CLIENT_SECRET" not in env + + def test_noop_without_llmgw_vars(self) -> None: + env = {"AUTH_TOKEN": "tok"} + assert _strip_gateway_creds(env) == () + assert env == {"AUTH_TOKEN": "tok"} + + +# ---- S2S token-file refresher wiring ---------------------------------------- + + +class TestTokenRefresherWiring: + """start() must publish the refresher's token file to the host env, and + teardown must stop the refresher and remove the file.""" + + @staticmethod + def _fake_jwt(claims: dict[str, Any]) -> str: + payload = base64.urlsafe_b64encode(json.dumps(claims).encode()).decode().rstrip("=") + return f"eyJhbGciOiJub25lIn0.{payload}.sig" + + @classmethod + def _arm_env(cls, monkeypatch: pytest.MonkeyPatch, **overrides: str) -> None: + """Put the adapter's env in the state that activates the refresher.""" + env = { + "LLMGW_CLIENT_ID": "eval-client", + "LLMGW_CLIENT_SECRET": "s3cret", + "LLMGW_URL": "https://alpha.uipath.com", + "AUTH_TOKEN": cls._fake_jwt({"client_id": "eval-client", "exp": int(time.time()) + 3600}), + } + env.update(overrides) + for name, value in env.items(): + monkeypatch.setenv(name, value) + for name in ("DELEGATE_AUTH_TOKEN_FILE", "AUTH_TOKEN_FILE"): + if name not in overrides: + monkeypatch.delenv(name, raising=False) + + @staticmethod + def _make_started_agent(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> DelegateSdkAgent: + monkeypatch.setattr(delegate_sdk_agent, "_resolve_stdio_bundle", lambda: tmp_path / "delegate_stdio.mjs") + + async def _fake_spawn_and_init(self: DelegateSdkAgent) -> None: + return None + + monkeypatch.setattr(DelegateSdkAgent, "_spawn_and_init", _fake_spawn_and_init) + return DelegateSdkAgent(_make_config(), DirectRoute()) + + @pytest.mark.asyncio + async def test_start_publishes_token_file_and_stop_removes_it( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + self._arm_env(monkeypatch) + minted = self._fake_jwt({"client_id": "eval-client", "exp": int(time.time()) + 7200}) + monkeypatch.setattr(_delegate_s2s_token_file, "_mint_s2s_token", lambda creds: minted) + agent = self._make_started_agent(tmp_path, monkeypatch) + + await agent.start(str(tmp_path)) + + try: + host_env = agent._host_env + assert host_env is not None + token_file = host_env["DELEGATE_AUTH_TOKEN_FILE"] + assert host_env["AUTH_TOKEN_FILE"] == token_file + assert "LLMGW_CLIENT_SECRET" not in host_env + assert await asyncio.to_thread(Path(token_file).read_text, encoding="utf-8") == minted + finally: + await agent.stop() + + assert agent._token_refresher is None + assert not Path(token_file).exists() + + @pytest.mark.asyncio + async def test_respawn_keeps_the_refresher_and_its_token_file( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The replacement host re-reads the same file, so respawn must not tear it down.""" + self._arm_env(monkeypatch) + monkeypatch.setattr(_delegate_s2s_token_file, "_mint_s2s_token", lambda creds: "tok") + agent = self._make_started_agent(tmp_path, monkeypatch) + await agent.start(str(tmp_path)) + + try: + host_env = agent._host_env + assert host_env is not None + token_file = host_env["DELEGATE_AUTH_TOKEN_FILE"] + + await agent._respawn_host() + + assert agent._token_refresher is not None + assert Path(token_file).exists() + finally: + await agent.stop() + + @pytest.mark.asyncio + async def test_start_survives_a_refresher_that_cannot_start( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Freshness is an enhancement — a broken refresher must not fail Agent start.""" + self._arm_env(monkeypatch) + + async def _boom(self: _delegate_s2s_token_file.S2sTokenFileRefresher) -> str: + raise OSError("no space left on device") + + monkeypatch.setattr(_delegate_s2s_token_file.S2sTokenFileRefresher, "start", _boom) + agent = self._make_started_agent(tmp_path, monkeypatch) + + await agent.start(str(tmp_path)) + + try: + assert agent._token_refresher is None + host_env = agent._host_env + assert host_env is not None + assert "DELEGATE_AUTH_TOKEN_FILE" not in host_env + finally: + await agent.stop() + + @pytest.mark.asyncio + async def test_start_leaves_env_alone_when_external_token_file_configured( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + self._arm_env(monkeypatch, DELEGATE_AUTH_TOKEN_FILE="/external/refresher/token") + # Belt and braces: if the gate above ever regresses, this keeps the test + # from making a real 30s POST to the IdP before the assertion fails. + monkeypatch.setattr(_delegate_s2s_token_file, "_mint_s2s_token", lambda creds: "tok") + agent = self._make_started_agent(tmp_path, monkeypatch) + + await agent.start(str(tmp_path)) + + try: + assert agent._token_refresher is None + host_env = agent._host_env + assert host_env is not None + assert host_env["DELEGATE_AUTH_TOKEN_FILE"] == "/external/refresher/token" + finally: + await agent.stop() + + +# ---- _warn_unsupported_fields ---------------------------------------------- + + +class TestUnsupportedFieldsWarning: + def test_warns_for_each_unsupported_field_set(self, caplog: pytest.LogCaptureFixture) -> None: + config = _make_config( + allowed_tools=["Read"], + system_prompt="Act as a pirate.", + ) + agent = DelegateSdkAgent(config) + with caplog.at_level(logging.WARNING, logger="coder_eval.agents.delegate_sdk_agent"): + agent._warn_unsupported_fields() + messages = " ".join(rec.message for rec in caplog.records) + assert "allowed_tools" in messages + assert "system_prompt" in messages + + def test_no_warning_when_all_defaults(self, caplog: pytest.LogCaptureFixture) -> None: + """Default config — including ``permission_mode='default'`` — must not warn. + + ``permission_mode`` is intentionally NOT in ``_UNSUPPORTED_FIELDS`` since + the Delegate SDK has no permission concept and the default is truthy; + listing it would log a WARNING on every run. + """ + config = _make_config() # permission_mode="default" + agent = DelegateSdkAgent(config) + with caplog.at_level(logging.WARNING, logger="coder_eval.agents.delegate_sdk_agent"): + agent._warn_unsupported_fields() + assert caplog.records == [] + + def test_non_default_permission_mode_does_not_warn(self, caplog: pytest.LogCaptureFixture) -> None: + """Even non-default permission_mode is silently ignored — no SDK equivalent.""" + config = _make_config(permission_mode="bypassPermissions") + agent = DelegateSdkAgent(config) + with caplog.at_level(logging.WARNING, logger="coder_eval.agents.delegate_sdk_agent"): + agent._warn_unsupported_fields() + assert "permission_mode" not in " ".join(rec.message for rec in caplog.records) + + +# ---- _build_init_options --------------------------------------------------- + + +class TestBuildInitOptions: + def test_model_passes_through(self) -> None: + agent = DelegateSdkAgent(_make_config(model="claude_sonnet_4_5")) + opts = agent._build_init_options() + assert opts["model"] == "claude_sonnet_4_5" + assert opts["enableSkills"] is False + # max_turns is no longer an init option (moved to per-communicate() call). + assert "maxSteps" not in opts + + def test_plugins_map_to_bundled_skills_path(self, tmp_path: Path) -> None: + plugin_root = tmp_path / "plugin" + plugin_root.mkdir() + agent = DelegateSdkAgent( + _make_config(plugins=[{"type": "local", "path": str(plugin_root)}]), + ) + opts = agent._build_init_options() + assert opts["enableSkills"] is True + assert Path(opts["bundledSkillsPath"]) == (plugin_root / "skills").resolve() + + def test_env_endpoints_override(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("BACKEND_URL", "http://backend.example") + monkeypatch.setenv("INTEROP_URL", "http://interop.example") + agent = DelegateSdkAgent(_make_config()) + opts = agent._build_init_options() + assert opts["backendUrl"] == "http://backend.example" + assert opts["interopUrl"] == "http://interop.example" + + def test_no_model_field_when_unset(self) -> None: + agent = DelegateSdkAgent(_make_config()) + opts = agent._build_init_options() + assert "model" not in opts + assert "maxSteps" not in opts + + def test_env_slug_defaults_to_alpha(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("DELEGATE_SDK_ENV", raising=False) + agent = DelegateSdkAgent(_make_config()) + opts = agent._build_init_options() + assert opts["env"] == "alpha" + + @pytest.mark.parametrize("value", ["staging", "production"]) + def test_env_slug_from_environment(self, monkeypatch: pytest.MonkeyPatch, value: str) -> None: + monkeypatch.setenv("DELEGATE_SDK_ENV", value) + agent = DelegateSdkAgent(_make_config()) + opts = agent._build_init_options() + assert opts["env"] == value + + def test_env_slug_falls_back_to_alpha_when_blank(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("DELEGATE_SDK_ENV", "") + agent = DelegateSdkAgent(_make_config()) + opts = agent._build_init_options() + assert opts["env"] == "alpha" + + def test_shell_path_prepend_omitted_when_sandbox_has_no_mock_dirs(self) -> None: + # Nothing to inject: the option must be absent rather than an empty list, + # so hosts that predate it (and the SDK's option merge) see a clean config. + agent = DelegateSdkAgent(_make_config()) + + opts = agent._build_init_options() + + assert "shellPathPrepend" not in opts + + def test_shell_path_prepend_forwards_env_path_prepend_in_order(self) -> None: + """The sandbox's mock_path_dirs (Agent ABC ``env_path_prepend``) reach the + agent's shell commands as the host's ``shellPathPrepend`` init option. + A prepend on this process could not work — shell tools execute inside the + interop service, whose PATH was fixed at spawn — so the SDK injects the + composed PATH per command instead. Order is PATH precedence.""" + agent = DelegateSdkAgent(_make_config()) + + opts = agent._build_init_options(env_path_prepend=["/sandbox/mocks", "/sandbox/bin"]) + + assert opts["shellPathPrepend"] == ["/sandbox/mocks", "/sandbox/bin"] + + @pytest.mark.parametrize("effort", ["low", "medium", "high", "xhigh"]) + def test_effort_forwarded_from_sdk_options(self, effort: str) -> None: + # `sdk_options.effort` is the cross-agent surface: the host maps it + # onto useAppStore.effort, which the ChatFramework already serialises + # as user_config.effort on every chat request. + agent = DelegateSdkAgent(_make_config(sdk_options={"effort": effort})) + opts = agent._build_init_options() + assert opts["effort"] == effort + + def test_effort_omitted_when_sdk_options_empty(self) -> None: + agent = DelegateSdkAgent(_make_config()) + opts = agent._build_init_options() + assert "effort" not in opts + + def test_other_sdk_options_keys_silently_dropped(self) -> None: + # Non-effort sdk_options keys are Claude Code SDK-specific and have no + # delegate equivalent. They must not appear in the host init options + # — the host would reject unknown fields — and the agent should not + # warn either, since shared experiment YAMLs may legitimately carry + # Claude-only keys that we silently ignore on the delegate path. + agent = DelegateSdkAgent( + _make_config(sdk_options={"effort": "high", "max_thinking_tokens": 8000}), + ) + opts = agent._build_init_options() + assert opts["effort"] == "high" + assert "max_thinking_tokens" not in opts + + +# ---- working directory forwarding ------------------------------------------ + + +class TestWorkingDirectoryOption: + def test_init_options_carry_working_directory(self, tmp_path: Path) -> None: + # The sandbox path rides init options (host chdir + SDK shell-cwd + # seeding) instead of a prompt prefix — the orchestrator already + # injects "Your working directory is: ..." into every prompt, so a + # prefix here would duplicate it. + agent = DelegateSdkAgent(_make_config()) + agent.working_directory = tmp_path + opts = agent._build_init_options() + assert opts["workingDirectory"] == str(tmp_path) + + def test_init_options_omit_working_directory_before_start(self) -> None: + agent = DelegateSdkAgent(_make_config()) + opts = agent._build_init_options() + assert "workingDirectory" not in opts + + def test_project_and_session_id_forwarded_when_set(self) -> None: + agent = DelegateSdkAgent(_make_config(project_id="proj-1", session_id="sess-1")) + opts = agent._build_init_options() + assert opts["projectId"] == "proj-1" + assert opts["sessionId"] == "sess-1" + + def test_project_and_session_id_omitted_when_empty(self) -> None: + # Empty (the default) must be dropped so the SDK keeps its per-session + # wiki dir and generates a fresh session — the whole point of the guard. + agent = DelegateSdkAgent(_make_config()) + opts = agent._build_init_options() + assert "projectId" not in opts + assert "sessionId" not in opts + + +# ---- communicate() via fake subprocess ------------------------------------- + + +class _FakeStreamWriter: + """Async writer that records lines written to it.""" + + def __init__(self) -> None: + self.lines: list[str] = [] + self.closed = False + + def write(self, data: bytes) -> None: + self.lines.append(data.decode("utf-8")) + + async def drain(self) -> None: # pragma: no cover - trivial + return None + + def close(self) -> None: # pragma: no cover - trivial + self.closed = True + + +class _FakeStreamReader: + """Async reader that yields pre-queued byte lines.""" + + def __init__(self, lines: list[bytes]) -> None: + self._queue: list[bytes] = list(lines) + + async def readline(self) -> bytes: + if not self._queue: + return b"" # EOF + return self._queue.pop(0) + + +class _NeverStreamReader: + """Reader whose readline never returns — simulates a turn that never sends + a result, so the wall-clock deadline fires. Blocks until cancelled.""" + + def __init__(self) -> None: + self._never = asyncio.Event() + + async def readline(self) -> bytes: + await self._never.wait() # pragma: no cover - cancelled at teardown + return b"" + + +class _LimitOverrunReader: + """Reader that raises ``LimitOverrunError`` once on the first ``readline``, + then signals EOF — simulates a single host line exceeding the StreamReader + byte limit. Drives the drain tasks' over-limit recovery branch.""" + + def __init__(self, consumed: int = 10) -> None: + self._raised = False + self._consumed = consumed + + async def readline(self) -> bytes: + if not self._raised: + self._raised = True + raise asyncio.LimitOverrunError("line too long", self._consumed) + return b"" # EOF on the next read + + async def readexactly(self, n: int) -> bytes: + return b"x" * n + + +class _FakeProcess: + """Minimal duck-type for asyncio.subprocess.Process used by DelegateSdkAgent.""" + + def __init__(self, stdout_lines: list[dict[str, Any]]) -> None: + encoded = [(json.dumps(obj) + "\n").encode("utf-8") for obj in stdout_lines] + self.stdout = _FakeStreamReader(encoded) + self.stderr = _FakeStreamReader([]) + self.stdin = _FakeStreamWriter() + self.returncode: int | None = None + + async def wait(self) -> int: # pragma: no cover - trivial + self.returncode = 0 + return 0 + + def kill(self) -> None: # pragma: no cover - trivial + self.returncode = -9 + + +def _install_fake_process(agent: DelegateSdkAgent, stdout_lines: list[dict[str, Any]], tmp_path: Path) -> _FakeProcess: + """Replace agent's subprocess with a fake that replays ``stdout_lines``. + + Also starts the stdout/stderr drain tasks — ``_drain_stdout`` is the only + writer to ``_stdout_queue``, which ``_read_line()`` blocks on. Without it, + every ``communicate()`` call hangs on ``queue.get()`` forever rather than + consuming the scripted host output. Must be called from inside a running + event loop (i.e., from ``@pytest.mark.asyncio`` tests). + """ + proc = _FakeProcess(stdout_lines) + agent._process = proc # type: ignore[assignment] + agent.working_directory = tmp_path + agent._stdout_task = asyncio.create_task(agent._drain_stdout(proc.stdout)) # type: ignore[arg-type] + agent._stderr_task = asyncio.create_task(agent._drain_stderr(proc.stderr)) # type: ignore[arg-type] + return proc + + +_WAF_BLOCK_PAGE_HEAD = ( + '<!DOCTYPE html>\n\t<html>\n\t\t<head>\n\t\t\t<meta charset="utf-8" />\n' + "\t\t\t<title>Continue with UiPath Platform\n" + "\t\t\t