Rebuild the eval engine on inspect_ai, keeping real Claude Code as the agent

.github/workflows/selftest.yml

@@ -44,6 +44,77 @@ jobs:
       - name: Run the suite
         run: python -m unittest discover -s tests -t . --verbose
 
+  sandbox:
+    name: Sandbox machinery (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # Both sandboxes a run can get. Ubuntu has Docker, which is the
+        # default and what CI uses; Windows has none, which is the other
+        # supported shape. The Windows leg is the only place the cross-platform
+        # sandbox helpers are exercised at all -- inspect's own assume a POSIX
+        # guest, and a scorer that cannot list the guest cannot grade it.
+        os: [ubuntu-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          # inspect-ai pulls in the order of eighty packages, so the download
+          # is most of this step. Keyed on pyproject, which is where the extra
+          # is declared and the only thing that changes what gets installed.
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install with the inspect extra
+        run: python -m pip install --upgrade pip && python -m pip install ".[inspect]"
+
+      # The case seeds a file and expects that same file. Nothing an agent
+      # does can satisfy it and nothing an agent does can break it, so what
+      # passes or fails is the layer underneath: the sandbox started, the
+      # fixture was staged into it, the guest was listed, and the listing was
+      # matched. That is where every cross-platform bug so far has been.
+      - name: Build a repo to test
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p fixture/demo-skill/evals fixture/demo-skill/fixtures
+          cat > fixture/demo-skill/SKILL.md <<'EOF'
+          ---
+          name: demo-skill
+          description: Does demonstrable things, for a test that needs a skill.
+          ---
+          EOF
+          echo "seeded by the case, not produced by the agent" \
+            > fixture/demo-skill/fixtures/seeded.txt
+          cat > fixture/demo-skill/evals/evals.json <<'EOF'
+          {
+            "evaluations": [
+              {"id": "demo-a", "skill_should_trigger": true, "prompt": "do the demo thing",
+               "workspace": "fixtures", "files_exist": ["seeded.txt"]},
+              {"id": "demo-b", "skill_should_trigger": true, "prompt": "another way to ask"},
+              {"id": "demo-c", "skill_should_trigger": true, "prompt": "a third phrasing"},
+              {"id": "demo-d", "skill_should_trigger": false, "prompt": "something adjacent"},
+              {"id": "demo-e", "skill_should_trigger": false, "prompt": "something else entirely"}
+            ]
+          }
+          EOF
+
+      # No key, no model, no agent: a stub solver stands in for the agent so
+      # only the machinery beneath it can pass the case. That is what makes
+      # this runnable on a pull request from a fork, which is exactly where a
+      # cross-platform sandbox bug shows up.
+      - name: Exercise the sandbox, the staging and the listing
+        # The expected provider is supplied per platform rather than read back
+        # from the harness. Ubuntu must get a real container; a leg that
+        # quietly fell back to `local` would pass every check below against the
+        # host filesystem and stay green under the name "Sandbox machinery".
+        run: >
+          python tools/sandbox_smoketest.py fixture --skill demo-skill
+          --expect-sandbox ${{ matrix.os == 'windows-latest' && 'local' || 'docker' }}
+
   action:
     name: Action against a throwaway repo
     runs-on: ubuntu-latest

.gitignore

@@ -11,5 +11,10 @@ dist/
 # Reports and kept transcripts, written into the repo under test
 .skillscope/
 
+# inspect's own run transcripts, wherever a tool was invoked from. Committing
+# one is easy to do by accident and they are large, opaque and reproducible.
+*.eval
+tools/logs/
+
 # Nested checkout of this repo, used by the reusable workflows in CI
 .skillscope-action/

docs/usage.md

@@ -208,6 +208,105 @@ Legs with a scoped environment run as a separate job, because a job's
 credentials are fixed before its matrix expands. A repo that declares no scoped
 environment gets one matrix, labels and all.
 
+## Which engine grades a run
+
+`--engine` chooses what actually runs the cases. The dataset, the CLI and the
+reports are identical whichever you pick; only the thing driving the agent
+changes.
+
+| `--engine` | What runs | Runs in | Needs |
+| --- | --- | --- | --- |
+| `legacy` (default) | The `claude` CLI, driven directly | the host | the CLI on `PATH` |
+| `claude-code` | The real CLI, via `inspect_swe` | a sandbox | `skillscope[verify]`, Linux only |
+| `claude-code-no-sandbox` | The real CLI, under `inspect_ai` | the host | `skillscope[inspect]`, the CLI on `PATH` |
+
+All three drive the agent a skill is written for, so what differs between them
+is **where the agent runs**, not what it is. That is the axis worth choosing
+along: the host measures the machine as it is, with whatever else is installed
+on it, and the sandbox measures the skill alone. When the two disagree, the
+disagreement is usually a fact about one of those environments rather than
+about the skill -- which is a thing one engine on its own cannot tell you.
+
+`claude-code` is a reporting leg, never a gate. Harness runs are
+nondeterministic and the harness is not what is being graded, so a divergence
+there is a question about the skill rather than a build failure.
+
+**Routing runs on all three**, and the choice matters more there than it does
+for behavioral. A stray user-level skill on the runner does not spoil one
+case's grade -- it is offered for every prompt, so it changes every decision at
+once while the run still reports a clean accuracy. `claude-code` is the only
+leg immune by construction: its guest has no `~/.claude` to contribute.
+
+The host legs handle it differently. `legacy` redirects the CLI's config dir
+when it can, warns when it cannot, and names any gate-crashing skill in the
+report, because it reads the CLI's session-init event.
+`claude-code-no-sandbox` cannot read that event, so it has no way to notice the
+same contamination or report it -- and therefore refuses to run a routing leg
+at all unless `ANTHROPIC_API_KEY` is set, which is what lets it redirect the
+config dir. Refusing beats being quietly wrong about every case.
+
+Stopping at the decision differs by leg, and it is the difference that decides
+what a routing run costs. `legacy` sees the activation in the CLI's stream
+after the call has run, then kills the process. `claude-code` stops earlier:
+its calls cross inspect's bridge, so the one that reveals the decision is
+declined before it runs, and `--max-tool-calls` / `--max-inspection-calls` ride
+the same path. `claude-code-no-sandbox` cannot do either — its CLI's output is
+buffered until the process exits, so the decision is only visible once it has
+been paid for. There it runs to the message cap, bounded by `--case-timeout`
+and by the CLI's own `--max-budget-usd`, and is correspondingly dearer per
+case. The report records which caps actually applied rather than which were
+asked for.
+
+### Where a sandboxed run is sandboxed
+
+Two separate decisions, made by different people.
+
+**Which provider** is a property of the runner, chosen with
+`SKILLSCOPE_SANDBOX`. Docker by default; `podman` on a host that has that
+instead; `local` to skip the container. `local` is for working locally rather
+than for CI, because a graded run that quietly dropped its sandbox would report
+the same numbers with none of the isolation.
+
+Podman needs three things, and each was discovered by the next one failing:
+
+* `pip install 'skillscope[podman]'`. The provider is registered by a separate
+  package through an entry point, so the podman binary alone is not enough.
+* `podman-compose`, and `INSPECT_PODMAN_COMPOSE=podman-compose`. Bare
+  `podman compose` is a shim that delegates to whichever compose provider it
+  finds, which on a host that also has Docker is Docker's -- and that then
+  talks to a daemon podman was chosen to avoid.
+* A search registry, because podman will not guess one. Docker assumes Docker
+  Hub for an image name with no registry; podman refuses, and the default
+  sandbox image is named without one. `unqualified-search-registries =
+  ["docker.io"]` in `/etc/containers/registries.conf`.
+
+Podman is worth the setup where the runner's user cannot reach the Docker
+socket, since it is daemonless and rootless and needs neither that nor group
+membership.
+
+**What the sandbox must provide** is a property of the skill, declared as
+`sandbox: compose.yaml` in its `evals/machine.yml`, resolved beside it. Skills get a container with
+no network by default; one that installs a server or pulls a model cannot run
+that way and says so. Selecting a provider does not discard what a skill asked
+for -- the compose file rides along.
+
+[`examples/skill-with-a-device/evals/`](../examples/skill-with-a-device/evals)
+is the pair, worked through: a `machine.yml` that asks for GPU runners and
+names a compose file, and the compose file that binds the devices in and
+grants egress. The two are not substitutes. `labels:` decides which machine the
+job lands on; `sandbox:` decides whether the container on it can see the
+hardware that machine has. A skill that sets only the first gets the right
+runner and a container that cannot reach its device.
+
+Windows is the exception to both: inspect's sandbox layer and every tool built
+on it assume a POSIX guest, so those legs run unsandboxed and trade isolation
+for running on the platform they are meant to test.
+
+To see what changing engine would do to your own datasets before changing it,
+[`tools/benchmark_engines.py`](../tools/benchmark_engines.py) runs the same
+cases through two engines and reports per-case agreement, measured against how
+much one engine already disagrees with itself.
+
 ## In CI: one job
 
 [`reusable.yml`](../.github/workflows/reusable.yml) grades a repo's skills with

examples/skill-with-a-device/evals/compose.yaml

@@ -0,0 +1,49 @@
+###############################################################################
+# Copyright Advanced Micro Devices, Inc.
+#
+# SPDX-License-Identifier: MIT
+###############################################################################
+
+# Example: the sandbox for a skill that needs a GPU and network egress.
+#
+# Named by `sandbox: compose.yaml` in the machine.yml beside this file. The
+# default sandbox is this minus `devices`, `group_add`, `security_opt`, `ipc`
+# and with `network_mode: none` -- so everything below is the opt-in, and a
+# skill that needs none of it should not ship a compose file at all.
+
+services:
+  # inspect runs the cases in the service named `default`. One service with a
+  # different name and `x-default: true` works too; several services without
+  # either is the error you get instead of a container.
+  default:
+    # The default image. Whatever you put here needs inspect's tool support to
+    # be reachable inside the container, so start from this one and add to it
+    # via `build:` rather than replacing it with a bare vendor image.
+    image: "aisiuk/inspect-tool-support"
+
+    # inspect execs into a container that is already up, so the container has
+    # to stay up on its own. Without these it exits immediately and every case
+    # fails on a sandbox that is not there.
+    command: "tail -f /dev/null"
+    init: true
+    stop_grace_period: 1s
+
+    # NETWORK. The default sandbox sets `network_mode: none`; omitting that key
+    # is what grants egress. Only do this for a skill that genuinely needs it
+    # -- pulling a model, reaching a package index -- because it is also the
+    # thing that lets a case reach the internet and stop being reproducible.
+    #
+    # `network_mode: none` here would restore the default.
+
+    # DEVICE. The GPU nodes, bound in from the host. The job has to already be
+    # on a machine that has them; that is what `labels:` in machine.yml is for.
+    devices:
+      - "/dev/kfd"
+      - "/dev/dri"
+
+    # What the userspace stack needs on top of the device nodes themselves.
+    group_add:
+      - video
+    security_opt:
+      - seccomp=unconfined
+    ipc: host

examples/skill-with-a-device/evals/machine.yml

@@ -0,0 +1,30 @@
+###############################################################################
+# Copyright Advanced Micro Devices, Inc.
+#
+# SPDX-License-Identifier: MIT
+###############################################################################
+
+# Example: a skill whose behavioral cases need real hardware.
+#
+# This file lives at <skill>/evals/machine.yml. It answers two questions that
+# are asked by different people, and it is worth keeping them apart:
+#
+#   * `os` / `labels` say which CI runner the job lands on -- a machine with
+#     the device physically in it. That is the repo's scheduling problem.
+#   * `sandbox` says what the container on that machine has to provide. That is
+#     the skill's problem, and without it the job lands on the right hardware
+#     and then runs in a container that cannot see it.
+#
+# Both are needed. Neither implies the other.
+
+os:
+  - Linux
+
+# The hardware the cases need, named as a runner label rather than a pool.
+labels:
+  - mi300x
+
+# Resolved beside this file, in evals/. Absent means the default container:
+# no network, no devices, which is what a skill that only reads and writes
+# files should want.
+sandbox: compose.yaml

pyproject.toml

@@ -15,16 +15,41 @@ requires-python = ">=3.10"
 license = { text = "MIT" }
 authors = [{ name = "Advanced Micro Devices, Inc." }]
 
-# The runner itself is standard library only, so a run needs no wheels beyond
-# this package. PyYAML is the one exception: it reads the optional
+# The legacy runner is standard library only, so a graded run needs no wheels
+# beyond this package. PyYAML is the one exception: it reads the optional
 # evals/machine.yml, which only CI planning touches.
 dependencies = ["pyyaml>=6.0"]
 
+# The engines built on inspect_ai (`--engine claude-code`, `--engine
+# claude-code-no-sandbox`). Kept an extra so the default path keeps the stdlib-only
+# property: a repo that has not migrated installs nothing new.
+[project.optional-dependencies]
+# `anthropic` is listed explicitly: inspect-ai treats every model provider as
+# optional, so installing it alone gets you a harness that cannot reach a model.
+inspect = ["inspect-ai>=0.3.263", "anthropic>=0.40"]
+
+# For a runner that has podman rather than docker. The provider registers
+# itself through an `inspect_ai` entry point, so installing it is the whole
+# setup; `SKILLSCOPE_SANDBOX=podman` then selects it.
+podman = ["skillscope[inspect]", "inspect-podman"]
+
+# The Claude Code verification leg (`--engine claude-code`). Kept out of the
+# `inspect` extra because it is a reporting-only cross-check, not something a
+# graded run needs -- and because it only works on a POSIX guest.
+#
+# 0.2.71 for `claude_code(effort=)`. Not cosmetic: without it this leg runs at
+# the model's default reasoning effort while every other leg runs at the one it
+# was given, and the gap between them reads as a fact about isolation. 0.2.70
+# raises `TypeError: Unexpected keyword argument(s): effort`, and a runner that
+# already had it satisfied the old floor and never upgraded -- so the trial
+# installed a version that could not run the leg.
+verify = ["skillscope[inspect]", "inspect-swe>=0.2.71"]
+
 [project.scripts]
 skillscope = "skillscope.cli:main"
 
 [tool.setuptools]
-packages = ["skillscope"]
+packages = ["skillscope", "skillscope.engine"]
 
 [tool.setuptools.package-data]
 skillscope = ["data/*.json", "schema/*.json"]

skillscope/agent.py

@@ -41,7 +41,7 @@
 from dataclasses import dataclass
 from pathlib import Path
 
-from . import datasets, deadline
+from . import datasets, deadline, usage
 
 DEFAULT_MODEL = os.environ.get("SKILLSCOPE_MODEL", "opus")
 DEFAULT_EFFORT = os.environ.get("SKILLSCOPE_EFFORT", "high")
@@ -70,10 +70,18 @@ def is_automated_env() -> bool:
     )
 
 
+# Model providers that reach no cloud service. The CI pin exists to keep paid
+# runs comparable between runs; one of these grades nothing and costs nothing,
+# so pinning it only turns a free wiring check into a run that needs a key.
+NO_PROVIDER_PREFIXES = ("mockllm",)
+
+
 def enforce_model_policy(model: str | None) -> str | None:
     """Coerce non-opus models to opus in CI; pass through otherwise."""
     if model is None or not is_automated_env() or "opus" in model.lower():
         return model
+    if model.lower().startswith(NO_PROVIDER_PREFIXES):
+        return model
     _safe_print(
         f"[skillscope] automated run: coercing model '{model}' -> "
         f"'{AUTOMATED_MODEL}' to pin the CI model."
@@ -387,6 +395,9 @@ def __init__(self, *, workspace: Path, events: list[dict], judge_model: str | No
 
         result_text = ""
         for ev in events:
+            # Recording what the run spent is what lets it be compared against
+            # the same cases on the other engine.
+            usage.record_stream_event(ev)
             if ev.get("type") == "result" and isinstance(ev.get("result"), str):
                 result_text = ev["result"]