Add cancel + Rich Interaction conformance scenarios (th-eae69d)

dotnet/server/integration-tests/ScenarioParityTests.cs

@@ -10,6 +10,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using SmooAI.SmoothOperator.Core;
 using SmooAI.SmoothOperator.Server.AspNetCore;
+using Xunit.Sdk;
 
 namespace SmooAI.SmoothOperator.Server.IntegrationTests;
 
@@ -57,12 +58,43 @@ public void JsonEquals_ComparesNumbersByValue_NotRepresentation()
         Assert.False(JsonEquals(JsonNode.Parse("""{"a": 1}"""), JsonNode.Parse("""{"a": 1, "b": 2}""")));
     }
 
+    /// <summary>This runner's id in a scenario's <c>knownDivergences</c> list.</summary>
+    private const string Lang = "dotnet";
+
     [Theory]
     [MemberData(nameof(Scenarios))]
     public async Task ScenarioParity(string name, string path)
     {
-        _ = name; // surfaced as the test id via MemberData
         var scenario = JsonNode.Parse(await File.ReadAllTextAsync(path))!.AsObject();
+
+        // `knownDivergences` lists the languages a scenario is known to fail on today,
+        // with `knownDivergencesReason` next to it. An EXPIRING marker, not a skip: a
+        // listed language that fails is reported and tolerated, but one that PASSES fails
+        // the build, so a marker cannot rot silently into a green test that proves nothing
+        // (the failure mode this corpus exists to catch).
+        var divergent = scenario["knownDivergences"]?.AsArray()
+            .Any(l => l?.GetValue<string>() == Lang) ?? false;
+        if (divergent)
+        {
+            try
+            {
+                await RunScenarioAsync(scenario);
+            }
+            catch (Exception ex) when (ex is XunitException or IOException or WebSocketException)
+            {
+                Console.WriteLine(
+                    $"[known divergence] {name}: {scenario["knownDivergencesReason"]?.GetValue<string>()}\n  {ex.Message}");
+                return;
+            }
+
+            Assert.Fail($"remove {Lang} from knownDivergences in {name} — it now passes");
+        }
+
+        await RunScenarioAsync(scenario);
+    }
+
+    private static async Task RunScenarioAsync(JsonObject scenario)
+    {
         var chat = BuildMock(scenario["mockLlmScript"]?.AsArray());
         var serverDirective = scenario["server"]?.AsObject();
         var tools = BuildTools(serverDirective?["tools"]?.AsArray());

go/server/scenario_parity_test.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"slices"
 	"strconv"
 	"strings"
 	"testing"
@@ -36,8 +37,19 @@ type scenario struct {
 	Server        scenarioServer   `json:"server"`
 	MockLlmScript []mockScriptStep `json:"mockLlmScript"`
 	Steps         []scenarioStep   `json:"steps"`
+
+	// KnownDivergences lists the languages this scenario is known to fail on today,
+	// with KnownDivergencesReason next to it. An EXPIRING marker, not a skip: a
+	// listed language that fails is reported and tolerated, but one that PASSES
+	// fails the build, so a marker cannot rot silently into a green test that proves
+	// nothing (the failure mode this corpus exists to catch).
+	KnownDivergences       []string `json:"knownDivergences"`
+	KnownDivergencesReason string   `json:"knownDivergencesReason"`
 }
 
+// lang is this runner's id in a scenario's knownDivergences list.
+const lang = "go"
+
 // scenarioServer is the optional `server` directive: deployment-time config the runner
 // applies when starting the server — the tools the agent may call, and the subset gated
 // behind write-confirmation HITL.
@@ -97,7 +109,48 @@ type matcher struct {
 
 // scenariosDir resolves spec/conformance/scenarios relative to the repo root (this
 // file lives at go/server/, so the root is three parents up).
-func scenariosDir(t *testing.T) string {
+// parityT is the slice of *testing.T the scenario runner actually uses. Narrowing
+// it to an interface lets a known-divergence scenario run against a recorder that
+// CAPTURES the first failure instead of failing the build — *testing.T's own
+// Fatalf is terminal and cannot be un-failed.
+type parityT interface {
+	Helper()
+	Fatalf(format string, args ...any)
+}
+
+// divergenceRecorder is a parityT that turns the first Fatalf into a panic
+// carrying the message, so the caller can recover it. Fatalf must not return
+// (callers rely on it aborting), and panic is the only way to do that while
+// leaving the real *testing.T untouched.
+type divergenceRecorder struct{ msg string }
+
+type divergenceFailure struct{ msg string }
+
+func (r *divergenceRecorder) Helper() {}
+
+func (r *divergenceRecorder) Fatalf(format string, args ...any) {
+	r.msg = fmt.Sprintf(format, args...)
+	panic(divergenceFailure{r.msg})
+}
+
+// runDivergent runs a scenario against a recorder, reporting whether it passed
+// and the failure message if it did not.
+func runDivergent(path string) (passed bool, msg string) {
+	rec := &divergenceRecorder{}
+	defer func() {
+		if r := recover(); r != nil {
+			if f, ok := r.(divergenceFailure); ok {
+				passed, msg = false, f.msg
+				return
+			}
+			panic(r)
+		}
+	}()
+	runScenario(rec, path)
+	return true, ""
+}
+
+func scenariosDir(t parityT) string {
 	t.Helper()
 	wd, err := os.Getwd()
 	if err != nil {
@@ -110,7 +163,7 @@ func scenariosDir(t *testing.T) string {
 // ("data.data.response") or, when it parses as a non-negative integer, an array by
 // position ("citations.0.id") — so a citation field can be asserted by index. Mirrors
 // the Python reference runner's array-aware dot helper.
-func dot(t *testing.T, obj map[string]any, path string) (any, bool) {
+func dot(t parityT, obj map[string]any, path string) (any, bool) {
 	t.Helper()
 	var cur any = obj
 	for _, part := range strings.Split(path, ".") {
@@ -136,7 +189,7 @@ func dot(t *testing.T, obj map[string]any, path string) (any, bool) {
 
 // buildMock loads a scenario's mockLlmScript into the engine's MockLlmProvider — the
 // deterministic record/replay source that makes the turn identical across languages.
-func buildMock(t *testing.T, script []mockScriptStep) *core.MockLlmProvider {
+func buildMock(t parityT, script []mockScriptStep) *core.MockLlmProvider {
 	t.Helper()
 	mock := core.NewMockLlmProvider()
 	for _, entry := range script {
@@ -275,20 +328,40 @@ func TestScenarioParity(t *testing.T) {
 		path := path
 		name := strings.TrimSuffix(filepath.Base(path), ".json")
 		t.Run(name, func(t *testing.T) {
-			runScenario(t, path)
+			sc, err := loadScenario(path)
+			if err != nil {
+				t.Fatalf("load scenario: %v", err)
+			}
+			if !slices.Contains(sc.KnownDivergences, lang) {
+				runScenario(t, path)
+				return
+			}
+			passed, msg := runDivergent(path)
+			if passed {
+				t.Fatalf("remove %s from knownDivergences in %s.json — it now passes", lang, name)
+			}
+			t.Logf("known divergence (%s): %s", sc.KnownDivergencesReason, msg)
 		})
 	}
 }
 
-func runScenario(t *testing.T, path string) {
-	t.Helper()
+func loadScenario(path string) (scenario, error) {
+	var sc scenario
 	raw, err := os.ReadFile(path)
 	if err != nil {
-		t.Fatalf("read scenario: %v", err)
+		return sc, fmt.Errorf("read scenario: %w", err)
 	}
-	var sc scenario
 	if err := json.Unmarshal(raw, &sc); err != nil {
-		t.Fatalf("parse scenario: %v", err)
+		return sc, fmt.Errorf("parse scenario: %w", err)
+	}
+	return sc, nil
+}
+
+func runScenario(t parityT, path string) {
+	t.Helper()
+	sc, err := loadScenario(path)
+	if err != nil {
+		t.Fatalf("%v", err)
 	}
 
 	mock := buildMock(t, sc.MockLlmScript)
@@ -334,7 +407,7 @@ func runScenario(t *testing.T, path string) {
 
 // nextEvent returns the next protocol event, skipping non-semantic keepalive/pong
 // frames (as the Python reference does).
-func nextEvent(t *testing.T, transport protocol.Transport) map[string]any {
+func nextEvent(t parityT, transport protocol.Transport) map[string]any {
 	t.Helper()
 	for {
 		select {
@@ -364,7 +437,7 @@ func nextEvent(t *testing.T, transport protocol.Transport) map[string]any {
 // a faithful port of the Python reference's _match_expected state machine: one-event
 // lookahead for `repeat` overrun, status / statusGte / assert checks, var capture, and
 // accumulate + assertAccumulated.
-func matchExpected(t *testing.T, transport protocol.Transport, matchers []matcher, vars map[string]any) {
+func matchExpected(t parityT, transport protocol.Transport, matchers []matcher, vars map[string]any) {
 	t.Helper()
 	var pending map[string]any // one-event lookahead when a `repeat` matcher overruns
 	for _, m := range matchers {

python/server/tests/test_scenario_parity.py

@@ -25,6 +25,9 @@
 SCENARIOS_DIR = Path(__file__).resolve().parents[3] / "spec" / "conformance" / "scenarios"
 SCENARIOS = sorted(SCENARIOS_DIR.glob("*.json"))
 
+#: This runner's id in a scenario's ``knownDivergences`` list.
+LANG = "python"
+
 
 def _dot(obj, path: str):
     """Resolve a dotted path (``data.data.response.responseParts``) into a nested
@@ -94,6 +97,21 @@ def _subst(value, vars_: dict):
 @pytest.mark.asyncio
 async def test_scenario_parity(path: Path) -> None:
     scenario = json.loads(path.read_text())
+    # `knownDivergences` lists the languages a scenario is known to fail on today,
+    # with `knownDivergencesReason` next to it. An EXPIRING marker, not a skip: a
+    # listed language that fails is reported and tolerated, but one that PASSES
+    # fails the build, so a marker cannot rot silently into a green test that
+    # proves nothing (which is the failure mode this whole corpus exists to catch).
+    if LANG in scenario.get("knownDivergences", []):
+        try:
+            await _run_scenario(scenario)
+        except AssertionError as exc:
+            pytest.xfail(f"known divergence ({scenario.get('knownDivergencesReason', '')}): {exc}")
+        pytest.fail(f"remove {LANG} from knownDivergences in {path.name} — it now passes")
+    await _run_scenario(scenario)
+
+
+async def _run_scenario(scenario: dict) -> None:
     mock = _build_mock(scenario.get("mockLlmScript", []))
     server_spec = scenario.get("server", {})
     tools = _build_tools(server_spec.get("tools", []))

rust/smooth-operator-server/tests/scenario_parity.rs

@@ -496,6 +496,23 @@ async fn match_expected(
     }
 }
 
+/// This runner's id in a scenario's `knownDivergences` list.
+const LANG: &str = "rust";
+
+/// Whether `path` marks this language as a known divergence.
+fn is_divergent(path: &Path) -> bool {
+    let Ok(raw) = std::fs::read_to_string(path) else {
+        return false;
+    };
+    let Ok(scenario) = serde_json::from_str::<Value>(&raw) else {
+        return false;
+    };
+    scenario
+        .get("knownDivergences")
+        .and_then(Value::as_array)
+        .is_some_and(|langs| langs.iter().any(|l| l.as_str() == Some(LANG)))
+}
+
 /// Drive one scenario file end-to-end through the reference server.
 async fn run_scenario(path: &Path) {
     let scenario: Value =
@@ -554,13 +571,32 @@ async fn scenario_parity_corpus() {
         scenarios_dir().display()
     );
     for path in &paths {
-        eprintln!(
-            "[scenario-parity] {}",
-            path.file_name().unwrap().to_string_lossy()
-        );
-        run_scenario(path).await;
+        let name = path.file_name().unwrap().to_string_lossy().to_string();
+        eprintln!("[scenario-parity] {name}");
+        if !is_divergent(path) {
+            run_scenario(path).await;
+            continue;
+        }
+        // `knownDivergences` lists the languages a scenario is known to fail on
+        // today, with `knownDivergencesReason` next to it. An EXPIRING marker, not
+        // a skip: a listed language that fails is reported and tolerated, but one
+        // that PASSES fails the build, so a marker cannot rot silently into a green
+        // test that proves nothing (the failure mode this corpus exists to catch).
+        // Spawned so the scenario's panic is caught by the JoinHandle rather than
+        // unwinding the whole corpus.
+        let owned = path.clone();
+        if tokio::spawn(async move { run_scenario(&owned).await })
+            .await
+            .is_ok()
+        {
+            panic!("remove {LANG} from knownDivergences in {name} — it now passes");
+        }
+        eprintln!("[scenario-parity] {name}: known divergence for {LANG}, tolerated");
     }
-    eprintln!("[scenario-parity] {} scenario(s) passed", paths.len());
+    eprintln!(
+        "[scenario-parity] {} scenario(s) accounted for",
+        paths.len()
+    );
 }
 
 /// Offline guard for the by-value JSON comparison (pearl th-4f1263). A corpus

spec/conformance/scenarios/README.md

@@ -76,6 +76,60 @@ Each server provides a small test that, for every `*.json` here:
 
 The **Python reference runner** is [`python/server/tests/test_scenario_parity.py`](../../../python/server/tests/test_scenario_parity.py) — port its ~80 lines into the TS/Go/C#/Rust server suites. When all five run this corpus green, the servers are at protocol parity.
 
+## Cancellation and Rich Interactions — the newest scenarios, and where the servers disagree
+
+Until pearl **th-eae69d** this corpus had no `cancel` scenario and no `interaction` scenario. That was the audit's headline finding: cancellation and Rich Interactions — the two newest features — were cross-checked only by fixture *shape* (does an `interaction_required` event match its schema), never by cross-language *behavior*. A port could be fully green on parity while implementing neither correctly, and six independent reviewers found the same divergences in four different languages because nothing in CI was positioned to catch them.
+
+Eight scenarios now cover them:
+
+| scenario | what it pins |
+|---|---|
+| `cancel-mid-turn` | a cancelled turn emits terminal `cancelled` (499, echoing the **turn's** requestId) in place of `eventual_response`, and frees the turn slot |
+| `cancel-no-active-turn-noop` | a `cancel` with no active turn emits **nothing** |
+| `interaction-park-resume` | `identity_intake` parks behind the `identity_form` capability; a matching `submit_interaction` resumes it |
+| `interaction-invalid-retryable` | invalid values → `interaction_invalid`, turn **stays parked**, twice, then a corrected submit resumes |
+| `interaction-stale-id-rejected` | a stale `interactionId` → `error`/`INTERACTION_MISMATCH`, turn stays parked |
+| `interaction-declined` | `declined: true` resolves the park without values |
+| `interaction-conversational-fallback` | a session that did NOT declare the capability gets the text fallback, never a park |
+| `interaction-choices-park-resume` | the second kind (`choices`) rides the same generic envelope |
+
+### Making cancellation deterministic
+
+A mock turn finishes faster than a `cancel` frame can race it, and the format has no "slow tool" directive (`server.tools` entries return a fixed string immediately). `cancel-mid-turn` therefore opens its in-flight window with a **write-confirmation park** (`server.confirmTools`) — the one pause this corpus can express. `cancel-no-active-turn-noop` asserts "nothing arrives" structurally, since no runner has a drain check: the cancel step expects zero events, so any stray event is consumed by the *next* step's first matcher and fails it.
+
+### `knownDivergences` — an expiring marker, not a skip
+
+A scenario may name the languages it is known to fail on today, with the reason and pearl id right next to it:
+
+```jsonc
+"knownDivergences": ["go", "typescript", "python", "dotnet"],
+"knownDivergencesReason": "th-eae69d — these four emit the raise tool's toolCall chunk BEFORE interaction_required …",
+```
+
+All five runners honour it, and the contract has two halves — the second is the one that matters:
+
+- a **listed** language that FAILS is reported (with the reason and the actual assertion) and does not fail the build;
+- a **listed** language that PASSES **fails the build**, with `remove <lang> from knownDivergences in <scenario> — it now passes`.
+
+Without that second half the markers rot silently and we recreate the exact "green tests that prove nothing" problem this corpus exists to catch. A marker is a tracked bug with an expiry, never an accepted difference — the entry comes out the moment the port is fixed, and the build tells you when that is.
+
+Implementation note per language, since `*testing.T` and panics do not catch alike: Rust runs a marked scenario on a `tokio::spawn` handle so its panic surfaces as a `JoinError`; Go narrows the runner's `*testing.T` to a small `parityT` interface so a marked scenario can run against a recorder whose `Fatalf` panics with the message instead of failing the build; TypeScript, Python and .NET just catch the assertion. ⚠️ `go test` caches results and does not invalidate on a scenario-JSON edit — use `-count=1` when iterating locally.
+
+### Known divergences these scenarios expose
+
+Recorded here as facts, not as license to weaken the scenarios. **Do not "fix" a scenario to make a port pass.**
+
+- **Park event vs. the raise tool's `stream_chunk` — Rust is 1 of 5, and Rust is right.** For a Rich Interaction, Rust emits `interaction_required` *before* the raise tool's `toolCall` chunk; Go, TypeScript, Python and .NET all emit the chunk first. **Ruled a port bug, not a protocol variant**, on three grounds: all five already defer the gated tool's chunk until after the prompt for the *other* park type — `hitl-write-confirmation`, a scenario **all five pass today** — so the four are internally inconsistent between their own two park paths while Rust is consistent; Rust is the designated reference and the ports mirror it; and semantically a client that renders tool calls would otherwise show "calling `request_identity_intake`…" before the card appears, leaking framework internals ahead of the semantic event. The four ports change, not these scenarios.
+- **~~A cancelled turn keeps running in Go and .NET~~ — FIXED (th-f2ac48, PR #514).** Recorded because it is what this scenario was built to catch, and because the fix is the corpus's first end-to-end proof of itself. Both ports used to leave the turn running after a `cancel`: the write-confirmation gate returned a deny instead of unwinding, the agent loop made one more model call, and the output was merely gagged (Go: `if turnCtx.Err() != nil { return }`). Cancellation was a mute button, not a stop button — real spend and real side-effect risk after a visitor hits Stop. Two independent proofs: re-running with one extra `mockLlmScript` entry made both pass (the entry was eaten by the cancelled turn), and `go test -race` reported a `DATA RACE` in core's `MockLlmProvider.ChatStream` where the cancelled turn's goroutine and the *next* turn's goroutine popped the same unguarded FIFO concurrently. That race was the one failure `knownDivergences` deliberately did **not** tolerate — it fires outside the runner's assertion path, and suppressing a data race is the opposite of what this corpus is for. The lesson if it recurs: do not "fix" it by guarding the mock's FIFO, which silences the evidence and leaves the bug.
+- **Ack payloads differ, so only `status` is asserted on a `submit_interaction` ack.** The five servers put different fields in `data` (Go omits `kind`/`values`; Python omits `kind`, and its decline ack omits `interactionId`/`declined`; .NET's decline ack omits `declined`). Asserting more would pin one language's shape rather than the protocol's.
+
 ## Adding a scenario
 
-Drop a `*.json` here; every server's runner picks it up automatically. Cover: multi-turn, tool-call + `confirm_tool_action` (HITL), citations, auth gating, error frames, and graceful-drain (cancel mid-turn → the turn still finishes).
+Drop a `*.json` here; every server's runner picks it up automatically — there is **no per-language skip, allowlist or xfail mechanism in any of the five runners**, and no way to add one in JSON (unknown keys are silently ignored everywhere). A new scenario lands on all five simultaneously; gating one would mean editing four runners.
+
+Two portability rules that bite:
+
+- **Never assert a fixed number of `stream_token` events** — the mocks chunk text differently per language. Always `repeat` + `accumulate` + `assertAccumulated`, and only on the top-level `token` field.
+- **Never assert `null`** to mean "field absent" — .NET's dot-path resolver returns `null` for a missing final segment while the other four fail, so such an assertion passes on exactly one server.
+
+Still uncovered: auth gating, and graceful-drain (disconnect mid-turn → the turn still finishes).