From 0cdd07d516ca8226dfbcc4b44a7b38508b31e08e Mon Sep 17 00:00:00 2001 From: olddonkey Date: Sun, 16 Aug 2026 21:43:02 -0700 Subject: [PATCH] test(fastwire): lock current service-tier behavior before the A1 refactor Phase A0 of the FastWire umbrella (lidge-jun/opencodex#1886): characterization suites only, zero production changes. Locks the three-state fastMode contract, the unclassified passthrough matrix, exact-model foreign-tier forwarding, the mixed requestedServiceTier logging semantics, routing/fingerprint projections, catalog tier-field byte goldens, and two known bugs (native chat ignores exact-model false; chat-to-responses conversion drops service_tier) as current behavior. Full suite at this commit: 12746 pass / 10 skip / 0 fail. Co-Authored-By: Claude Fable 5 --- .../fastwire-characterization-routing.test.ts | 132 ++++++++++ tests/fastwire-characterization-wire.test.ts | 239 ++++++++++++++++++ 2 files changed, 371 insertions(+) create mode 100644 tests/fastwire-characterization-routing.test.ts create mode 100644 tests/fastwire-characterization-wire.test.ts diff --git a/tests/fastwire-characterization-routing.test.ts b/tests/fastwire-characterization-routing.test.ts new file mode 100644 index 0000000000..c1f4322296 --- /dev/null +++ b/tests/fastwire-characterization-routing.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, test } from "bun:test"; +import { applyCatalogModelMetadata } from "../src/codex/catalog/effort"; +import type { CatalogModel, RawEntry } from "../src/codex/catalog/parsing"; +import { candidateCapabilityEvidence } from "../src/routing/capability"; +import { resolveProductionBehaviorValues } from "../src/routing/compatibility/behavior"; +import { evaluatePolicyProfile } from "../src/routing/evaluator"; +import type { OcxConfig, OcxProviderConfig } from "../src/types"; + +describe("FastWire characterization: routing profile service-tier evidence", () => { + test("require.serviceTier sees supportsServiceTier=true plus chatServiceTier=false as unsupported", () => { + const provider: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://chat-no-tier.example.test/v1", + authMode: "key", + apiKey: "sk-test", + supportsServiceTier: true, + chatServiceTier: false, + }; + const config = { + port: 0, + defaultProvider: "chat-no-tier", + providers: { "chat-no-tier": provider }, + routingProfiles: { + fast: { + candidates: [{ provider: "chat-no-tier", model: "model" }], + require: { serviceTier: "supported" }, + }, + }, + } as OcxConfig; + const capability = candidateCapabilityEvidence(config, "chat-no-tier", "model"); + + expect(capability.serviceTier).toBe("unsupported"); + const result = evaluatePolicyProfile(config, "fast", {}, [{ + provider: "chat-no-tier", + model: "model", + capability, + }]); + expect(result.candidates[0]).toMatchObject({ + eligible: false, + requirements: [{ + id: "service-tier", + expected: "supported", + actual: "unsupported", + outcome: "unsatisfied", + }], + }); + expect(result.selectedIndex).toBeNull(); + }); +}); + +describe("FastWire characterization: compatibility fingerprint projection", () => { + const cases: Array<{ label: string; expected: boolean; provider: OcxProviderConfig }> = [ + { + label: "supported", + expected: true, + provider: { + adapter: "openai-responses", + baseUrl: "https://supported.example.test/v1", + supportsServiceTier: true, + }, + }, + { + label: "unsupported", + expected: false, + provider: { + adapter: "openai-responses", + baseUrl: "https://unsupported.example.test/v1", + supportsServiceTier: false, + }, + }, + ]; + + test.each(cases)("projects $label service-tier behavior", ({ label, expected, provider }) => { + const config = { + port: 0, + defaultProvider: label, + fastMode: true, + providers: { [label]: provider }, + } as OcxConfig; + const values = resolveProductionBehaviorValues( + config, + label, + "model", + provider, + "fastwire-characterization-salt", + ); + + expect(values?.["responses.serviceTier"]).toEqual({ + source: "provider_config", + value: expected, + }); + expect(values?.["runtime.fastMode"]).toEqual({ + source: "global_config", + value: true, + }); + }); +}); + +describe("FastWire characterization: catalog service-tier bytes", () => { + function apply(supportsServiceTier?: boolean): RawEntry { + const entry: RawEntry = {}; + const model: CatalogModel = { + id: "model", + provider: "fixture", + ...(supportsServiceTier === undefined ? {} : { supportsServiceTier }), + }; + applyCatalogModelMetadata(entry, model); + return entry; + } + + test("supportsServiceTier=true emits the current narrow byte golden", () => { + const entry = apply(true); + const projection = { + default_service_tier: entry.default_service_tier, + service_tiers: entry.service_tiers, + additional_speed_tiers: entry.additional_speed_tiers, + }; + expect(JSON.stringify(projection)).toBe( + '{"default_service_tier":null,"service_tiers":[{"id":"priority","name":"Fast","description":"1.5x speed, increased usage"}],"additional_speed_tiers":["fast"]}', + ); + }); + + test.each([ + { label: "false", supportsServiceTier: false }, + { label: "unset", supportsServiceTier: undefined }, + ])("supportsServiceTier=$label omits all catalog tier fields", ({ supportsServiceTier }) => { + const entry = apply(supportsServiceTier); + expect(entry).not.toHaveProperty("default_service_tier"); + expect(entry).not.toHaveProperty("service_tiers"); + expect(entry).not.toHaveProperty("additional_speed_tiers"); + }); +}); diff --git a/tests/fastwire-characterization-wire.test.ts b/tests/fastwire-characterization-wire.test.ts new file mode 100644 index 0000000000..f8291ab47a --- /dev/null +++ b/tests/fastwire-characterization-wire.test.ts @@ -0,0 +1,239 @@ +import { afterEach, describe, expect, spyOn, test } from "bun:test"; +import { buildOpenAIChatPassthroughRequest } from "../src/adapters/openai-chat"; +import { chatCompletionsToResponsesBody } from "../src/chat/inbound"; +import * as adapterResolveModule from "../src/server/adapter-resolve"; +import type { RequestLogContext } from "../src/server/request-log"; +import { handleResponses } from "../src/server/responses/core"; +import type { OcxConfig, OcxProviderConfig } from "../src/types"; + +const originalFetch = globalThis.fetch; + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +async function driveResponses(args: { + provider: OcxProviderConfig; + model?: string; + callerTier?: string; + fastMode?: boolean; +}): Promise<{ outboundBody: Record; logCtx: RequestLogContext }> { + const providerName = "fastwire-fixture"; + const model = args.model ?? "model"; + const bodies: Record[] = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body ?? "{}")) as Record); + return new Response("data: [DONE]\n\n", { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + }) as typeof fetch; + + const config = { + port: 0, + defaultProvider: providerName, + providers: { [providerName]: args.provider }, + ...(args.fastMode === undefined ? {} : { fastMode: args.fastMode }), + } as OcxConfig; + const logCtx: RequestLogContext = { model: "", provider: "" }; + const requestBody = { + model: `${providerName}/${model}`, + input: "ping", + stream: true, + ...(args.callerTier === undefined ? {} : { service_tier: args.callerTier }), + }; + + await handleResponses( + new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(requestBody), + }), + config, + logCtx, + {}, + ); + + expect(bodies).toHaveLength(1); + return { outboundBody: bodies[0]!, logCtx }; +} + +const supportedResponsesProvider = (): OcxProviderConfig => ({ + adapter: "openai-responses", + baseUrl: "https://supported.example.test/v1", + authMode: "key", + apiKey: "sk-test", + supportsServiceTier: true, +}); + +const unclassifiedResponsesProvider = (): OcxProviderConfig => ({ + adapter: "openai-responses", + baseUrl: "https://unclassified.example.test/v1", + authMode: "key", + apiKey: "sk-test", +}); + +describe("FastWire characterization: supported-route fastMode tri-state", () => { + test("fastMode=true overrides caller flex with priority", async () => { + const { outboundBody } = await driveResponses({ + provider: supportedResponsesProvider(), + callerTier: "flex", + fastMode: true, + }); + expect(outboundBody.service_tier).toBe("priority"); + }); + + test("fastMode=false removes the caller tier", async () => { + const { outboundBody } = await driveResponses({ + provider: supportedResponsesProvider(), + callerTier: "turbo-x", + fastMode: false, + }); + expect(outboundBody).not.toHaveProperty("service_tier"); + }); + + test("fastMode=undefined preserves caller flex", async () => { + const { outboundBody } = await driveResponses({ + provider: supportedResponsesProvider(), + callerTier: "flex", + }); + expect(outboundBody.service_tier).toBe("flex"); + }); +}); + +describe("FastWire characterization: unclassified support matrix", () => { + const cells = ([true, false, undefined] as const).flatMap(fastMode => + (["priority", "fast", "flex"] as const).map(callerTier => ({ fastMode, callerTier })) + ); + + test.each(cells)( + "support=undefined preserves caller $callerTier with fastMode=$fastMode", + async ({ fastMode, callerTier }) => { + const { outboundBody } = await driveResponses({ + provider: unclassifiedResponsesProvider(), + callerTier, + fastMode, + }); + expect(outboundBody.service_tier).toBe(callerTier); + }, + ); +}); + +describe("FastWire characterization: exact-model Chat tier forwarding", () => { + test.each(["flex", "turbo-x"])( + "exact model true forwards foreign caller tier %s without chatServiceTier", + async callerTier => { + const { outboundBody } = await driveResponses({ + provider: { + adapter: "openai-chat", + baseUrl: "https://chat.example.test/v1", + authMode: "key", + apiKey: "sk-test", + modelSupportsServiceTier: { model: true }, + }, + callerTier, + }); + expect(outboundBody.service_tier).toBe(callerTier); + }, + ); +}); + +describe("FastWire characterization: requestedServiceTier timing", () => { + test("records the raw caller tier when fastMode overrides the wire tier", async () => { + const { outboundBody, logCtx } = await driveResponses({ + provider: supportedResponsesProvider(), + callerTier: "flex", + fastMode: true, + }); + expect(outboundBody.service_tier).toBe("priority"); + expect(logCtx.requestedServiceTier).toBe("flex"); + }); + + test("clears the caller tier after an unsupported route strips it", async () => { + const { outboundBody, logCtx } = await driveResponses({ + provider: { + ...supportedResponsesProvider(), + supportsServiceTier: false, + }, + callerTier: "priority", + }); + expect(outboundBody).not.toHaveProperty("service_tier"); + expect(logCtx.requestedServiceTier).toBeUndefined(); + }); +}); + +describe("FastWire characterization: rawBody observation point", () => { + test("fastMode injection is visible in parsed._rawBody when the adapter is invoked", async () => { + let adapterRawBody: Record | undefined; + const adapterSpy = spyOn(adapterResolveModule, "resolveAdapter").mockReturnValue({ + name: "openai-responses", + passthrough: true, + async buildRequest(parsed) { + adapterRawBody = JSON.parse(JSON.stringify(parsed._rawBody)) as Record; + throw new Error("fastwire rawBody observation complete"); + }, + } as ReturnType); + + try { + const providerName = "fastwire-raw-body"; + const config = { + port: 0, + defaultProvider: providerName, + fastMode: true, + providers: { [providerName]: supportedResponsesProvider() }, + } as OcxConfig; + const request = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: `${providerName}/model`, + input: "ping", + stream: true, + service_tier: "flex", + }), + }); + + await expect(handleResponses(request, config, { model: "", provider: "" }, {})) + .rejects.toThrow("fastwire rawBody observation complete"); + // A1 intentionally moves fast-mode injection out of `_rawBody`; update this + // characterization when that observation point changes. + expect(adapterRawBody?.service_tier).toBe("priority"); + } finally { + adapterSpy.mockRestore(); + } + }); +}); + +describe("FastWire characterization: known bugs", () => { + test("characterization (known bug): native chat passthrough ignores exact-model false", () => { + const request = buildOpenAIChatPassthroughRequest( + { + adapter: "openai-chat", + baseUrl: "https://native-chat.example.test/v1", + authMode: "key", + apiKey: "sk-test", + supportsServiceTier: true, + chatServiceTier: true, + modelSupportsServiceTier: { model: false }, + }, + { + model: "model", + messages: [{ role: "user", content: "ping" }], + service_tier: "flex", + }, + "model", + false, + ); + const body = JSON.parse(request.body) as Record; + expect(body.service_tier).toBe("flex"); + }); + + test("characterization (known bug): chat-to-responses conversion drops service_tier", () => { + const body = chatCompletionsToResponsesBody({ + model: "model", + messages: [{ role: "user", content: "ping" }], + service_tier: "priority", + }); + expect(body).not.toHaveProperty("service_tier"); + }); +});