From 684b22a154f04a4e3f015eacff739f65ca65a5d7 Mon Sep 17 00:00:00 2001 From: Alice Moore <86723305+3mdistal@users.noreply.github.com> Date: Fri, 21 Aug 2026 00:41:02 +0000 Subject: [PATCH 1/9] fix: preserve Content database property mutations --- .changeset/calm-tools-preserve-inputs.md | 5 ++ packages/core/src/eval/agent-runner.ts | 3 + packages/core/src/eval/runner.spec.ts | 1 + packages/core/src/eval/types.ts | 5 ++ .../content/actions/add-database-item.ts | 4 +- .../content/actions/update-database-item.ts | 2 +- .../actions/upsert-database-item-by-key.ts | 4 +- templates/content/package.json | 1 + templates/content/parity/README.md | 13 +++- .../database-row-property-input.test.ts | 29 ++++++++ .../__tests__/eval-scenario-coverage.test.ts | 68 ++++++++++++++++++- templates/content/parity/eval-scenarios.ts | 22 ++++++ ...n-database-create-property-preservation.ts | 45 ++++++++++++ templates/content/parity/scenario-to-eval.ts | 56 +++++++++++++++ 14 files changed, 250 insertions(+), 8 deletions(-) create mode 100644 .changeset/calm-tools-preserve-inputs.md create mode 100644 templates/content/parity/__tests__/database-row-property-input.test.ts create mode 100644 templates/content/parity/run-database-create-property-preservation.ts diff --git a/.changeset/calm-tools-preserve-inputs.md b/.changeset/calm-tools-preserve-inputs.md new file mode 100644 index 0000000000..268e397a4a --- /dev/null +++ b/.changeset/calm-tools-preserve-inputs.md @@ -0,0 +1,5 @@ +--- +"@agent-native/core": patch +--- + +Expose model-produced tool inputs to eval scorers so argument-level agent behavior can be verified. diff --git a/packages/core/src/eval/agent-runner.ts b/packages/core/src/eval/agent-runner.ts index b8599bd85b..3b370ae7f2 100644 --- a/packages/core/src/eval/agent-runner.ts +++ b/packages/core/src/eval/agent-runner.ts @@ -120,6 +120,7 @@ export async function createAgentRunner( let text = ""; const toolCalls: string[] = []; + const toolCallDetails: Array<{ name: string; input: unknown }> = []; let ok = true; let error: string | undefined; @@ -134,6 +135,7 @@ export async function createAgentRunner( break; case "tool_start": toolCalls.push(event.tool); + toolCallDetails.push({ name: event.tool, input: event.input }); break; case "error": ok = false; @@ -165,6 +167,7 @@ export async function createAgentRunner( return { text, toolCalls, + toolCallDetails, ok, error, runId, diff --git a/packages/core/src/eval/runner.spec.ts b/packages/core/src/eval/runner.spec.ts index fef74b3690..3fa997625c 100644 --- a/packages/core/src/eval/runner.spec.ts +++ b/packages/core/src/eval/runner.spec.ts @@ -299,6 +299,7 @@ describe("createAgentRunner over a mocked runAgentLoop (no real model)", () => { const out = await runner.runAgent({ prompt: "hi" }); expect(out.text).toBe("Hello world"); expect(out.toolCalls).toEqual(["search"]); + expect(out.toolCallDetails).toEqual([{ name: "search", input: {} }]); expect(out.ok).toBe(true); // End-to-end: a contains scorer over the real collected text. diff --git a/packages/core/src/eval/types.ts b/packages/core/src/eval/types.ts index 3648416981..a99a0849a9 100644 --- a/packages/core/src/eval/types.ts +++ b/packages/core/src/eval/types.ts @@ -31,6 +31,11 @@ export interface AgentRunOutput { readonly text: string; /** Names of tools/actions the agent invoked, in call order. */ readonly toolCalls: readonly string[]; + /** Tool names and model-produced inputs, in call order. */ + readonly toolCallDetails?: readonly { + readonly name: string; + readonly input: unknown; + }[]; /** Whether the run completed without a terminal error event. */ readonly ok: boolean; /** Terminal error message, if the run errored. */ diff --git a/templates/content/actions/add-database-item.ts b/templates/content/actions/add-database-item.ts index 318f9984a2..35ca108741 100644 --- a/templates/content/actions/add-database-item.ts +++ b/templates/content/actions/add-database-item.ts @@ -20,7 +20,9 @@ const schema = databaseMutationEnvelopeSchema.extend({ propertyValues: z .record(z.string(), z.unknown()) .optional() - .describe("Strict property values keyed by property definition ID"), + .describe( + "Strict property values keyed by property definition ID. Use exact property definition IDs as keys. Include every schema-valid writable property value the user explicitly requested; when the request contains at least one such value, never pass an empty object. Do not invent or clear unmentioned properties.", + ), }); export default defineAction({ diff --git a/templates/content/actions/update-database-item.ts b/templates/content/actions/update-database-item.ts index 889c33c15d..3f79784ffc 100644 --- a/templates/content/actions/update-database-item.ts +++ b/templates/content/actions/update-database-item.ts @@ -20,7 +20,7 @@ const schema = databaseMutationEnvelopeSchema.extend({ .record(z.string(), z.unknown()) .optional() .describe( - "Sparse strict patch keyed by property definition ID; omitted fields are preserved and explicit null clears a value", + "Sparse strict patch keyed by property definition ID; omitted fields are preserved and explicit null clears a value. Use exact property definition IDs as keys. Include every schema-valid writable property value the user explicitly requested; when the request contains at least one such value, never pass an empty object. Do not invent or clear unmentioned properties.", ), }); diff --git a/templates/content/actions/upsert-database-item-by-key.ts b/templates/content/actions/upsert-database-item-by-key.ts index c9a86fb303..334d57f055 100644 --- a/templates/content/actions/upsert-database-item-by-key.ts +++ b/templates/content/actions/upsert-database-item-by-key.ts @@ -21,7 +21,9 @@ const schema = databaseMutationEnvelopeSchema.extend({ propertyValues: z .record(z.string(), z.unknown()) .optional() - .describe("Sparse strict values keyed by property definition ID"), + .describe( + "Sparse strict values keyed by property definition ID. Use exact property definition IDs as keys. Include every schema-valid writable property value the user explicitly requested; when the request contains at least one such value, never pass an empty object. Do not invent or clear unmentioned properties.", + ), }); export default defineAction({ diff --git a/templates/content/package.json b/templates/content/package.json index 5c5e767d47..6cead972bc 100644 --- a/templates/content/package.json +++ b/templates/content/package.json @@ -12,6 +12,7 @@ "test:parity": "vitest --run parity", "test:parity-capabilities": "vitest --run parity actions/content-database-lifecycle.db.test.ts actions/bind-content-database-source-field.db.test.ts actions/_local-file-documents.test.ts actions/builder-source-review-gates.db.test.ts", "eval:parity": "agent-native eval parity", + "eval:property-preservation": "tsx parity/run-database-create-property-preservation.ts", "format.fix": "oxfmt --write .", "typecheck": "agent-native typecheck", "migrate:production": "tsx scripts/migrate-production.ts", diff --git a/templates/content/parity/README.md b/templates/content/parity/README.md index 05ac6c5b35..4c84876e57 100644 --- a/templates/content/parity/README.md +++ b/templates/content/parity/README.md @@ -10,7 +10,7 @@ PR 2.2 adds two executable tiers: existing action tests, make no model calls, and require no private provider credentials. - Gated agent evals run through `agent-native eval parity`. They are opt-in via - `CONTENT_PARITY_EVALS=1`, capped to four initial scenarios, and should be + `CONTENT_PARITY_EVALS=1`, kept to a small explicit scenario set, and should be reserved for manual or nightly checks. ## Deterministic Checks @@ -33,14 +33,23 @@ cd templates/content CONTENT_PARITY_EVALS=1 ANTHROPIC_API_KEY=... ./node_modules/.bin/agent-native eval parity ``` +The database-create property-preservation regression has a dedicated +fixture-only runner so it can inspect model-produced arguments without loading +or executing unrelated Content actions: + +```bash +CONTENT_PARITY_EVALS=1 pnpm eval:property-preservation +``` + With `CONTENT_PARITY_EVALS` unset, parity evals return skipped rows and do not call the agent runner. The CLI still exits `0`, but both readable and JSON reports mark each row with `status: "skipped"` and a `skipReason` such as `Skipped because CONTENT_PARITY_EVALS is unset`. -With the gate set, the eval files run the four PR 2.2 scenarios: +With the gate set, the eval files include these scenarios: - `database-source-scope` +- `database-create-property-preservation` - `document-search-edit` - `local-file-source-truth` - `builder-source-review-readonly` diff --git a/templates/content/parity/__tests__/database-row-property-input.test.ts b/templates/content/parity/__tests__/database-row-property-input.test.ts new file mode 100644 index 0000000000..e3ed8af8a2 --- /dev/null +++ b/templates/content/parity/__tests__/database-row-property-input.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; + +import addDatabaseItem from "../../actions/add-database-item"; +import updateDatabaseItem from "../../actions/update-database-item"; +import upsertDatabaseItemByKey from "../../actions/upsert-database-item-by-key"; + +const rowMutationActions = [ + ["add-database-item", addDatabaseItem], + ["update-database-item", updateDatabaseItem], + ["upsert-database-item-by-key", upsertDatabaseItemByKey], +] as const; + +describe("database row property inputs", () => { + it.each(rowMutationActions)( + "%s tells the agent to preserve explicitly requested writable values", + (_name, action) => { + const propertyValues = action.tool.parameters.properties.propertyValues; + expect(propertyValues.description).toContain( + "Include every schema-valid writable property value the user explicitly requested", + ); + expect(propertyValues.description).toContain( + "never pass an empty object", + ); + expect(propertyValues.description).toContain( + "Do not invent or clear unmentioned properties", + ); + }, + ); +}); diff --git a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts index 65fb938762..b3d39b7935 100644 --- a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts +++ b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts @@ -37,10 +37,11 @@ describe("Content parity eval scenarios", () => { expect(invalid).toEqual([]); }); - it("keeps PR 2.5 capped to the five bundled gated scenarios", () => { + it("keeps the bundled gated scenarios explicit", () => { expect(parityEvalScenarios.map((scenario) => scenario.id).sort()).toEqual([ "builder-source-review-readonly", "database-bulk-row-reliability", + "database-create-property-preservation", "database-source-scope", "document-search-edit", "local-file-source-truth", @@ -83,14 +84,16 @@ describe("Content parity eval scenarios", () => { ); expect(report.failed).toBe(0); - expect(report.skipped).toBe(5); + expect(report.skipped).toBe(6); expect(report.results.every((row) => row.status === "skipped")).toBe(true); }); it("runs scorer-backed evals when the gate is set", async () => { process.env.CONTENT_PARITY_EVALS = "1"; - const scenario = parityEvalScenarios[0]; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-source-scope", + )!; const evalCase = scenarioToEval(scenario); const row = await scoreEval(evalCase, { runAgent: vi.fn(async () => ({ @@ -141,4 +144,63 @@ describe("Content parity eval scenarios", () => { expect(expectedToolsScore?.reason).toContain("remove-database-items"); expect(row.status).toBe("failed"); }); + + it("fails when database creation drops explicitly requested properties", async () => { + process.env.CONTENT_PARITY_EVALS = "1"; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", + )!; + const evalCase = scenarioToEval(scenario); + const row = await scoreEval(evalCase, { + runAgent: vi.fn(async () => ({ + text: scenario.successSignals.join("\n"), + toolCalls: ["add-database-item"], + toolCallDetails: [ + { name: "add-database-item", input: { propertyValues: {} } }, + ], + ok: true, + runId: "content-parity:empty-property-values", + durationMs: 1, + })), + engine: {} as never, + model: "test-model", + analyzeContext: vi.fn(), + }); + + expect( + row.scores.find((score) => score.scorer === "expected_property_values"), + ).toMatchObject({ passed: false, score: 0 }); + expect(row.status).toBe("failed"); + }); + + it("accepts exact database creation properties without extras", async () => { + process.env.CONTENT_PARITY_EVALS = "1"; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", + )!; + const evalCase = scenarioToEval(scenario); + const row = await scoreEval(evalCase, { + runAgent: vi.fn(async () => ({ + text: scenario.successSignals.join("\n"), + toolCalls: ["add-database-item"], + toolCallDetails: [ + { + name: "add-database-item", + input: { propertyValues: scenario.expectedPropertyValues }, + }, + ], + ok: true, + runId: "content-parity:exact-property-values", + durationMs: 1, + })), + engine: {} as never, + model: "test-model", + analyzeContext: vi.fn(), + }); + + expect( + row.scores.find((score) => score.scorer === "expected_property_values"), + ).toMatchObject({ passed: true, score: 1 }); + expect(row.status).toBe("passed"); + }); }); diff --git a/templates/content/parity/eval-scenarios.ts b/templates/content/parity/eval-scenarios.ts index b8bcfb76dd..996de81c62 100644 --- a/templates/content/parity/eval-scenarios.ts +++ b/templates/content/parity/eval-scenarios.ts @@ -8,9 +8,31 @@ export interface ParityEvalScenario { prompt: string; successSignals: string[]; expectedTools?: string[]; + expectedPropertyValues?: Record; } export const parityEvalScenarios: ParityEvalScenario[] = [ + { + id: "database-create-property-preservation", + title: "Database create property preservation", + capabilityIds: ["database.rows"], + gateEnv: "CONTENT_PARITY_EVALS", + defaultState: "skipped", + requiresPrivateCredentials: false, + prompt: + "Create exactly one row in the already-verified fixture Content database. Do not rediscover or alter the target. Use target spaceId fixture_personal_space, databaseId fixture_feedback_database, and documentId fixture_feedback_document; expectedSchemaRevision fixture_schema_revision; title [FIXTURE] preserve explicit properties; idempotencyKey fixture-create-property-preservation-v1. Set property fixture_status_property to status-cannot-verify and property fixture_evidence_property to Baseline fixture preserve-me. No Blocks value was requested. Call add-database-item once with every exact target constraint and both property values, then report its result truthfully.", + successSignals: [ + "Uses add-database-item once for the exact fixture target.", + "Preserves both explicitly requested writable property values.", + "Does not invent a Blocks value or another property.", + "Reports an action failure rather than claiming a row was created if the fixture is unavailable.", + ], + expectedTools: ["add-database-item"], + expectedPropertyValues: { + fixture_status_property: "status-cannot-verify", + fixture_evidence_property: "Baseline fixture preserve-me", + }, + }, { id: "database-bulk-row-reliability", title: "Bulk database row reliability", diff --git a/templates/content/parity/run-database-create-property-preservation.ts b/templates/content/parity/run-database-create-property-preservation.ts new file mode 100644 index 0000000000..1713a9fd52 --- /dev/null +++ b/templates/content/parity/run-database-create-property-preservation.ts @@ -0,0 +1,45 @@ +import { createAgentRunner, runEvals } from "@agent-native/core/eval"; + +import addDatabaseItem from "../actions/add-database-item.ts"; +import { parityEvalScenarios } from "./eval-scenarios.ts"; +import { scenarioToEval } from "./scenario-to-eval.ts"; + +const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", +); +if (!scenario) { + throw new Error("Missing database create property preservation scenario."); +} + +const evalCase = scenarioToEval(scenario); +evalCase.scorers = evalCase.scorers.filter( + (scorer) => + scorer.name === "expected_tools" || + scorer.name === "expected_property_values", +); + +const runner = await createAgentRunner({ + actions: { + "add-database-item": { + ...addDatabaseItem, + run: async (input) => ({ fixtureOnly: true, received: input }), + }, + }, + systemPrompt: + "You are Content's AI document assistant. Use the registered Content action and preserve exact user-supplied target constraints, property IDs, and property values. Never invent fields or claim an action succeeded when it failed.", +}); + +const report = await runEvals([evalCase], runner, { persist: false }); +console.log( + JSON.stringify( + { + engine: runner.engine.name, + model: runner.model, + report, + }, + null, + 2, + ), +); + +process.exitCode = report.failed === 0 ? 0 : 1; diff --git a/templates/content/parity/scenario-to-eval.ts b/templates/content/parity/scenario-to-eval.ts index 6385b1d8c2..103d45cab1 100644 --- a/templates/content/parity/scenario-to-eval.ts +++ b/templates/content/parity/scenario-to-eval.ts @@ -30,6 +30,59 @@ function expectedToolScorer(expectedTools: string[]) { }); } +function normalizePropertyValues(input: unknown): Record { + if (!input || typeof input !== "object" || Array.isArray(input)) return {}; + const propertyValues = (input as Record).propertyValues; + if (!propertyValues) return {}; + if (!Array.isArray(propertyValues)) { + return typeof propertyValues === "object" + ? (propertyValues as Record) + : {}; + } + return Object.fromEntries( + propertyValues.flatMap((entry) => { + if (!entry || typeof entry !== "object") return []; + const { propertyId, value } = entry as Record; + return typeof propertyId === "string" ? [[propertyId, value]] : []; + }), + ); +} + +function expectedPropertyValuesScorer(expected: Record) { + return createScorer< + AgentRunOutput, + { + received: Record; + missing: string[]; + unexpected: string[]; + } + >({ + name: "expected_property_values", + analyze(run) { + const detail = run.toolCallDetails?.find( + (call) => call.name === "add-database-item", + ); + const received = normalizePropertyValues(detail?.input); + const missing = Object.entries(expected) + .filter(([propertyId, value]) => received[propertyId] !== value) + .map(([propertyId]) => propertyId); + const unexpected = Object.keys(received).filter( + (propertyId) => !(propertyId in expected), + ); + return { received, missing, unexpected }; + }, + generateScore({ missing, unexpected }) { + return missing.length === 0 && unexpected.length === 0 ? 1 : 0; + }, + generateReason({ analysis: { received, missing, unexpected } }) { + if (missing.length === 0 && unexpected.length === 0) { + return "Agent preserved every expected property ID and exact value without inventing another property."; + } + return `Received propertyValues ${JSON.stringify(received)}; missing or changed: ${missing.join(", ") || "none"}; unexpected: ${unexpected.join(", ") || "none"}`; + }, + }); +} + export function scenarioToEval(scenario: ParityEvalScenario): Eval { const name = `content-parity:${scenario.id}`; @@ -52,6 +105,9 @@ export function scenarioToEval(scenario: ParityEvalScenario): Eval { ...(scenario.expectedTools?.length ? [expectedToolScorer(scenario.expectedTools)] : []), + ...(scenario.expectedPropertyValues + ? [expectedPropertyValuesScorer(scenario.expectedPropertyValues)] + : []), ], }); } From 74eb67f387a624a2555159a14c0c2f4cf086d6e5 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 20 Aug 2026 21:17:39 -0400 Subject: [PATCH 2/9] fix: normalize Content AI property entries --- .../actions/_database-property-input.ts | 53 +++++++++++++++++++ .../content/actions/add-database-item.ts | 19 ++++--- .../content/actions/update-database-item.ts | 22 +++++--- .../actions/upsert-database-item-by-key.ts | 22 +++++--- .../database-row-property-input.test.ts | 53 ++++++++++++++++--- templates/content/parity/eval-scenarios.ts | 2 +- templates/content/parity/scenario-to-eval.ts | 4 +- 7 files changed, 146 insertions(+), 29 deletions(-) create mode 100644 templates/content/actions/_database-property-input.ts diff --git a/templates/content/actions/_database-property-input.ts b/templates/content/actions/_database-property-input.ts new file mode 100644 index 0000000000..dd98815eb2 --- /dev/null +++ b/templates/content/actions/_database-property-input.ts @@ -0,0 +1,53 @@ +import { ActionContractError } from "@agent-native/core"; +import { z } from "zod"; + +export const databasePropertyValuesSchema = z + .record(z.string(), z.unknown()) + .optional() + .describe( + "Programmatic property values keyed by exact property definition ID.", + ); + +export const databasePropertyEntriesSchema = z + .array( + z.object({ + propertyId: z + .string() + .min(1) + .describe("Exact immutable property definition ID"), + value: z.unknown().describe("Schema-valid value for this property"), + }), + ) + .max(1_000) + .optional() + .describe( + "Property values as explicit entries. Include one entry for every schema-valid writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", + ); + +export function normalizeDatabasePropertyInput(input: { + propertyEntries?: Array<{ propertyId: string; value: unknown }>; + propertyValues?: Record; +}): Record | undefined { + if (input.propertyEntries && input.propertyValues) { + throw new ActionContractError( + "Provide propertyEntries or propertyValues, not both.", + { errorCode: "AMBIGUOUS_PROPERTY_INPUT" }, + ); + } + if (!input.propertyEntries) return input.propertyValues; + + const values: Record = {}; + for (const entry of input.propertyEntries) { + if (Object.prototype.hasOwnProperty.call(values, entry.propertyId)) { + throw new ActionContractError( + `Property entry ${entry.propertyId} was provided more than once.`, + { + errorCode: "DUPLICATE_PROPERTY_INPUT", + details: { propertyId: entry.propertyId }, + }, + ); + } + values[entry.propertyId] = entry.value; + } + return values; +} diff --git a/templates/content/actions/add-database-item.ts b/templates/content/actions/add-database-item.ts index 35ca108741..629872af8b 100644 --- a/templates/content/actions/add-database-item.ts +++ b/templates/content/actions/add-database-item.ts @@ -3,6 +3,11 @@ import { buildDeepLink } from "@agent-native/core/server"; import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; +import { + databasePropertyEntriesSchema, + databasePropertyValuesSchema, + normalizeDatabasePropertyInput, +} from "./_database-property-input.js"; import { createDatabaseRow, databaseMutationEnvelopeSchema, @@ -17,17 +22,14 @@ const schema = databaseMutationEnvelopeSchema.extend({ .max(500) .optional() .describe("New row page title"), - propertyValues: z - .record(z.string(), z.unknown()) - .optional() - .describe( - "Strict property values keyed by property definition ID. Use exact property definition IDs as keys. Include every schema-valid writable property value the user explicitly requested; when the request contains at least one such value, never pass an empty object. Do not invent or clear unmentioned properties.", - ), + propertyValues: databasePropertyValuesSchema, + propertyEntries: databasePropertyEntriesSchema, }); export default defineAction({ description: "Create one row in an exact ordinary Content database using its discovered schema revision. Strictly validates every non-Blocks property, applies the side effect once per idempotency key, and returns a verified receipt with stable row identity.", + agentInputSchema: schema.omit({ propertyValues: true }), publicAgent: { expose: true, readOnly: false, @@ -54,7 +56,10 @@ export default defineAction({ }, }, run: async (args): Promise => { - const result = await createDatabaseRow(args); + const result = await createDatabaseRow({ + ...args, + propertyValues: normalizeDatabasePropertyInput(args), + }); const response = await getContentDatabaseResponse( result.receipt.target.databaseId, { diff --git a/templates/content/actions/update-database-item.ts b/templates/content/actions/update-database-item.ts index 3f79784ffc..66bac08d6b 100644 --- a/templates/content/actions/update-database-item.ts +++ b/templates/content/actions/update-database-item.ts @@ -3,6 +3,11 @@ import { buildDeepLink } from "@agent-native/core/server"; import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; +import { + databasePropertyEntriesSchema, + databasePropertyValuesSchema, + normalizeDatabasePropertyInput, +} from "./_database-property-input.js"; import { databaseMutationEnvelopeSchema, updateDatabaseRow, @@ -16,17 +21,16 @@ const schema = databaseMutationEnvelopeSchema.extend({ .min(1) .describe("Row revision returned by get-content-database"), title: z.string().trim().min(1).max(500).optional(), - propertyValues: z - .record(z.string(), z.unknown()) - .optional() - .describe( - "Sparse strict patch keyed by property definition ID; omitted fields are preserved and explicit null clears a value. Use exact property definition IDs as keys. Include every schema-valid writable property value the user explicitly requested; when the request contains at least one such value, never pass an empty object. Do not invent or clear unmentioned properties.", - ), + propertyValues: databasePropertyValuesSchema, + propertyEntries: databasePropertyEntriesSchema.describe( + "Sparse property patch as explicit entries; omitted fields are preserved and explicit null clears a value. Include one entry for every schema-valid writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", + ), }); export default defineAction({ description: "Sparsely update one exact Content database row by stable item and document IDs. Requires schema and row revisions, validates every non-Blocks property, and returns a verified idempotent receipt.", + agentInputSchema: schema.omit({ propertyValues: true }), schema, http: { method: "PUT" }, audit: { @@ -44,7 +48,11 @@ export default defineAction({ : "Updated Content database row"; }, }, - run: updateDatabaseRow, + run: (args) => + updateDatabaseRow({ + ...args, + propertyValues: normalizeDatabasePropertyInput(args), + }), link: ({ result }) => { const documentId = (result as ContentDatabaseRowMutationResult | null) ?.receipt.row.documentId; diff --git a/templates/content/actions/upsert-database-item-by-key.ts b/templates/content/actions/upsert-database-item-by-key.ts index 334d57f055..8dbcb7f603 100644 --- a/templates/content/actions/upsert-database-item-by-key.ts +++ b/templates/content/actions/upsert-database-item-by-key.ts @@ -3,6 +3,11 @@ import { buildDeepLink } from "@agent-native/core/server"; import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; +import { + databasePropertyEntriesSchema, + databasePropertyValuesSchema, + normalizeDatabasePropertyInput, +} from "./_database-property-input.js"; import { databaseMutationEnvelopeSchema, upsertDatabaseRow, @@ -18,17 +23,16 @@ const schema = databaseMutationEnvelopeSchema.extend({ "Use null to assert the key is absent and create; use the discovered row revision to update an existing key", ), title: z.string().trim().min(1).max(500).optional(), - propertyValues: z - .record(z.string(), z.unknown()) - .optional() - .describe( - "Sparse strict values keyed by property definition ID. Use exact property definition IDs as keys. Include every schema-valid writable property value the user explicitly requested; when the request contains at least one such value, never pass an empty object. Do not invent or clear unmentioned properties.", - ), + propertyValues: databasePropertyValuesSchema, + propertyEntries: databasePropertyEntriesSchema.describe( + "Sparse property values as explicit entries. Include one entry for every schema-valid writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", + ), }); export default defineAction({ description: "Create or sparsely update one Content database row by that database's explicitly configured natural key. Requires schema and row compare-and-swap revisions and returns a verified idempotent receipt.", + agentInputSchema: schema.omit({ propertyValues: true }), schema, audit: { recordInputs: false, @@ -45,7 +49,11 @@ export default defineAction({ : "Upserted Content database row by natural key"; }, }, - run: upsertDatabaseRow, + run: (args) => + upsertDatabaseRow({ + ...args, + propertyValues: normalizeDatabasePropertyInput(args), + }), link: ({ result }) => { const documentId = (result as ContentDatabaseRowMutationResult | null) ?.receipt.row.documentId; diff --git a/templates/content/parity/__tests__/database-row-property-input.test.ts b/templates/content/parity/__tests__/database-row-property-input.test.ts index e3ed8af8a2..2c2338289e 100644 --- a/templates/content/parity/__tests__/database-row-property-input.test.ts +++ b/templates/content/parity/__tests__/database-row-property-input.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; +import { normalizeDatabasePropertyInput } from "../../actions/_database-property-input"; import addDatabaseItem from "../../actions/add-database-item"; import updateDatabaseItem from "../../actions/update-database-item"; import upsertDatabaseItemByKey from "../../actions/upsert-database-item-by-key"; @@ -14,16 +15,56 @@ describe("database row property inputs", () => { it.each(rowMutationActions)( "%s tells the agent to preserve explicitly requested writable values", (_name, action) => { - const propertyValues = action.tool.parameters.properties.propertyValues; - expect(propertyValues.description).toContain( - "Include every schema-valid writable property value the user explicitly requested", + const properties = action.tool.parameters.properties; + expect(properties).not.toHaveProperty("propertyValues"); + const propertyEntries = properties.propertyEntries; + expect(propertyEntries.type).toBe("array"); + expect(propertyEntries.items.properties.propertyId.description).toContain( + "Exact immutable property definition ID", ); - expect(propertyValues.description).toContain( - "never pass an empty object", + expect(propertyEntries.description).toContain( + "Include one entry for every schema-valid writable property value the user requested", ); - expect(propertyValues.description).toContain( + expect(propertyEntries.description).toContain( + "never pass an empty array", + ); + expect(propertyEntries.description).toContain( "Do not invent or clear unmentioned properties", ); }, ); + + it("normalizes model-friendly entries into the strict action contract", () => { + expect( + normalizeDatabasePropertyInput({ + propertyEntries: [ + { propertyId: "status-id", value: "ready" }, + { propertyId: "evidence-id", value: "preserve me" }, + ], + }), + ).toEqual({ + "status-id": "ready", + "evidence-id": "preserve me", + }); + }); + + it("rejects duplicate property entries instead of silently overwriting", () => { + expect(() => + normalizeDatabasePropertyInput({ + propertyEntries: [ + { propertyId: "status-id", value: "ready" }, + { propertyId: "status-id", value: "changed" }, + ], + }), + ).toThrow(/provided more than once/); + }); + + it("rejects ambiguous entry and record inputs", () => { + expect(() => + normalizeDatabasePropertyInput({ + propertyEntries: [{ propertyId: "status-id", value: "ready" }], + propertyValues: { "status-id": "ready" }, + }), + ).toThrow(/not both/); + }); }); diff --git a/templates/content/parity/eval-scenarios.ts b/templates/content/parity/eval-scenarios.ts index 996de81c62..c459e1f376 100644 --- a/templates/content/parity/eval-scenarios.ts +++ b/templates/content/parity/eval-scenarios.ts @@ -20,7 +20,7 @@ export const parityEvalScenarios: ParityEvalScenario[] = [ defaultState: "skipped", requiresPrivateCredentials: false, prompt: - "Create exactly one row in the already-verified fixture Content database. Do not rediscover or alter the target. Use target spaceId fixture_personal_space, databaseId fixture_feedback_database, and documentId fixture_feedback_document; expectedSchemaRevision fixture_schema_revision; title [FIXTURE] preserve explicit properties; idempotencyKey fixture-create-property-preservation-v1. Set property fixture_status_property to status-cannot-verify and property fixture_evidence_property to Baseline fixture preserve-me. No Blocks value was requested. Call add-database-item once with every exact target constraint and both property values, then report its result truthfully.", + "Create exactly one row in the already-verified fixture Content database. Do not rediscover or alter the target. Use target spaceId fixture_personal_space, databaseId fixture_feedback_database, and documentId fixture_feedback_document; expectedSchemaRevision fixture_schema_revision; title [FIXTURE] preserve explicit properties; idempotencyKey fixture-create-property-preservation-v1. Set property fixture_status_property to status-cannot-verify and property fixture_evidence_property to Baseline fixture preserve-me. No Blocks value was requested. Call add-database-item once with every exact target constraint and both property entries, then report its result truthfully.", successSignals: [ "Uses add-database-item once for the exact fixture target.", "Preserves both explicitly requested writable property values.", diff --git a/templates/content/parity/scenario-to-eval.ts b/templates/content/parity/scenario-to-eval.ts index 103d45cab1..c657048da4 100644 --- a/templates/content/parity/scenario-to-eval.ts +++ b/templates/content/parity/scenario-to-eval.ts @@ -32,7 +32,9 @@ function expectedToolScorer(expectedTools: string[]) { function normalizePropertyValues(input: unknown): Record { if (!input || typeof input !== "object" || Array.isArray(input)) return {}; - const propertyValues = (input as Record).propertyValues; + const propertyValues = + (input as Record).propertyEntries ?? + (input as Record).propertyValues; if (!propertyValues) return {}; if (!Array.isArray(propertyValues)) { return typeof propertyValues === "object" From fd85e17054cb04954cd150e1485e44bc94b36f0b Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Fri, 21 Aug 2026 07:43:48 -0400 Subject: [PATCH 3/9] fix: canonicalize Content mutation inputs --- .../actions/_database-property-input.ts | 20 +++ .../content/actions/add-database-item.ts | 9 +- .../content/actions/update-database-item.ts | 8 +- .../actions/upsert-database-item-by-key.ts | 8 +- .../database-row-property-input.test.ts | 44 ++++++- .../__tests__/eval-scenario-coverage.test.ts | 75 +++++++++++ templates/content/parity/scenario-to-eval.ts | 119 ++++++++++++++---- 7 files changed, 240 insertions(+), 43 deletions(-) diff --git a/templates/content/actions/_database-property-input.ts b/templates/content/actions/_database-property-input.ts index dd98815eb2..30552617c3 100644 --- a/templates/content/actions/_database-property-input.ts +++ b/templates/content/actions/_database-property-input.ts @@ -51,3 +51,23 @@ export function normalizeDatabasePropertyInput(input: { } return values; } + +export function canonicalizeDatabasePropertyInput< + T extends { + propertyEntries?: Array<{ propertyId: string; value: unknown }>; + propertyValues?: Record; + }, +>( + input: T, +): Omit & { + propertyValues?: Record; +} { + const { propertyEntries, propertyValues, ...canonicalInput } = input; + return { + ...canonicalInput, + propertyValues: normalizeDatabasePropertyInput({ + propertyEntries, + propertyValues, + }), + }; +} diff --git a/templates/content/actions/add-database-item.ts b/templates/content/actions/add-database-item.ts index 629872af8b..8666c33bdf 100644 --- a/templates/content/actions/add-database-item.ts +++ b/templates/content/actions/add-database-item.ts @@ -4,9 +4,9 @@ import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; import { + canonicalizeDatabasePropertyInput, databasePropertyEntriesSchema, databasePropertyValuesSchema, - normalizeDatabasePropertyInput, } from "./_database-property-input.js"; import { createDatabaseRow, @@ -56,10 +56,9 @@ export default defineAction({ }, }, run: async (args): Promise => { - const result = await createDatabaseRow({ - ...args, - propertyValues: normalizeDatabasePropertyInput(args), - }); + const result = await createDatabaseRow( + canonicalizeDatabasePropertyInput(args), + ); const response = await getContentDatabaseResponse( result.receipt.target.databaseId, { diff --git a/templates/content/actions/update-database-item.ts b/templates/content/actions/update-database-item.ts index 66bac08d6b..4fcd904e53 100644 --- a/templates/content/actions/update-database-item.ts +++ b/templates/content/actions/update-database-item.ts @@ -4,9 +4,9 @@ import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; import { + canonicalizeDatabasePropertyInput, databasePropertyEntriesSchema, databasePropertyValuesSchema, - normalizeDatabasePropertyInput, } from "./_database-property-input.js"; import { databaseMutationEnvelopeSchema, @@ -48,11 +48,7 @@ export default defineAction({ : "Updated Content database row"; }, }, - run: (args) => - updateDatabaseRow({ - ...args, - propertyValues: normalizeDatabasePropertyInput(args), - }), + run: (args) => updateDatabaseRow(canonicalizeDatabasePropertyInput(args)), link: ({ result }) => { const documentId = (result as ContentDatabaseRowMutationResult | null) ?.receipt.row.documentId; diff --git a/templates/content/actions/upsert-database-item-by-key.ts b/templates/content/actions/upsert-database-item-by-key.ts index 8dbcb7f603..93dcb1ef43 100644 --- a/templates/content/actions/upsert-database-item-by-key.ts +++ b/templates/content/actions/upsert-database-item-by-key.ts @@ -4,9 +4,9 @@ import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; import { + canonicalizeDatabasePropertyInput, databasePropertyEntriesSchema, databasePropertyValuesSchema, - normalizeDatabasePropertyInput, } from "./_database-property-input.js"; import { databaseMutationEnvelopeSchema, @@ -49,11 +49,7 @@ export default defineAction({ : "Upserted Content database row by natural key"; }, }, - run: (args) => - upsertDatabaseRow({ - ...args, - propertyValues: normalizeDatabasePropertyInput(args), - }), + run: (args) => upsertDatabaseRow(canonicalizeDatabasePropertyInput(args)), link: ({ result }) => { const documentId = (result as ContentDatabaseRowMutationResult | null) ?.receipt.row.documentId; diff --git a/templates/content/parity/__tests__/database-row-property-input.test.ts b/templates/content/parity/__tests__/database-row-property-input.test.ts index 2c2338289e..aaa40f1e3d 100644 --- a/templates/content/parity/__tests__/database-row-property-input.test.ts +++ b/templates/content/parity/__tests__/database-row-property-input.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from "vitest"; -import { normalizeDatabasePropertyInput } from "../../actions/_database-property-input"; +import { + canonicalizeDatabasePropertyInput, + normalizeDatabasePropertyInput, +} from "../../actions/_database-property-input"; +import { digest } from "../../actions/_database-row-mutation"; import addDatabaseItem from "../../actions/add-database-item"; import updateDatabaseItem from "../../actions/update-database-item"; import upsertDatabaseItemByKey from "../../actions/upsert-database-item-by-key"; @@ -67,4 +71,42 @@ describe("database row property inputs", () => { }), ).toThrow(/not both/); }); + + it("removes the model-only representation before canonical hashing", () => { + const canonical = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyEntries: [ + { propertyId: "status-id", value: "ready" }, + { propertyId: "evidence-id", value: "preserve me" }, + ], + }); + + expect(canonical).toEqual({ + idempotencyKey: "same-intent", + propertyValues: { + "status-id": "ready", + "evidence-id": "preserve me", + }, + }); + expect(canonical).not.toHaveProperty("propertyEntries"); + }); + + it("gives equivalent entry and record inputs the same canonical digest", () => { + const fromEntries = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyEntries: [ + { propertyId: "status-id", value: "ready" }, + { propertyId: "evidence-id", value: "preserve me" }, + ], + }); + const fromRecord = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyValues: { + "evidence-id": "preserve me", + "status-id": "ready", + }, + }); + + expect(digest(fromEntries)).toBe(digest(fromRecord)); + }); }); diff --git a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts index b3d39b7935..f7aba9894c 100644 --- a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts +++ b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts @@ -203,4 +203,79 @@ describe("Content parity eval scenarios", () => { ).toMatchObject({ passed: true, score: 1 }); expect(row.status).toBe("passed"); }); + + it.each([ + { + name: "duplicate property entries", + toolCallDetails: [ + { + name: "add-database-item", + input: { + propertyEntries: [ + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-status-property-id", value: "ready" }, + ], + }, + }, + ], + }, + { + name: "ambiguous property formats", + toolCallDetails: [ + { + name: "add-database-item", + input: { + propertyEntries: [ + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-status-property-id", value: "ready" }, + ], + propertyValues: { + "parity-text-property-id": "preserve me", + "parity-status-property-id": "ready", + }, + }, + }, + ], + }, + { + name: "an extra row mutation", + toolCallDetails: [ + { + name: "add-database-item", + input: { + propertyEntries: [ + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-status-property-id", value: "ready" }, + ], + }, + }, + { name: "update-database-item", input: {} }, + ], + }, + ])("rejects $name", async ({ toolCallDetails }) => { + process.env.CONTENT_PARITY_EVALS = "1"; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", + )!; + const evalCase = scenarioToEval(scenario); + const row = await scoreEval(evalCase, { + runAgent: vi.fn(async () => ({ + text: scenario.successSignals.join("\n"), + toolCalls: toolCallDetails.map((call) => call.name), + toolCallDetails, + ok: true, + runId: "content-parity:invalid-property-input", + durationMs: 1, + })), + engine: {} as never, + model: "test-model", + analyzeContext: vi.fn(), + }); + + expect( + row.scores.find((score) => score.scorer === "expected_property_values"), + ).toMatchObject({ passed: false, score: 0 }); + expect(row.status).toBe("failed"); + }); }); diff --git a/templates/content/parity/scenario-to-eval.ts b/templates/content/parity/scenario-to-eval.ts index c657048da4..d6766456a3 100644 --- a/templates/content/parity/scenario-to-eval.ts +++ b/templates/content/parity/scenario-to-eval.ts @@ -30,26 +30,68 @@ function expectedToolScorer(expectedTools: string[]) { }); } -function normalizePropertyValues(input: unknown): Record { - if (!input || typeof input !== "object" || Array.isArray(input)) return {}; - const propertyValues = - (input as Record).propertyEntries ?? - (input as Record).propertyValues; - if (!propertyValues) return {}; - if (!Array.isArray(propertyValues)) { - return typeof propertyValues === "object" - ? (propertyValues as Record) - : {}; +function analyzePropertyValues(input: unknown): { + received: Record; + invalid: string[]; +} { + if (!input || typeof input !== "object" || Array.isArray(input)) { + return { received: {}, invalid: ["tool input is not an object"] }; } - return Object.fromEntries( - propertyValues.flatMap((entry) => { - if (!entry || typeof entry !== "object") return []; - const { propertyId, value } = entry as Record; - return typeof propertyId === "string" ? [[propertyId, value]] : []; - }), - ); + const record = input as Record; + const hasEntries = record.propertyEntries !== undefined; + const hasValues = record.propertyValues !== undefined; + if (hasEntries && hasValues) { + return { + received: {}, + invalid: ["propertyEntries and propertyValues were both provided"], + }; + } + if (hasValues) { + if ( + !record.propertyValues || + typeof record.propertyValues !== "object" || + Array.isArray(record.propertyValues) + ) { + return { received: {}, invalid: ["propertyValues is not a record"] }; + } + return { + received: record.propertyValues as Record, + invalid: [], + }; + } + if (!hasEntries || !Array.isArray(record.propertyEntries)) { + return { received: {}, invalid: ["propertyEntries is not an array"] }; + } + + const received: Record = {}; + const invalid: string[] = []; + for (const entry of record.propertyEntries) { + if (!entry || typeof entry !== "object" || Array.isArray(entry)) { + invalid.push("propertyEntries contains a non-object entry"); + continue; + } + const { propertyId, value } = entry as Record; + if (typeof propertyId !== "string" || propertyId.length === 0) { + invalid.push("propertyEntries contains an invalid propertyId"); + continue; + } + if (Object.prototype.hasOwnProperty.call(received, propertyId)) { + invalid.push(`propertyEntries contains duplicate ID ${propertyId}`); + continue; + } + received[propertyId] = value; + } + return { received, invalid }; } +const databaseRowMutationTools = new Set([ + "add-database-item", + "update-database-item", + "upsert-database-item-by-key", + "duplicate-database-items", + "remove-database-items", +]); + function expectedPropertyValuesScorer(expected: Record) { return createScorer< AgentRunOutput, @@ -57,30 +99,57 @@ function expectedPropertyValuesScorer(expected: Record) { received: Record; missing: string[]; unexpected: string[]; + invalid: string[]; + mutationCalls: string[]; } >({ name: "expected_property_values", analyze(run) { - const detail = run.toolCallDetails?.find( + const mutationCalls = (run.toolCallDetails ?? []) + .filter((call) => databaseRowMutationTools.has(call.name)) + .map((call) => call.name); + const createCalls = (run.toolCallDetails ?? []).filter( (call) => call.name === "add-database-item", ); - const received = normalizePropertyValues(detail?.input); + const analysis = analyzePropertyValues(createCalls[0]?.input); + const invalid = [...analysis.invalid]; + if (createCalls.length !== 1) { + invalid.push( + `expected exactly one add-database-item call, received ${createCalls.length}`, + ); + } + if (mutationCalls.length !== 1) { + invalid.push( + `expected exactly one row mutation, received ${mutationCalls.length}`, + ); + } + const received = analysis.received; const missing = Object.entries(expected) .filter(([propertyId, value]) => received[propertyId] !== value) .map(([propertyId]) => propertyId); const unexpected = Object.keys(received).filter( (propertyId) => !(propertyId in expected), ); - return { received, missing, unexpected }; + return { received, missing, unexpected, invalid, mutationCalls }; }, - generateScore({ missing, unexpected }) { - return missing.length === 0 && unexpected.length === 0 ? 1 : 0; + generateScore({ missing, unexpected, invalid }) { + return missing.length === 0 && + unexpected.length === 0 && + invalid.length === 0 + ? 1 + : 0; }, - generateReason({ analysis: { received, missing, unexpected } }) { - if (missing.length === 0 && unexpected.length === 0) { + generateReason({ + analysis: { received, missing, unexpected, invalid, mutationCalls }, + }) { + if ( + missing.length === 0 && + unexpected.length === 0 && + invalid.length === 0 + ) { return "Agent preserved every expected property ID and exact value without inventing another property."; } - return `Received propertyValues ${JSON.stringify(received)}; missing or changed: ${missing.join(", ") || "none"}; unexpected: ${unexpected.join(", ") || "none"}`; + return `Received propertyValues ${JSON.stringify(received)}; mutations: ${mutationCalls.join(", ") || "none"}; missing or changed: ${missing.join(", ") || "none"}; unexpected: ${unexpected.join(", ") || "none"}; invalid: ${invalid.join("; ") || "none"}`; }, }); } From 75344400b32aa39343e775df620697882050fc07 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Fri, 21 Aug 2026 07:52:10 -0400 Subject: [PATCH 4/9] test: require successful Content mutation evals --- packages/core/src/eval/agent-runner.ts | 30 +++- packages/core/src/eval/runner.spec.ts | 23 ++- packages/core/src/eval/types.ts | 5 +- .../__tests__/eval-scenario-coverage.test.ts | 153 +++++++++++++----- templates/content/parity/eval-scenarios.ts | 27 +++- templates/content/parity/scenario-to-eval.ts | 63 +++++++- 6 files changed, 250 insertions(+), 51 deletions(-) diff --git a/packages/core/src/eval/agent-runner.ts b/packages/core/src/eval/agent-runner.ts index 3b370ae7f2..0c316ec3e7 100644 --- a/packages/core/src/eval/agent-runner.ts +++ b/packages/core/src/eval/agent-runner.ts @@ -120,7 +120,14 @@ export async function createAgentRunner( let text = ""; const toolCalls: string[] = []; - const toolCallDetails: Array<{ name: string; input: unknown }> = []; + const toolCallDetails: Array<{ + name: string; + id?: string; + input: unknown; + completed?: boolean; + isError?: boolean; + result?: string; + }> = []; let ok = true; let error: string | undefined; @@ -135,8 +142,25 @@ export async function createAgentRunner( break; case "tool_start": toolCalls.push(event.tool); - toolCallDetails.push({ name: event.tool, input: event.input }); + toolCallDetails.push({ + name: event.tool, + id: event.id, + input: event.input, + }); + break; + case "tool_done": { + const detail = event.id + ? toolCallDetails.find((call) => call.id === event.id) + : toolCallDetails.find( + (call) => call.name === event.tool && !call.completed, + ); + if (detail) { + detail.completed = true; + detail.isError = event.isError === true; + detail.result = event.result; + } break; + } case "error": ok = false; error = event.error; @@ -167,7 +191,7 @@ export async function createAgentRunner( return { text, toolCalls, - toolCallDetails, + toolCallDetails: toolCallDetails.map(({ id: _id, ...detail }) => detail), ok, error, runId, diff --git a/packages/core/src/eval/runner.spec.ts b/packages/core/src/eval/runner.spec.ts index 3fa997625c..d0b5df129c 100644 --- a/packages/core/src/eval/runner.spec.ts +++ b/packages/core/src/eval/runner.spec.ts @@ -276,7 +276,18 @@ describe("createAgentRunner over a mocked runAgentLoop (no real model)", () => { const runLoop = vi.fn( async (opts: { send: (e: AgentChatEvent) => void }) => { opts.send({ type: "text", text: "Hello " }); - opts.send({ type: "tool_start", tool: "search", input: {} }); + opts.send({ + type: "tool_start", + tool: "search", + id: "search-1", + input: {}, + }); + opts.send({ + type: "tool_done", + tool: "search", + id: "search-1", + result: '{"ok":true}', + }); opts.send({ type: "text", text: "world" }); return { inputTokens: 0, @@ -299,7 +310,15 @@ describe("createAgentRunner over a mocked runAgentLoop (no real model)", () => { const out = await runner.runAgent({ prompt: "hi" }); expect(out.text).toBe("Hello world"); expect(out.toolCalls).toEqual(["search"]); - expect(out.toolCallDetails).toEqual([{ name: "search", input: {} }]); + expect(out.toolCallDetails).toEqual([ + { + name: "search", + input: {}, + completed: true, + isError: false, + result: '{"ok":true}', + }, + ]); expect(out.ok).toBe(true); // End-to-end: a contains scorer over the real collected text. diff --git a/packages/core/src/eval/types.ts b/packages/core/src/eval/types.ts index a99a0849a9..d9f17f8fef 100644 --- a/packages/core/src/eval/types.ts +++ b/packages/core/src/eval/types.ts @@ -31,10 +31,13 @@ export interface AgentRunOutput { readonly text: string; /** Names of tools/actions the agent invoked, in call order. */ readonly toolCalls: readonly string[]; - /** Tool names and model-produced inputs, in call order. */ + /** Tool names, model-produced inputs, and execution outcomes in call order. */ readonly toolCallDetails?: readonly { readonly name: string; readonly input: unknown; + readonly completed?: boolean; + readonly isError?: boolean; + readonly result?: string; }[]; /** Whether the run completed without a terminal error event. */ readonly ok: boolean; diff --git a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts index f7aba9894c..02bc290662 100644 --- a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts +++ b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts @@ -7,6 +7,22 @@ import { scenarioToEval } from "../scenario-to-eval"; const OLD_GATE = process.env.CONTENT_PARITY_EVALS; +function successfulCreateCall( + scenario: (typeof parityEvalScenarios)[number], + propertyInput: Record, +) { + return { + name: "add-database-item", + input: { + ...scenario.expectedCreateEnvelope, + ...propertyInput, + }, + completed: true, + isError: false, + result: '{"fixtureOnly":true}', + }; +} + afterEach(() => { if (OLD_GATE === undefined) { delete process.env.CONTENT_PARITY_EVALS; @@ -156,7 +172,7 @@ describe("Content parity eval scenarios", () => { text: scenario.successSignals.join("\n"), toolCalls: ["add-database-item"], toolCallDetails: [ - { name: "add-database-item", input: { propertyValues: {} } }, + successfulCreateCall(scenario, { propertyValues: {} }), ], ok: true, runId: "content-parity:empty-property-values", @@ -184,10 +200,9 @@ describe("Content parity eval scenarios", () => { text: scenario.successSignals.join("\n"), toolCalls: ["add-database-item"], toolCallDetails: [ - { - name: "add-database-item", - input: { propertyValues: scenario.expectedPropertyValues }, - }, + successfulCreateCall(scenario, { + propertyValues: scenario.expectedPropertyValues, + }), ], ok: true, runId: "content-parity:exact-property-values", @@ -205,59 +220,57 @@ describe("Content parity eval scenarios", () => { }); it.each([ - { + (scenario: (typeof parityEvalScenarios)[number]) => ({ name: "duplicate property entries", toolCallDetails: [ - { - name: "add-database-item", - input: { - propertyEntries: [ - { propertyId: "parity-text-property-id", value: "preserve me" }, - { propertyId: "parity-text-property-id", value: "preserve me" }, - { propertyId: "parity-status-property-id", value: "ready" }, - ], - }, - }, + successfulCreateCall(scenario, { + propertyEntries: [ + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-status-property-id", value: "ready" }, + ], + }), ], - }, - { + }), + (scenario: (typeof parityEvalScenarios)[number]) => ({ name: "ambiguous property formats", toolCallDetails: [ - { - name: "add-database-item", - input: { - propertyEntries: [ - { propertyId: "parity-text-property-id", value: "preserve me" }, - { propertyId: "parity-status-property-id", value: "ready" }, - ], - propertyValues: { - "parity-text-property-id": "preserve me", - "parity-status-property-id": "ready", - }, + successfulCreateCall(scenario, { + propertyEntries: [ + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-status-property-id", value: "ready" }, + ], + propertyValues: { + "parity-text-property-id": "preserve me", + "parity-status-property-id": "ready", }, - }, + }), ], - }, - { + }), + (scenario: (typeof parityEvalScenarios)[number]) => ({ name: "an extra row mutation", toolCallDetails: [ + successfulCreateCall(scenario, { + propertyEntries: [ + { propertyId: "parity-text-property-id", value: "preserve me" }, + { propertyId: "parity-status-property-id", value: "ready" }, + ], + }), { - name: "add-database-item", - input: { - propertyEntries: [ - { propertyId: "parity-text-property-id", value: "preserve me" }, - { propertyId: "parity-status-property-id", value: "ready" }, - ], - }, + name: "update-database-item", + input: {}, + completed: true, + isError: false, + result: "{}", }, - { name: "update-database-item", input: {} }, ], - }, - ])("rejects $name", async ({ toolCallDetails }) => { + }), + ])("rejects invalid property behavior", async (buildCase) => { process.env.CONTENT_PARITY_EVALS = "1"; const scenario = parityEvalScenarios.find( (candidate) => candidate.id === "database-create-property-preservation", )!; + const { toolCallDetails } = buildCase(scenario); const evalCase = scenarioToEval(scenario); const row = await scoreEval(evalCase, { runAgent: vi.fn(async () => ({ @@ -278,4 +291,60 @@ describe("Content parity eval scenarios", () => { ).toMatchObject({ passed: false, score: 0 }); expect(row.status).toBe("failed"); }); + + it.each([ + { + name: "wrong target", + mutate(call: ReturnType) { + return { + ...call, + input: { + ...(call.input as Record), + target: { + ...((call.input as Record).target as Record< + string, + unknown + >), + databaseId: "wrong-database", + }, + }, + }; + }, + }, + { + name: "failed execution", + mutate(call: ReturnType) { + return { ...call, isError: true, result: "fixture rejected" }; + }, + }, + ])("rejects $name", async ({ mutate }) => { + process.env.CONTENT_PARITY_EVALS = "1"; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", + )!; + const call = mutate( + successfulCreateCall(scenario, { + propertyValues: scenario.expectedPropertyValues, + }), + ); + const evalCase = scenarioToEval(scenario); + const row = await scoreEval(evalCase, { + runAgent: vi.fn(async () => ({ + text: scenario.successSignals.join("\n"), + toolCalls: ["add-database-item"], + toolCallDetails: [call], + ok: true, + runId: "content-parity:rejected-create", + durationMs: 1, + })), + engine: {} as never, + model: "test-model", + analyzeContext: vi.fn(), + }); + + expect( + row.scores.find((score) => score.scorer === "expected_property_values"), + ).toMatchObject({ passed: false, score: 0 }); + expect(row.status).toBe("failed"); + }); }); diff --git a/templates/content/parity/eval-scenarios.ts b/templates/content/parity/eval-scenarios.ts index c459e1f376..3c26553e2b 100644 --- a/templates/content/parity/eval-scenarios.ts +++ b/templates/content/parity/eval-scenarios.ts @@ -9,6 +9,17 @@ export interface ParityEvalScenario { successSignals: string[]; expectedTools?: string[]; expectedPropertyValues?: Record; + expectedCreateEnvelope?: { + target: { + authorityScope: { kind: "personal"; id: string }; + spaceId: string; + databaseId: string; + databaseDocumentId: string; + }; + expectedSchemaRevision: string; + idempotencyKey: string; + title: string; + }; } export const parityEvalScenarios: ParityEvalScenario[] = [ @@ -20,7 +31,7 @@ export const parityEvalScenarios: ParityEvalScenario[] = [ defaultState: "skipped", requiresPrivateCredentials: false, prompt: - "Create exactly one row in the already-verified fixture Content database. Do not rediscover or alter the target. Use target spaceId fixture_personal_space, databaseId fixture_feedback_database, and documentId fixture_feedback_document; expectedSchemaRevision fixture_schema_revision; title [FIXTURE] preserve explicit properties; idempotencyKey fixture-create-property-preservation-v1. Set property fixture_status_property to status-cannot-verify and property fixture_evidence_property to Baseline fixture preserve-me. No Blocks value was requested. Call add-database-item once with every exact target constraint and both property entries, then report its result truthfully.", + "Create exactly one row in the already-verified fixture Content database. Do not rediscover or alter the target. Use target authorityScope { kind: personal, id: fixture-owner@example.test }, spaceId fixture_personal_space, databaseId fixture_feedback_database, and databaseDocumentId fixture_feedback_document; expectedSchemaRevision fixture_schema_revision; title [FIXTURE] preserve explicit properties; idempotencyKey fixture-create-property-preservation-v1. Set property fixture_status_property to status-cannot-verify and property fixture_evidence_property to Baseline fixture preserve-me. No Blocks value was requested. Call add-database-item once with every exact target constraint and both property entries, then report its result truthfully.", successSignals: [ "Uses add-database-item once for the exact fixture target.", "Preserves both explicitly requested writable property values.", @@ -32,6 +43,20 @@ export const parityEvalScenarios: ParityEvalScenario[] = [ fixture_status_property: "status-cannot-verify", fixture_evidence_property: "Baseline fixture preserve-me", }, + expectedCreateEnvelope: { + target: { + authorityScope: { + kind: "personal", + id: "fixture-owner@example.test", + }, + spaceId: "fixture_personal_space", + databaseId: "fixture_feedback_database", + databaseDocumentId: "fixture_feedback_document", + }, + expectedSchemaRevision: "fixture_schema_revision", + idempotencyKey: "fixture-create-property-preservation-v1", + title: "[FIXTURE] preserve explicit properties", + }, }, { id: "database-bulk-row-reliability", diff --git a/templates/content/parity/scenario-to-eval.ts b/templates/content/parity/scenario-to-eval.ts index d6766456a3..514fa94cea 100644 --- a/templates/content/parity/scenario-to-eval.ts +++ b/templates/content/parity/scenario-to-eval.ts @@ -92,7 +92,40 @@ const databaseRowMutationTools = new Set([ "remove-database-items", ]); -function expectedPropertyValuesScorer(expected: Record) { +function matchesCreateEnvelope( + input: Record, + expected: NonNullable, +): boolean { + const target = input.target; + if (!target || typeof target !== "object" || Array.isArray(target)) { + return false; + } + const actualTarget = target as Record; + const authorityScope = actualTarget.authorityScope; + if ( + !authorityScope || + typeof authorityScope !== "object" || + Array.isArray(authorityScope) + ) { + return false; + } + const actualAuthority = authorityScope as Record; + return ( + actualAuthority.kind === expected.target.authorityScope.kind && + actualAuthority.id === expected.target.authorityScope.id && + actualTarget.spaceId === expected.target.spaceId && + actualTarget.databaseId === expected.target.databaseId && + actualTarget.databaseDocumentId === expected.target.databaseDocumentId && + input.expectedSchemaRevision === expected.expectedSchemaRevision && + input.idempotencyKey === expected.idempotencyKey && + input.title === expected.title + ); +} + +function expectedPropertyValuesScorer( + expected: Record, + expectedEnvelope?: ParityEvalScenario["expectedCreateEnvelope"], +) { return createScorer< AgentRunOutput, { @@ -123,6 +156,27 @@ function expectedPropertyValuesScorer(expected: Record) { `expected exactly one row mutation, received ${mutationCalls.length}`, ); } + const createInput = createCalls[0]?.input; + if ( + expectedEnvelope && + (!createInput || + typeof createInput !== "object" || + Array.isArray(createInput) || + !matchesCreateEnvelope( + createInput as Record, + expectedEnvelope, + )) + ) { + invalid.push( + "create target, schema revision, idempotency key, or title did not match the fixture", + ); + } + if (!createCalls[0]?.completed || createCalls[0]?.isError) { + invalid.push("add-database-item did not complete successfully"); + } + if (!run.ok) { + invalid.push("agent run did not complete successfully"); + } const received = analysis.received; const missing = Object.entries(expected) .filter(([propertyId, value]) => received[propertyId] !== value) @@ -177,7 +231,12 @@ export function scenarioToEval(scenario: ParityEvalScenario): Eval { ? [expectedToolScorer(scenario.expectedTools)] : []), ...(scenario.expectedPropertyValues - ? [expectedPropertyValuesScorer(scenario.expectedPropertyValues)] + ? [ + expectedPropertyValuesScorer( + scenario.expectedPropertyValues, + scenario.expectedCreateEnvelope, + ), + ] : []), ], }); From 0e7b09d59debbbb8154acbeb6c038fe52f7824e8 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:06:51 -0400 Subject: [PATCH 5/9] chore: publish branch work in packages/core, templates/content (7 files) --- packages/core/src/eval/agent-runner.ts | 2 + packages/core/src/eval/runner.spec.ts | 25 ++++++- packages/core/src/eval/types.ts | 1 + .../actions/_database-property-input.ts | 5 +- .../database-row-property-input.test.ts | 33 ++++++++ .../__tests__/eval-scenario-coverage.test.ts | 75 +++++++++++++++++++ templates/content/parity/scenario-to-eval.ts | 50 ++++++++++++- 7 files changed, 186 insertions(+), 5 deletions(-) diff --git a/packages/core/src/eval/agent-runner.ts b/packages/core/src/eval/agent-runner.ts index 0c316ec3e7..c986194a0e 100644 --- a/packages/core/src/eval/agent-runner.ts +++ b/packages/core/src/eval/agent-runner.ts @@ -125,6 +125,7 @@ export async function createAgentRunner( id?: string; input: unknown; completed?: boolean; + completedSideEffect?: boolean; isError?: boolean; result?: string; }> = []; @@ -156,6 +157,7 @@ export async function createAgentRunner( ); if (detail) { detail.completed = true; + detail.completedSideEffect = event.completedSideEffect; detail.isError = event.isError === true; detail.result = event.result; } diff --git a/packages/core/src/eval/runner.spec.ts b/packages/core/src/eval/runner.spec.ts index d0b5df129c..389844d1dd 100644 --- a/packages/core/src/eval/runner.spec.ts +++ b/packages/core/src/eval/runner.spec.ts @@ -287,6 +287,20 @@ describe("createAgentRunner over a mocked runAgentLoop (no real model)", () => { tool: "search", id: "search-1", result: '{"ok":true}', + completedSideEffect: true, + }); + opts.send({ + type: "tool_start", + tool: "update", + id: "update-1", + input: {}, + }); + opts.send({ + type: "tool_done", + tool: "update", + id: "update-1", + result: '{"ok":false}', + completedSideEffect: false, }); opts.send({ type: "text", text: "world" }); return { @@ -309,15 +323,24 @@ describe("createAgentRunner over a mocked runAgentLoop (no real model)", () => { const out = await runner.runAgent({ prompt: "hi" }); expect(out.text).toBe("Hello world"); - expect(out.toolCalls).toEqual(["search"]); + expect(out.toolCalls).toEqual(["search", "update"]); expect(out.toolCallDetails).toEqual([ { name: "search", input: {}, completed: true, + completedSideEffect: true, isError: false, result: '{"ok":true}', }, + { + name: "update", + input: {}, + completed: true, + completedSideEffect: false, + isError: false, + result: '{"ok":false}', + }, ]); expect(out.ok).toBe(true); diff --git a/packages/core/src/eval/types.ts b/packages/core/src/eval/types.ts index d9f17f8fef..81e32e03a8 100644 --- a/packages/core/src/eval/types.ts +++ b/packages/core/src/eval/types.ts @@ -36,6 +36,7 @@ export interface AgentRunOutput { readonly name: string; readonly input: unknown; readonly completed?: boolean; + readonly completedSideEffect?: boolean; readonly isError?: boolean; readonly result?: string; }[]; diff --git a/templates/content/actions/_database-property-input.ts b/templates/content/actions/_database-property-input.ts index 30552617c3..053de8f77a 100644 --- a/templates/content/actions/_database-property-input.ts +++ b/templates/content/actions/_database-property-input.ts @@ -36,7 +36,10 @@ export function normalizeDatabasePropertyInput(input: { } if (!input.propertyEntries) return input.propertyValues; - const values: Record = {}; + const values: Record = Object.create(null) as Record< + string, + unknown + >; for (const entry of input.propertyEntries) { if (Object.prototype.hasOwnProperty.call(values, entry.propertyId)) { throw new ActionContractError( diff --git a/templates/content/parity/__tests__/database-row-property-input.test.ts b/templates/content/parity/__tests__/database-row-property-input.test.ts index aaa40f1e3d..6e31ef055b 100644 --- a/templates/content/parity/__tests__/database-row-property-input.test.ts +++ b/templates/content/parity/__tests__/database-row-property-input.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { canonicalizeDatabasePropertyInput, + databasePropertyEntriesSchema, normalizeDatabasePropertyInput, } from "../../actions/_database-property-input"; import { digest } from "../../actions/_database-row-mutation"; @@ -72,6 +73,23 @@ describe("database row property inputs", () => { ).toThrow(/not both/); }); + it("preserves __proto__ as an ordinary property definition ID", () => { + const propertyEntries = databasePropertyEntriesSchema.parse([ + { propertyId: "__proto__", value: "preserve me" }, + ]); + const normalized = normalizeDatabasePropertyInput({ + propertyEntries, + }); + + expect(normalized).toBeDefined(); + expect(Object.getPrototypeOf(normalized)).toBeNull(); + expect(Object.keys(normalized!)).toEqual(["__proto__"]); + expect(Object.prototype.hasOwnProperty.call(normalized, "__proto__")).toBe( + true, + ); + expect(normalized?.["__proto__"]).toBe("preserve me"); + }); + it("removes the model-only representation before canonical hashing", () => { const canonical = canonicalizeDatabasePropertyInput({ idempotencyKey: "same-intent", @@ -109,4 +127,19 @@ describe("database row property inputs", () => { expect(digest(fromEntries)).toBe(digest(fromRecord)); }); + + it("includes __proto__ property values in the canonical digest", () => { + const withPrototypeNamedProperty = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyEntries: [{ propertyId: "__proto__", value: "preserve me" }], + }); + const withoutProperty = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyValues: {}, + }); + + expect(digest(withPrototypeNamedProperty)).not.toBe( + digest(withoutProperty), + ); + }); }); diff --git a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts index 02bc290662..bf0892c049 100644 --- a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts +++ b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts @@ -18,6 +18,7 @@ function successfulCreateCall( ...propertyInput, }, completed: true, + completedSideEffect: true, isError: false, result: '{"fixtureOnly":true}', }; @@ -317,6 +318,80 @@ describe("Content parity eval scenarios", () => { return { ...call, isError: true, result: "fixture rejected" }; }, }, + { + name: "skipped side effect", + mutate(call: ReturnType) { + return { ...call, completedSideEffect: false }; + }, + }, + { + name: "an extra top-level field", + mutate(call: ReturnType) { + return { + ...call, + input: { + ...(call.input as Record), + hallucinated: true, + }, + }; + }, + }, + { + name: "an extra target field", + mutate(call: ReturnType) { + const input = call.input as Record; + return { + ...call, + input: { + ...input, + target: { + ...(input.target as Record), + hallucinated: true, + }, + }, + }; + }, + }, + { + name: "an extra authority field", + mutate(call: ReturnType) { + const input = call.input as Record; + const target = input.target as Record; + return { + ...call, + input: { + ...input, + target: { + ...target, + authorityScope: { + ...(target.authorityScope as Record), + hallucinated: true, + }, + }, + }, + }; + }, + }, + { + name: "an extra property-entry field", + mutate(call: ReturnType) { + const input = call.input as Record; + const { propertyValues, ...withoutPropertyValues } = input; + return { + ...call, + input: { + ...withoutPropertyValues, + propertyEntries: Object.entries( + propertyValues as Record, + ).map(([propertyId, value]) => ({ + propertyId, + value, + hallucinated: true, + })), + }, + }; + }, + }, ])("rejects $name", async ({ mutate }) => { process.env.CONTENT_PARITY_EVALS = "1"; const scenario = parityEvalScenarios.find( diff --git a/templates/content/parity/scenario-to-eval.ts b/templates/content/parity/scenario-to-eval.ts index 514fa94cea..6f7e50d378 100644 --- a/templates/content/parity/scenario-to-eval.ts +++ b/templates/content/parity/scenario-to-eval.ts @@ -30,6 +30,18 @@ function expectedToolScorer(expectedTools: string[]) { }); } +function hasExactKeys( + record: Record, + expectedKeys: readonly string[], +): boolean { + const actualKeys = Object.keys(record).sort(); + const sortedExpectedKeys = [...expectedKeys].sort(); + return ( + actualKeys.length === sortedExpectedKeys.length && + actualKeys.every((key, index) => key === sortedExpectedKeys[index]) + ); +} + function analyzePropertyValues(input: unknown): { received: Record; invalid: string[]; @@ -63,7 +75,10 @@ function analyzePropertyValues(input: unknown): { return { received: {}, invalid: ["propertyEntries is not an array"] }; } - const received: Record = {}; + const received: Record = Object.create(null) as Record< + string, + unknown + >; const invalid: string[] = []; for (const entry of record.propertyEntries) { if (!entry || typeof entry !== "object" || Array.isArray(entry)) { @@ -71,6 +86,14 @@ function analyzePropertyValues(input: unknown): { continue; } const { propertyId, value } = entry as Record; + if ( + !hasExactKeys(entry as Record, ["propertyId", "value"]) + ) { + invalid.push( + "propertyEntries contains an entry with unrecognized fields", + ); + continue; + } if (typeof propertyId !== "string" || propertyId.length === 0) { invalid.push("propertyEntries contains an invalid propertyId"); continue; @@ -111,6 +134,22 @@ function matchesCreateEnvelope( } const actualAuthority = authorityScope as Record; return ( + hasExactKeys(input, [ + "target", + "expectedSchemaRevision", + "idempotencyKey", + "title", + input.propertyEntries === undefined + ? "propertyValues" + : "propertyEntries", + ]) && + hasExactKeys(actualTarget, [ + "authorityScope", + "spaceId", + "databaseId", + "databaseDocumentId", + ]) && + hasExactKeys(actualAuthority, ["kind", "id"]) && actualAuthority.kind === expected.target.authorityScope.kind && actualAuthority.id === expected.target.authorityScope.id && actualTarget.spaceId === expected.target.spaceId && @@ -171,7 +210,11 @@ function expectedPropertyValuesScorer( "create target, schema revision, idempotency key, or title did not match the fixture", ); } - if (!createCalls[0]?.completed || createCalls[0]?.isError) { + if ( + !createCalls[0]?.completed || + createCalls[0]?.completedSideEffect !== true || + createCalls[0]?.isError + ) { invalid.push("add-database-item did not complete successfully"); } if (!run.ok) { @@ -182,7 +225,8 @@ function expectedPropertyValuesScorer( .filter(([propertyId, value]) => received[propertyId] !== value) .map(([propertyId]) => propertyId); const unexpected = Object.keys(received).filter( - (propertyId) => !(propertyId in expected), + (propertyId) => + !Object.prototype.hasOwnProperty.call(expected, propertyId), ); return { received, missing, unexpected, invalid, mutationCalls }; }, From d01629941a4bddd18d7186747706525f8441995c Mon Sep 17 00:00:00 2001 From: Alice Moore <86723305+3mdistal@users.noreply.github.com> Date: Mon, 24 Aug 2026 17:13:50 +0000 Subject: [PATCH 6/9] fix: make Content database mutations agent-safe --- .../actions/_database-property-input.ts | 172 ++++++++++++++++-- .../content/actions/_database-row-mutation.ts | 103 +++++++++-- .../content/actions/add-database-item.ts | 6 +- .../roadmap-capability-projection.db.test.ts | 109 ++++++----- .../content/actions/update-database-item.ts | 8 +- .../upsert-database-item-by-key.db.test.ts | 47 ++++- .../actions/upsert-database-item-by-key.ts | 8 +- .../database-row-property-input.test.ts | 148 ++++++++++++--- .../__tests__/eval-scenario-coverage.test.ts | 119 +++++++++--- templates/content/parity/eval-scenarios.ts | 21 ++- templates/content/parity/scenario-to-eval.ts | 101 ++++++++-- 11 files changed, 681 insertions(+), 161 deletions(-) diff --git a/templates/content/actions/_database-property-input.ts b/templates/content/actions/_database-property-input.ts index 053de8f77a..5e89b4ccbc 100644 --- a/templates/content/actions/_database-property-input.ts +++ b/templates/content/actions/_database-property-input.ts @@ -1,6 +1,129 @@ import { ActionContractError } from "@agent-native/core"; import { z } from "zod"; +const nullable = (schema: T) => + z.union([schema, z.null()]); + +const propertyIdSchema = z + .string() + .min(1) + .describe("Exact immutable property definition ID"); + +const stringPropertyEntry = ( + propertyType: "text" | "place" | "phone" | "url" | "email", + valueDescription: string, +) => + z + .object({ + propertyId: propertyIdSchema, + propertyType: z.literal(propertyType), + value: nullable(z.string()).describe(valueDescription), + }) + .strict(); + +const optionPropertyEntry = (propertyType: "select" | "status") => + z + .object({ + propertyId: propertyIdSchema, + propertyType: z.literal(propertyType), + value: nullable(z.string()).describe( + "Exact option ID or exact option label from the discovered property contract; null explicitly clears the value", + ), + }) + .strict(); + +export const databasePropertyEntrySchema = z.discriminatedUnion( + "propertyType", + [ + stringPropertyEntry("text", "Text value; null explicitly clears the value"), + stringPropertyEntry( + "place", + "Place text; null explicitly clears the value", + ), + stringPropertyEntry( + "phone", + "Phone text; null explicitly clears the value", + ), + stringPropertyEntry( + "url", + "Absolute http/https URL; null explicitly clears the value", + ), + stringPropertyEntry( + "email", + "Email address; null explicitly clears the value", + ), + z + .object({ + propertyId: propertyIdSchema, + propertyType: z.literal("number"), + value: nullable(z.number().finite()).describe( + "Finite number; use a JSON number rather than numeric text, or null to explicitly clear", + ), + }) + .strict(), + z + .object({ + propertyId: propertyIdSchema, + propertyType: z.literal("checkbox"), + value: nullable(z.boolean()).describe( + "Boolean; use true or false rather than text, or null to explicitly clear", + ), + }) + .strict(), + optionPropertyEntry("select"), + optionPropertyEntry("status"), + z + .object({ + propertyId: propertyIdSchema, + propertyType: z.literal("multi_select"), + value: nullable(z.array(z.string())).describe( + "Option IDs or exact option labels from the discovered property contract; null explicitly clears the value", + ), + }) + .strict(), + z + .object({ + propertyId: propertyIdSchema, + propertyType: z.literal("date"), + value: nullable( + z.union([ + z.string(), + z + .object({ + start: z.string(), + end: z.string().optional(), + includeTime: z.boolean().optional(), + }) + .strict(), + ]), + ).describe( + "ISO date/date-time string or { start, end?, includeTime? }; null explicitly clears the value", + ), + }) + .strict(), + z + .object({ + propertyId: propertyIdSchema, + propertyType: z.literal("person"), + value: nullable(z.array(z.string())).describe( + "Person identifiers from the discovered property contract; null explicitly clears the value", + ), + }) + .strict(), + z + .object({ + propertyId: propertyIdSchema, + propertyType: z.literal("files_media"), + value: nullable(z.array(z.string())).describe( + "Absolute http/https file URLs; null explicitly clears the value", + ), + }) + .strict(), + ], +); + +export type DatabasePropertyEntry = z.infer; + export const databasePropertyValuesSchema = z .record(z.string(), z.unknown()) .optional() @@ -9,37 +132,41 @@ export const databasePropertyValuesSchema = z ); export const databasePropertyEntriesSchema = z - .array( - z.object({ - propertyId: z - .string() - .min(1) - .describe("Exact immutable property definition ID"), - value: z.unknown().describe("Schema-valid value for this property"), - }), - ) + .array(databasePropertyEntrySchema) .max(1_000) .optional() .describe( - "Property values as explicit entries. Include one entry for every schema-valid writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", + "Typed property values as explicit entries. Copy each propertyType from the discovered mutation contract and include one entry for every writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", ); export function normalizeDatabasePropertyInput(input: { - propertyEntries?: Array<{ propertyId: string; value: unknown }>; + propertyEntries?: DatabasePropertyEntry[]; propertyValues?: Record; -}): Record | undefined { +}): { + propertyValues: Record | undefined; + propertyTypeAssertions: Record | undefined; +} { if (input.propertyEntries && input.propertyValues) { throw new ActionContractError( "Provide propertyEntries or propertyValues, not both.", { errorCode: "AMBIGUOUS_PROPERTY_INPUT" }, ); } - if (!input.propertyEntries) return input.propertyValues; + if (!input.propertyEntries) { + return { + propertyValues: input.propertyValues, + propertyTypeAssertions: undefined, + }; + } const values: Record = Object.create(null) as Record< string, unknown >; + const propertyTypes: Record = Object.create(null) as Record< + string, + string + >; for (const entry of input.propertyEntries) { if (Object.prototype.hasOwnProperty.call(values, entry.propertyId)) { throw new ActionContractError( @@ -51,26 +178,33 @@ export function normalizeDatabasePropertyInput(input: { ); } values[entry.propertyId] = entry.value; + propertyTypes[entry.propertyId] = entry.propertyType; } - return values; + return { + propertyValues: values, + propertyTypeAssertions: propertyTypes, + }; } export function canonicalizeDatabasePropertyInput< T extends { - propertyEntries?: Array<{ propertyId: string; value: unknown }>; + propertyEntries?: DatabasePropertyEntry[]; propertyValues?: Record; }, >( input: T, ): Omit & { propertyValues?: Record; + propertyTypeAssertions?: Record; } { const { propertyEntries, propertyValues, ...canonicalInput } = input; + const normalized = normalizeDatabasePropertyInput({ + propertyEntries, + propertyValues, + }); return { ...canonicalInput, - propertyValues: normalizeDatabasePropertyInput({ - propertyEntries, - propertyValues, - }), + propertyValues: normalized.propertyValues, + propertyTypeAssertions: normalized.propertyTypeAssertions, }; } diff --git a/templates/content/actions/_database-row-mutation.ts b/templates/content/actions/_database-row-mutation.ts index 5bab2f32e3..5b8372030a 100644 --- a/templates/content/actions/_database-row-mutation.ts +++ b/templates/content/actions/_database-row-mutation.ts @@ -34,11 +34,13 @@ import { } from "./_position-utils.js"; import { nanoid } from "./_property-utils.js"; +const databaseMutationAuthorityScopeSchema = z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("personal"), id: z.string().min(1) }), + z.object({ kind: z.literal("organization"), id: z.string().min(1) }), +]); + export const databaseMutationTargetSchema = z.object({ - authorityScope: z.discriminatedUnion("kind", [ - z.object({ kind: z.literal("personal"), id: z.string().min(1) }), - z.object({ kind: z.literal("organization"), id: z.string().min(1) }), - ]), + authorityScope: databaseMutationAuthorityScopeSchema, spaceId: z.string().min(1).describe("Exact Content space ID"), databaseId: z.string().min(1).describe("Exact Content database ID"), databaseDocumentId: z @@ -47,8 +49,35 @@ export const databaseMutationTargetSchema = z.object({ .describe("Exact page ID backing the Content database"), }); +export const databaseMutationTargetInputSchema = z.object({ + authorityScope: databaseMutationAuthorityScopeSchema + .optional() + .describe( + "Optional legacy assertion only. Agents must omit it; the authenticated server derives authority from the selected database.", + ), + spaceId: z + .string() + .min(1) + .describe("Exact Content space ID returned by database discovery"), + databaseId: z + .string() + .min(1) + .describe( + "Exact Content database ID returned by database discovery; never derive it from a title or number in the request", + ), + databaseDocumentId: z + .string() + .min(1) + .describe( + "Exact page ID backing the database, returned by database discovery", + ), +}); + +export const databaseMutationAgentTargetSchema = + databaseMutationTargetInputSchema.omit({ authorityScope: true }); + export const databaseMutationEnvelopeSchema = z.object({ - target: databaseMutationTargetSchema, + target: databaseMutationTargetInputSchema, expectedSchemaRevision: z .string() .min(1) @@ -59,6 +88,9 @@ export const databaseMutationEnvelopeSchema = z.object({ export type DatabaseMutationTarget = z.infer< typeof databaseMutationTargetSchema >; +export type DatabaseMutationTargetInput = z.infer< + typeof databaseMutationTargetInputSchema +>; type DatabaseRow = typeof schema.contentDatabases.$inferSelect; type DefinitionRow = typeof schema.documentPropertyDefinitions.$inferSelect; @@ -82,11 +114,12 @@ export interface RowSnapshot { export type DatabaseRowMutationOperation = "create" | "update" | "upsert"; export interface CreateDatabaseRowMutationInput { - target: DatabaseMutationTarget; + target: DatabaseMutationTargetInput; expectedSchemaRevision: string; idempotencyKey: string; title?: string; propertyValues?: Record; + propertyTypeAssertions?: Record; } export interface UpdateDatabaseRowMutationInput extends CreateDatabaseRowMutationInput { @@ -211,7 +244,7 @@ function acceptedShape(type: DocumentPropertyType): string { } export async function loadContext( - target: DatabaseMutationTarget, + target: DatabaseMutationTargetInput, role: "viewer" | "editor", db: Db = getDb(), accessAlreadyResolved = false, @@ -260,8 +293,9 @@ export async function loadContext( ? { kind: "organization" as const, id: database.orgId } : { kind: "personal" as const, id: database.ownerEmail }; if ( - target.authorityScope.kind !== authorityScope.kind || - target.authorityScope.id !== authorityScope.id || + (target.authorityScope !== undefined && + (target.authorityScope.kind !== authorityScope.kind || + target.authorityScope.id !== authorityScope.id)) || database.spaceId !== target.spaceId || database.documentId !== target.databaseDocumentId || databaseDocument.spaceId !== target.spaceId || @@ -322,7 +356,12 @@ export async function getDatabaseMutationContract( ); return { target: { - ...target, + authorityScope: context.database.orgId + ? { kind: "organization", id: context.database.orgId } + : { kind: "personal", id: context.database.ownerEmail }, + spaceId: context.database.spaceId!, + databaseId: context.database.id, + databaseDocumentId: context.database.documentId, }, schemaRevision: context.schemaRevision, naturalKeyPropertyId: context.database.naturalKeyPropertyId, @@ -695,14 +734,45 @@ export function revisionPropertyIds(context: MutationContext) { ); } -function payloadDigest( +export function databaseMutationPayloadDigest( operation: DatabaseRowMutationOperation, input: | CreateDatabaseRowMutationInput | UpdateDatabaseRowMutationInput | UpsertDatabaseRowMutationInput, ) { - return digest({ operation, ...input }); + const { propertyTypeAssertions: _propertyTypeAssertions, ...canonicalInput } = + input; + return digest({ operation, ...canonicalInput }); +} + +function assertPropertyTypeAssertions( + context: MutationContext, + assertions: Record | undefined, +) { + if (!assertions) return; + const definitionsById = new Map( + context.definitions.map((definition) => [definition.id, definition]), + ); + for (const [propertyId, assertedType] of Object.entries(assertions)) { + const definition = definitionsById.get(propertyId); + if (!definition) { + throw new ActionContractError( + `Unknown property definition "${propertyId}".`, + { + errorCode: "UNKNOWN_PROPERTY", + details: { propertyId }, + statusCode: 400, + }, + ); + } + if (definition.type !== assertedType) { + invalidProperty( + definition, + `typed entry declared propertyType "${assertedType}" but the discovered property type is "${definition.type}"`, + ); + } + } } function resultForReceipt( @@ -1117,7 +1187,8 @@ export async function createDatabaseRow( input: CreateDatabaseRowMutationInput, ): Promise { const initial = await loadContext(input.target, "editor"); - const inputDigest = payloadDigest("create", input); + assertPropertyTypeAssertions(initial, input.propertyTypeAssertions); + const inputDigest = databaseMutationPayloadDigest("create", input); const replay = await replayReceipt( initial, input.idempotencyKey, @@ -1185,8 +1256,9 @@ export async function updateDatabaseRow( input: UpdateDatabaseRowMutationInput, ): Promise { const initial = await loadContext(input.target, "editor"); + assertPropertyTypeAssertions(initial, input.propertyTypeAssertions); await assertAccess("document", input.documentId, "editor"); - const inputDigest = payloadDigest("update", input); + const inputDigest = databaseMutationPayloadDigest("update", input); const replay = await replayReceipt( initial, input.idempotencyKey, @@ -1265,6 +1337,7 @@ export async function upsertDatabaseRow( input: UpsertDatabaseRowMutationInput, ): Promise { const initial = await loadContext(input.target, "editor"); + assertPropertyTypeAssertions(initial, input.propertyTypeAssertions); const replayKeyPropertyId = initial.database.naturalKeyPropertyId; const replayKeyDefinition = replayKeyPropertyId ? initial.definitions.find( @@ -1285,7 +1358,7 @@ export async function upsertDatabaseRow( "must match the upsert keyValue when provided in propertyValues", ); } - const inputDigest = payloadDigest("upsert", input); + const inputDigest = databaseMutationPayloadDigest("upsert", input); const replay = await replayReceipt( initial, input.idempotencyKey, diff --git a/templates/content/actions/add-database-item.ts b/templates/content/actions/add-database-item.ts index 8666c33bdf..423d675b0b 100644 --- a/templates/content/actions/add-database-item.ts +++ b/templates/content/actions/add-database-item.ts @@ -10,6 +10,7 @@ import { } from "./_database-property-input.js"; import { createDatabaseRow, + databaseMutationAgentTargetSchema, databaseMutationEnvelopeSchema, } from "./_database-row-mutation.js"; import { getContentDatabaseResponse } from "./_database-utils.js"; @@ -25,11 +26,14 @@ const schema = databaseMutationEnvelopeSchema.extend({ propertyValues: databasePropertyValuesSchema, propertyEntries: databasePropertyEntriesSchema, }); +const agentSchema = schema + .extend({ target: databaseMutationAgentTargetSchema }) + .omit({ propertyValues: true }); export default defineAction({ description: "Create one row in an exact ordinary Content database using its discovered schema revision. Strictly validates every non-Blocks property, applies the side effect once per idempotency key, and returns a verified receipt with stable row identity.", - agentInputSchema: schema.omit({ propertyValues: true }), + agentInputSchema: agentSchema, publicAgent: { expose: true, readOnly: false, diff --git a/templates/content/actions/roadmap-capability-projection.db.test.ts b/templates/content/actions/roadmap-capability-projection.db.test.ts index 648cf0dc71..221bbf1d19 100644 --- a/templates/content/actions/roadmap-capability-projection.db.test.ts +++ b/templates/content/actions/roadmap-capability-projection.db.test.ts @@ -143,6 +143,7 @@ describe("private roadmap capability projection", () => { databaseId, name, type, + naturalKey: name === "Capability ID", }), ); const property = configured.properties.find( @@ -158,28 +159,45 @@ describe("private roadmap capability projection", () => { const firstReceipts = new Map< string, - { itemId: string; documentId: string } + { itemId: string; documentId: string; rowRevision: string } >(); + const discovered = await asUser(OWNER, () => + getDatabase.run({ databaseId, limit: 1, offset: 0 }), + ); + if (!("database" in discovered) || !discovered.mutationContract) + throw new Error("Projection database has no mutation contract."); + const mutationEnvelope = { + target: { + spaceId: discovered.mutationContract.target.spaceId, + databaseId: discovered.mutationContract.target.databaseId, + databaseDocumentId: + discovered.mutationContract.target.databaseDocumentId, + }, + expectedSchemaRevision: discovered.mutationContract.schemaRevision, + }; + const propertyValuesFor = (capability: Capability) => ({ + [keyPropertyId]: capability.id, + [propertyIds.get("State")!]: capability.state, + [propertyIds.get("Publicness")!]: capability.publicness, + [propertyIds.get("User promise")!]: capability.userPromise, + [propertyIds.get("Source revision")!]: SOURCE_REVISION, + }); for (const capability of capabilities) { const receipt = await asUser(OWNER, () => upsert.run({ - databaseId, - keyPropertyId, + ...mutationEnvelope, + idempotencyKey: `roadmap-projection-${capability.id}`, keyValue: capability.id, + expectedRowRevision: null, title: capability.name, - body: capability.body, - propertyValues: { - [propertyIds.get("State")!]: capability.state, - [propertyIds.get("Publicness")!]: capability.publicness, - [propertyIds.get("User promise")!]: capability.userPromise, - [propertyIds.get("Source revision")!]: SOURCE_REVISION, - }, + propertyValues: propertyValuesFor(capability), }), ); - expect(receipt.status).toBe("created"); + expect(receipt.receipt.outcome).toBe("created"); firstReceipts.set(capability.id, { - itemId: receipt.itemId, - documentId: receipt.documentId, + itemId: receipt.receipt.row.itemId, + documentId: receipt.receipt.row.documentId, + rowRevision: receipt.receipt.row.rowRevision, }); } @@ -191,55 +209,53 @@ describe("private roadmap capability projection", () => { throw new Error("Changed Capability is missing its first receipt."); const changedReceipt = await asUser(OWNER, () => upsert.run({ - databaseId, - keyPropertyId, + ...mutationEnvelope, + idempotencyKey: "roadmap-projection-change", keyValue: changedCapability.id, + expectedRowRevision: changedIdentity.rowRevision, title: `${changedCapability.name} — changed`, }), ); - expect(changedReceipt).toMatchObject({ - status: "updated", - ...changedIdentity, + expect(changedReceipt.receipt).toMatchObject({ + outcome: "updated", + row: { + itemId: changedIdentity.itemId, + documentId: changedIdentity.documentId, + }, }); const restoredReceipt = await asUser(OWNER, () => upsert.run({ - databaseId, - keyPropertyId, + ...mutationEnvelope, + idempotencyKey: "roadmap-projection-restore", keyValue: changedCapability.id, + expectedRowRevision: changedReceipt.receipt.row.rowRevision, title: changedCapability.name, - body: changedCapability.body, - propertyValues: { - [propertyIds.get("State")!]: changedCapability.state, - [propertyIds.get("Publicness")!]: changedCapability.publicness, - [propertyIds.get("User promise")!]: changedCapability.userPromise, - [propertyIds.get("Source revision")!]: SOURCE_REVISION, - }, + propertyValues: propertyValuesFor(changedCapability), }), ); - expect(restoredReceipt).toMatchObject({ - status: "updated", - ...changedIdentity, + expect(restoredReceipt.receipt).toMatchObject({ + outcome: "updated", + row: { + itemId: changedIdentity.itemId, + documentId: changedIdentity.documentId, + }, }); for (const capability of capabilities) { const receipt = await asUser(OWNER, () => upsert.run({ - databaseId, - keyPropertyId, + ...mutationEnvelope, + idempotencyKey: `roadmap-projection-${capability.id}`, keyValue: capability.id, + expectedRowRevision: null, title: capability.name, - body: capability.body, - propertyValues: { - [propertyIds.get("State")!]: capability.state, - [propertyIds.get("Publicness")!]: capability.publicness, - [propertyIds.get("User promise")!]: capability.userPromise, - [propertyIds.get("Source revision")!]: SOURCE_REVISION, - }, + propertyValues: propertyValuesFor(capability), }), ); - expect(receipt).toMatchObject({ - status: "unchanged", - ...firstReceipts.get(capability.id), + expect(receipt.receipt).toMatchObject({ + outcome: "created", + idempotency: { result: "replayed" }, + row: firstReceipts.get(capability.id), }); } @@ -282,7 +298,14 @@ describe("private roadmap capability projection", () => { expect(readbackIds).toEqual( new Set(capabilities.map((capability) => capability.id)), ); - expect(readbackIdentity).toEqual(firstReceipts); + expect(readbackIdentity).toEqual( + new Map( + [...firstReceipts].map(([key, { itemId, documentId }]) => [ + key, + { itemId, documentId }, + ]), + ), + ); const uniqueRoot = await asUser(OWNER, () => searchDocuments.run({ diff --git a/templates/content/actions/update-database-item.ts b/templates/content/actions/update-database-item.ts index 4fcd904e53..3ceb2e3e65 100644 --- a/templates/content/actions/update-database-item.ts +++ b/templates/content/actions/update-database-item.ts @@ -9,6 +9,7 @@ import { databasePropertyValuesSchema, } from "./_database-property-input.js"; import { + databaseMutationAgentTargetSchema, databaseMutationEnvelopeSchema, updateDatabaseRow, } from "./_database-row-mutation.js"; @@ -23,14 +24,17 @@ const schema = databaseMutationEnvelopeSchema.extend({ title: z.string().trim().min(1).max(500).optional(), propertyValues: databasePropertyValuesSchema, propertyEntries: databasePropertyEntriesSchema.describe( - "Sparse property patch as explicit entries; omitted fields are preserved and explicit null clears a value. Include one entry for every schema-valid writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", + "Sparse typed property patch as explicit entries; omitted fields are preserved and explicit null clears a value. Copy each propertyType from the discovered mutation contract and include one entry for every writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", ), }); +const agentSchema = schema + .extend({ target: databaseMutationAgentTargetSchema }) + .omit({ propertyValues: true }); export default defineAction({ description: "Sparsely update one exact Content database row by stable item and document IDs. Requires schema and row revisions, validates every non-Blocks property, and returns a verified idempotent receipt.", - agentInputSchema: schema.omit({ propertyValues: true }), + agentInputSchema: agentSchema, schema, http: { method: "PUT" }, audit: { diff --git a/templates/content/actions/upsert-database-item-by-key.db.test.ts b/templates/content/actions/upsert-database-item-by-key.db.test.ts index a817f07ffa..b0cc74cf97 100644 --- a/templates/content/actions/upsert-database-item-by-key.db.test.ts +++ b/templates/content/actions/upsert-database-item-by-key.db.test.ts @@ -95,7 +95,6 @@ function envelope( ) { return { target: { - authorityScope: discovered.target.authorityScope, spaceId: discovered.target.spaceId, databaseId: discovered.target.databaseId, databaseDocumentId: discovered.target.databaseDocumentId, @@ -215,6 +214,52 @@ describe("reliable Content database row mutations", () => { ); }); + it("derives authority and validates typed agent property entries against the discovered contract", async () => { + const ids = await fixture(); + const evidenceId = await addProperty({ + ...ids, + name: "Evidence", + type: "text", + }); + const discovered = await contract(ids.databaseId); + const input = { + ...envelope(discovered, "typed-agent-create"), + title: "Typed agent row", + propertyEntries: [ + { + propertyId: evidenceId, + propertyType: "text" as const, + value: "preserve me", + }, + ], + }; + + const created = await asOwner(() => createRow.run(input)); + expect(created.receipt.target.authorityScope).toEqual({ + kind: "personal", + id: OWNER, + }); + expect(created.receipt.readback.propertyValues[evidenceId]).toBe( + "preserve me", + ); + + await expect( + asOwner(() => + createRow.run({ + ...input, + idempotencyKey: "typed-agent-mismatched-type", + propertyEntries: [ + { + propertyId: evidenceId, + propertyType: "number", + value: 3314, + }, + ], + }), + ), + ).rejects.toMatchObject({ errorCode: "INVALID_PROPERTY_VALUE" }); + }); + it("creates every supported non-Blocks value without coercion and returns one durable verified receipt", async () => { const ids = await fixture(); const propertyIds = { diff --git a/templates/content/actions/upsert-database-item-by-key.ts b/templates/content/actions/upsert-database-item-by-key.ts index 93dcb1ef43..0dd5d4a7e4 100644 --- a/templates/content/actions/upsert-database-item-by-key.ts +++ b/templates/content/actions/upsert-database-item-by-key.ts @@ -9,6 +9,7 @@ import { databasePropertyValuesSchema, } from "./_database-property-input.js"; import { + databaseMutationAgentTargetSchema, databaseMutationEnvelopeSchema, upsertDatabaseRow, } from "./_database-row-mutation.js"; @@ -25,14 +26,17 @@ const schema = databaseMutationEnvelopeSchema.extend({ title: z.string().trim().min(1).max(500).optional(), propertyValues: databasePropertyValuesSchema, propertyEntries: databasePropertyEntriesSchema.describe( - "Sparse property values as explicit entries. Include one entry for every schema-valid writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", + "Sparse typed property values as explicit entries. Copy each propertyType from the discovered mutation contract and include one entry for every writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", ), }); +const agentSchema = schema + .extend({ target: databaseMutationAgentTargetSchema }) + .omit({ propertyValues: true }); export default defineAction({ description: "Create or sparsely update one Content database row by that database's explicitly configured natural key. Requires schema and row compare-and-swap revisions and returns a verified idempotent receipt.", - agentInputSchema: schema.omit({ propertyValues: true }), + agentInputSchema: agentSchema, schema, audit: { recordInputs: false, diff --git a/templates/content/parity/__tests__/database-row-property-input.test.ts b/templates/content/parity/__tests__/database-row-property-input.test.ts index 6e31ef055b..037dc942b9 100644 --- a/templates/content/parity/__tests__/database-row-property-input.test.ts +++ b/templates/content/parity/__tests__/database-row-property-input.test.ts @@ -5,7 +5,7 @@ import { databasePropertyEntriesSchema, normalizeDatabasePropertyInput, } from "../../actions/_database-property-input"; -import { digest } from "../../actions/_database-row-mutation"; +import { databaseMutationPayloadDigest } from "../../actions/_database-row-mutation"; import addDatabaseItem from "../../actions/add-database-item"; import updateDatabaseItem from "../../actions/update-database-item"; import upsertDatabaseItemByKey from "../../actions/upsert-database-item-by-key"; @@ -24,11 +24,13 @@ describe("database row property inputs", () => { expect(properties).not.toHaveProperty("propertyValues"); const propertyEntries = properties.propertyEntries; expect(propertyEntries.type).toBe("array"); - expect(propertyEntries.items.properties.propertyId.description).toContain( + expect(JSON.stringify(propertyEntries.items)).toContain( "Exact immutable property definition ID", ); + expect(JSON.stringify(propertyEntries.items)).toContain("propertyType"); + expect(JSON.stringify(propertyEntries.items)).not.toContain('"value":{}'); expect(propertyEntries.description).toContain( - "Include one entry for every schema-valid writable property value the user requested", + "include one entry for every writable property value the user requested", ); expect(propertyEntries.description).toContain( "never pass an empty array", @@ -36,6 +38,7 @@ describe("database row property inputs", () => { expect(propertyEntries.description).toContain( "Do not invent or clear unmentioned properties", ); + expect(properties.target.properties).not.toHaveProperty("authorityScope"); }, ); @@ -43,13 +46,27 @@ describe("database row property inputs", () => { expect( normalizeDatabasePropertyInput({ propertyEntries: [ - { propertyId: "status-id", value: "ready" }, - { propertyId: "evidence-id", value: "preserve me" }, + { + propertyId: "status-id", + propertyType: "status", + value: "ready", + }, + { + propertyId: "evidence-id", + propertyType: "text", + value: "preserve me", + }, ], }), ).toEqual({ - "status-id": "ready", - "evidence-id": "preserve me", + propertyValues: { + "status-id": "ready", + "evidence-id": "preserve me", + }, + propertyTypeAssertions: { + "status-id": "status", + "evidence-id": "text", + }, }); }); @@ -57,8 +74,16 @@ describe("database row property inputs", () => { expect(() => normalizeDatabasePropertyInput({ propertyEntries: [ - { propertyId: "status-id", value: "ready" }, - { propertyId: "status-id", value: "changed" }, + { + propertyId: "status-id", + propertyType: "status", + value: "ready", + }, + { + propertyId: "status-id", + propertyType: "status", + value: "changed", + }, ], }), ).toThrow(/provided more than once/); @@ -67,7 +92,13 @@ describe("database row property inputs", () => { it("rejects ambiguous entry and record inputs", () => { expect(() => normalizeDatabasePropertyInput({ - propertyEntries: [{ propertyId: "status-id", value: "ready" }], + propertyEntries: [ + { + propertyId: "status-id", + propertyType: "status", + value: "ready", + }, + ], propertyValues: { "status-id": "ready" }, }), ).toThrow(/not both/); @@ -75,27 +106,70 @@ describe("database row property inputs", () => { it("preserves __proto__ as an ordinary property definition ID", () => { const propertyEntries = databasePropertyEntriesSchema.parse([ - { propertyId: "__proto__", value: "preserve me" }, + { + propertyId: "__proto__", + propertyType: "text", + value: "preserve me", + }, ]); const normalized = normalizeDatabasePropertyInput({ propertyEntries, }); - expect(normalized).toBeDefined(); - expect(Object.getPrototypeOf(normalized)).toBeNull(); - expect(Object.keys(normalized!)).toEqual(["__proto__"]); - expect(Object.prototype.hasOwnProperty.call(normalized, "__proto__")).toBe( - true, - ); - expect(normalized?.["__proto__"]).toBe("preserve me"); + expect(Object.getPrototypeOf(normalized.propertyValues)).toBeNull(); + expect(Object.keys(normalized.propertyValues!)).toEqual(["__proto__"]); + expect( + Object.prototype.hasOwnProperty.call( + normalized.propertyValues, + "__proto__", + ), + ).toBe(true); + expect(normalized.propertyValues?.["__proto__"]).toBe("preserve me"); + expect(normalized.propertyTypeAssertions?.["__proto__"]).toBe("text"); + }); + + it("requires a schema-visible type and its matching JSON value shape", () => { + expect(() => + databasePropertyEntriesSchema.parse([ + { propertyId: "count-id", value: 3314 }, + ]), + ).toThrow(/propertyType/); + expect(() => + databasePropertyEntriesSchema.parse([ + { + propertyId: "count-id", + propertyType: "number", + value: "3314", + }, + ]), + ).toThrow(); + expect( + databasePropertyEntriesSchema.parse([ + { + propertyId: "count-id", + propertyType: "number", + value: 3314, + }, + ]), + ).toEqual([ + { propertyId: "count-id", propertyType: "number", value: 3314 }, + ]); }); it("removes the model-only representation before canonical hashing", () => { const canonical = canonicalizeDatabasePropertyInput({ idempotencyKey: "same-intent", propertyEntries: [ - { propertyId: "status-id", value: "ready" }, - { propertyId: "evidence-id", value: "preserve me" }, + { + propertyId: "status-id", + propertyType: "status", + value: "ready", + }, + { + propertyId: "evidence-id", + propertyType: "text", + value: "preserve me", + }, ], }); @@ -105,6 +179,10 @@ describe("database row property inputs", () => { "status-id": "ready", "evidence-id": "preserve me", }, + propertyTypeAssertions: { + "status-id": "status", + "evidence-id": "text", + }, }); expect(canonical).not.toHaveProperty("propertyEntries"); }); @@ -113,8 +191,16 @@ describe("database row property inputs", () => { const fromEntries = canonicalizeDatabasePropertyInput({ idempotencyKey: "same-intent", propertyEntries: [ - { propertyId: "status-id", value: "ready" }, - { propertyId: "evidence-id", value: "preserve me" }, + { + propertyId: "status-id", + propertyType: "status", + value: "ready", + }, + { + propertyId: "evidence-id", + propertyType: "text", + value: "preserve me", + }, ], }); const fromRecord = canonicalizeDatabasePropertyInput({ @@ -125,21 +211,29 @@ describe("database row property inputs", () => { }, }); - expect(digest(fromEntries)).toBe(digest(fromRecord)); + expect(databaseMutationPayloadDigest("create", fromEntries)).toBe( + databaseMutationPayloadDigest("create", fromRecord), + ); }); it("includes __proto__ property values in the canonical digest", () => { const withPrototypeNamedProperty = canonicalizeDatabasePropertyInput({ idempotencyKey: "same-intent", - propertyEntries: [{ propertyId: "__proto__", value: "preserve me" }], + propertyEntries: [ + { + propertyId: "__proto__", + propertyType: "text", + value: "preserve me", + }, + ], }); const withoutProperty = canonicalizeDatabasePropertyInput({ idempotencyKey: "same-intent", propertyValues: {}, }); - expect(digest(withPrototypeNamedProperty)).not.toBe( - digest(withoutProperty), - ); + expect( + databaseMutationPayloadDigest("create", withPrototypeNamedProperty), + ).not.toBe(databaseMutationPayloadDigest("create", withoutProperty)); }); }); diff --git a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts index bf0892c049..4e00f6e9bc 100644 --- a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts +++ b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts @@ -24,6 +24,45 @@ function successfulCreateCall( }; } +function typedExpectedEntries(scenario: (typeof parityEvalScenarios)[number]) { + return Object.entries(scenario.expectedPropertyValues ?? {}).map( + ([propertyId, value]) => ({ + propertyId, + propertyType: scenario.expectedPropertyTypes?.[propertyId], + value, + }), + ); +} + +const discoveryCalls = ["list-content-databases", "get-content-database"]; + +function successfulDiscoveryDetails( + scenario: (typeof parityEvalScenarios)[number], +) { + return [ + { + name: "list-content-databases", + input: { title: "PR #3314 feedback" }, + completed: true, + isError: false, + result: JSON.stringify({ + databases: [scenario.expectedCreateEnvelope?.target], + }), + }, + { + name: "get-content-database", + input: { + databaseId: scenario.expectedCreateEnvelope?.target.databaseId, + }, + completed: true, + isError: false, + result: JSON.stringify({ + mutationContract: scenario.expectedCreateEnvelope, + }), + }, + ]; +} + afterEach(() => { if (OLD_GATE === undefined) { delete process.env.CONTENT_PARITY_EVALS; @@ -171,9 +210,10 @@ describe("Content parity eval scenarios", () => { const row = await scoreEval(evalCase, { runAgent: vi.fn(async () => ({ text: scenario.successSignals.join("\n"), - toolCalls: ["add-database-item"], + toolCalls: [...discoveryCalls, "add-database-item"], toolCallDetails: [ - successfulCreateCall(scenario, { propertyValues: {} }), + ...successfulDiscoveryDetails(scenario), + successfulCreateCall(scenario, { propertyEntries: [] }), ], ok: true, runId: "content-parity:empty-property-values", @@ -199,10 +239,11 @@ describe("Content parity eval scenarios", () => { const row = await scoreEval(evalCase, { runAgent: vi.fn(async () => ({ text: scenario.successSignals.join("\n"), - toolCalls: ["add-database-item"], + toolCalls: [...discoveryCalls, "add-database-item"], toolCallDetails: [ + ...successfulDiscoveryDetails(scenario), successfulCreateCall(scenario, { - propertyValues: scenario.expectedPropertyValues, + propertyEntries: typedExpectedEntries(scenario), }), ], ok: true, @@ -226,9 +267,21 @@ describe("Content parity eval scenarios", () => { toolCallDetails: [ successfulCreateCall(scenario, { propertyEntries: [ - { propertyId: "parity-text-property-id", value: "preserve me" }, - { propertyId: "parity-text-property-id", value: "preserve me" }, - { propertyId: "parity-status-property-id", value: "ready" }, + { + propertyId: "fixture_evidence_property", + propertyType: "text", + value: "Baseline fixture preserve-me", + }, + { + propertyId: "fixture_evidence_property", + propertyType: "text", + value: "Baseline fixture preserve-me", + }, + { + propertyId: "fixture_status_property", + propertyType: "status", + value: "status-cannot-verify", + }, ], }), ], @@ -238,8 +291,16 @@ describe("Content parity eval scenarios", () => { toolCallDetails: [ successfulCreateCall(scenario, { propertyEntries: [ - { propertyId: "parity-text-property-id", value: "preserve me" }, - { propertyId: "parity-status-property-id", value: "ready" }, + { + propertyId: "fixture_evidence_property", + propertyType: "text", + value: "Baseline fixture preserve-me", + }, + { + propertyId: "fixture_status_property", + propertyType: "status", + value: "status-cannot-verify", + }, ], propertyValues: { "parity-text-property-id": "preserve me", @@ -253,8 +314,16 @@ describe("Content parity eval scenarios", () => { toolCallDetails: [ successfulCreateCall(scenario, { propertyEntries: [ - { propertyId: "parity-text-property-id", value: "preserve me" }, - { propertyId: "parity-status-property-id", value: "ready" }, + { + propertyId: "fixture_evidence_property", + propertyType: "text", + value: "Baseline fixture preserve-me", + }, + { + propertyId: "fixture_status_property", + propertyType: "status", + value: "status-cannot-verify", + }, ], }), { @@ -276,8 +345,14 @@ describe("Content parity eval scenarios", () => { const row = await scoreEval(evalCase, { runAgent: vi.fn(async () => ({ text: scenario.successSignals.join("\n"), - toolCalls: toolCallDetails.map((call) => call.name), - toolCallDetails, + toolCalls: [ + ...discoveryCalls, + ...toolCallDetails.map((call) => call.name), + ], + toolCallDetails: [ + ...successfulDiscoveryDetails(scenario), + ...toolCallDetails, + ], ok: true, runId: "content-parity:invalid-property-input", durationMs: 1, @@ -376,16 +451,14 @@ describe("Content parity eval scenarios", () => { name: "an extra property-entry field", mutate(call: ReturnType) { const input = call.input as Record; - const { propertyValues, ...withoutPropertyValues } = input; return { ...call, input: { - ...withoutPropertyValues, - propertyEntries: Object.entries( - propertyValues as Record, - ).map(([propertyId, value]) => ({ - propertyId, - value, + ...input, + propertyEntries: ( + input.propertyEntries as Array> + ).map((entry) => ({ + ...entry, hallucinated: true, })), }, @@ -399,15 +472,15 @@ describe("Content parity eval scenarios", () => { )!; const call = mutate( successfulCreateCall(scenario, { - propertyValues: scenario.expectedPropertyValues, + propertyEntries: typedExpectedEntries(scenario), }), ); const evalCase = scenarioToEval(scenario); const row = await scoreEval(evalCase, { runAgent: vi.fn(async () => ({ text: scenario.successSignals.join("\n"), - toolCalls: ["add-database-item"], - toolCallDetails: [call], + toolCalls: [...discoveryCalls, "add-database-item"], + toolCallDetails: [...successfulDiscoveryDetails(scenario), call], ok: true, runId: "content-parity:rejected-create", durationMs: 1, diff --git a/templates/content/parity/eval-scenarios.ts b/templates/content/parity/eval-scenarios.ts index 3c26553e2b..d8a91fc765 100644 --- a/templates/content/parity/eval-scenarios.ts +++ b/templates/content/parity/eval-scenarios.ts @@ -9,9 +9,9 @@ export interface ParityEvalScenario { successSignals: string[]; expectedTools?: string[]; expectedPropertyValues?: Record; + expectedPropertyTypes?: Record; expectedCreateEnvelope?: { target: { - authorityScope: { kind: "personal"; id: string }; spaceId: string; databaseId: string; databaseDocumentId: string; @@ -31,24 +31,29 @@ export const parityEvalScenarios: ParityEvalScenario[] = [ defaultState: "skipped", requiresPrivateCredentials: false, prompt: - "Create exactly one row in the already-verified fixture Content database. Do not rediscover or alter the target. Use target authorityScope { kind: personal, id: fixture-owner@example.test }, spaceId fixture_personal_space, databaseId fixture_feedback_database, and databaseDocumentId fixture_feedback_document; expectedSchemaRevision fixture_schema_revision; title [FIXTURE] preserve explicit properties; idempotencyKey fixture-create-property-preservation-v1. Set property fixture_status_property to status-cannot-verify and property fixture_evidence_property to Baseline fixture preserve-me. No Blocks value was requested. Call add-database-item once with every exact target constraint and both property entries, then report its result truthfully.", + "Find the fixture Content database titled PR #3314 feedback — do not treat 3314 as its database ID. Discover its exact database target and mutation contract through Content actions, then create exactly one row titled [FIXTURE] preserve explicit properties with idempotency key fixture-create-property-preservation-v1. Set Status to status-cannot-verify and Evidence to Baseline fixture preserve-me using the discovered immutable property IDs and property types. Omit authority identity because the authenticated server owns it. No Blocks value was requested. Call add-database-item once, then report its result truthfully.", successSignals: [ - "Uses add-database-item once for the exact fixture target.", + "Discovers the exact database and mutation contract before creating.", + "Uses add-database-item once for the discovered stable target without authoring authority identity.", "Preserves both explicitly requested writable property values.", "Does not invent a Blocks value or another property.", "Reports an action failure rather than claiming a row was created if the fixture is unavailable.", ], - expectedTools: ["add-database-item"], + expectedTools: [ + "list-content-databases", + "get-content-database", + "add-database-item", + ], expectedPropertyValues: { fixture_status_property: "status-cannot-verify", fixture_evidence_property: "Baseline fixture preserve-me", }, + expectedPropertyTypes: { + fixture_status_property: "status", + fixture_evidence_property: "text", + }, expectedCreateEnvelope: { target: { - authorityScope: { - kind: "personal", - id: "fixture-owner@example.test", - }, spaceId: "fixture_personal_space", databaseId: "fixture_feedback_database", databaseDocumentId: "fixture_feedback_document", diff --git a/templates/content/parity/scenario-to-eval.ts b/templates/content/parity/scenario-to-eval.ts index 6f7e50d378..39e985bb59 100644 --- a/templates/content/parity/scenario-to-eval.ts +++ b/templates/content/parity/scenario-to-eval.ts @@ -44,10 +44,15 @@ function hasExactKeys( function analyzePropertyValues(input: unknown): { received: Record; + receivedTypes: Record; invalid: string[]; } { if (!input || typeof input !== "object" || Array.isArray(input)) { - return { received: {}, invalid: ["tool input is not an object"] }; + return { + received: {}, + receivedTypes: {}, + invalid: ["tool input is not an object"], + }; } const record = input as Record; const hasEntries = record.propertyEntries !== undefined; @@ -55,6 +60,7 @@ function analyzePropertyValues(input: unknown): { if (hasEntries && hasValues) { return { received: {}, + receivedTypes: {}, invalid: ["propertyEntries and propertyValues were both provided"], }; } @@ -64,30 +70,50 @@ function analyzePropertyValues(input: unknown): { typeof record.propertyValues !== "object" || Array.isArray(record.propertyValues) ) { - return { received: {}, invalid: ["propertyValues is not a record"] }; + return { + received: {}, + receivedTypes: {}, + invalid: ["propertyValues is not a record"], + }; } return { received: record.propertyValues as Record, - invalid: [], + receivedTypes: {}, + invalid: ["propertyValues bypassed the typed agent input"], }; } if (!hasEntries || !Array.isArray(record.propertyEntries)) { - return { received: {}, invalid: ["propertyEntries is not an array"] }; + return { + received: {}, + receivedTypes: {}, + invalid: ["propertyEntries is not an array"], + }; } const received: Record = Object.create(null) as Record< string, unknown >; + const receivedTypes: Record = Object.create(null) as Record< + string, + string + >; const invalid: string[] = []; for (const entry of record.propertyEntries) { if (!entry || typeof entry !== "object" || Array.isArray(entry)) { invalid.push("propertyEntries contains a non-object entry"); continue; } - const { propertyId, value } = entry as Record; + const { propertyId, propertyType, value } = entry as Record< + string, + unknown + >; if ( - !hasExactKeys(entry as Record, ["propertyId", "value"]) + !hasExactKeys(entry as Record, [ + "propertyId", + "propertyType", + "value", + ]) ) { invalid.push( "propertyEntries contains an entry with unrecognized fields", @@ -98,13 +124,20 @@ function analyzePropertyValues(input: unknown): { invalid.push("propertyEntries contains an invalid propertyId"); continue; } + if (typeof propertyType !== "string" || propertyType.length === 0) { + invalid.push( + `propertyEntries contains an invalid type for ${propertyId}`, + ); + continue; + } if (Object.prototype.hasOwnProperty.call(received, propertyId)) { invalid.push(`propertyEntries contains duplicate ID ${propertyId}`); continue; } received[propertyId] = value; + receivedTypes[propertyId] = propertyType; } - return { received, invalid }; + return { received, receivedTypes, invalid }; } const databaseRowMutationTools = new Set([ @@ -124,15 +157,6 @@ function matchesCreateEnvelope( return false; } const actualTarget = target as Record; - const authorityScope = actualTarget.authorityScope; - if ( - !authorityScope || - typeof authorityScope !== "object" || - Array.isArray(authorityScope) - ) { - return false; - } - const actualAuthority = authorityScope as Record; return ( hasExactKeys(input, [ "target", @@ -144,14 +168,10 @@ function matchesCreateEnvelope( : "propertyEntries", ]) && hasExactKeys(actualTarget, [ - "authorityScope", "spaceId", "databaseId", "databaseDocumentId", ]) && - hasExactKeys(actualAuthority, ["kind", "id"]) && - actualAuthority.kind === expected.target.authorityScope.kind && - actualAuthority.id === expected.target.authorityScope.id && actualTarget.spaceId === expected.target.spaceId && actualTarget.databaseId === expected.target.databaseId && actualTarget.databaseDocumentId === expected.target.databaseDocumentId && @@ -163,6 +183,7 @@ function matchesCreateEnvelope( function expectedPropertyValuesScorer( expected: Record, + expectedTypes: Record | undefined, expectedEnvelope?: ParityEvalScenario["expectedCreateEnvelope"], ) { return createScorer< @@ -195,6 +216,45 @@ function expectedPropertyValuesScorer( `expected exactly one row mutation, received ${mutationCalls.length}`, ); } + const orderedCalls = run.toolCalls; + const listIndex = orderedCalls.indexOf("list-content-databases"); + const inspectIndex = orderedCalls.indexOf("get-content-database"); + const createIndex = orderedCalls.indexOf("add-database-item"); + if ( + listIndex < 0 || + inspectIndex <= listIndex || + createIndex <= inspectIndex + ) { + invalid.push("database discovery did not precede the create mutation"); + } + const listCall = (run.toolCallDetails ?? []).find( + (call) => call.name === "list-content-databases", + ); + const inspectCall = (run.toolCallDetails ?? []).find( + (call) => call.name === "get-content-database", + ); + const listInput = listCall?.input as Record | undefined; + const inspectInput = inspectCall?.input as + | Record + | undefined; + if ( + expectedEnvelope && + (listInput?.title !== "PR #3314 feedback" || + inspectInput?.databaseId !== expectedEnvelope.target.databaseId) + ) { + invalid.push( + "discovery did not resolve the requested title to the exact create target", + ); + } + for (const [propertyId, expectedType] of Object.entries( + expectedTypes ?? {}, + )) { + if (analysis.receivedTypes[propertyId] !== expectedType) { + invalid.push( + `property ${propertyId} did not declare discovered type ${expectedType}`, + ); + } + } const createInput = createCalls[0]?.input; if ( expectedEnvelope && @@ -278,6 +338,7 @@ export function scenarioToEval(scenario: ParityEvalScenario): Eval { ? [ expectedPropertyValuesScorer( scenario.expectedPropertyValues, + scenario.expectedPropertyTypes, scenario.expectedCreateEnvelope, ), ] From 3097054346823e73370e7f7a7d92f47d79e14e01 Mon Sep 17 00:00:00 2001 From: "Builder.io" Date: Tue, 25 Aug 2026 13:02:02 +0000 Subject: [PATCH 7/9] fix: address review feedback on Content mutation parity - Exclude legacy authorityScope from the mutation payload digest so retries through old callers replay instead of colliding on IDEMPOTENCY_KEY_REUSED. - Reject empty propertyEntries arrays instead of silently canonicalizing them to a no-op mutation. - Check durable replay before typed property assertion validation so a retried committed mutation replays even if the property's type later changed. - Register list-content-databases and get-content-database fixtures in the dedicated property-preservation runner so discovery can succeed before the create call. --- .../actions/_database-property-input.ts | 1 + .../content/actions/_database-row-mutation.ts | 19 ++-- ...n-database-create-property-preservation.ts | 97 ++++++++++++++++++- 3 files changed, 110 insertions(+), 7 deletions(-) diff --git a/templates/content/actions/_database-property-input.ts b/templates/content/actions/_database-property-input.ts index 5e89b4ccbc..4a9a6bf548 100644 --- a/templates/content/actions/_database-property-input.ts +++ b/templates/content/actions/_database-property-input.ts @@ -133,6 +133,7 @@ export const databasePropertyValuesSchema = z export const databasePropertyEntriesSchema = z .array(databasePropertyEntrySchema) + .min(1) .max(1_000) .optional() .describe( diff --git a/templates/content/actions/_database-row-mutation.ts b/templates/content/actions/_database-row-mutation.ts index 5b8372030a..33f07ab716 100644 --- a/templates/content/actions/_database-row-mutation.ts +++ b/templates/content/actions/_database-row-mutation.ts @@ -741,9 +741,13 @@ export function databaseMutationPayloadDigest( | UpdateDatabaseRowMutationInput | UpsertDatabaseRowMutationInput, ) { - const { propertyTypeAssertions: _propertyTypeAssertions, ...canonicalInput } = - input; - return digest({ operation, ...canonicalInput }); + const { + propertyTypeAssertions: _propertyTypeAssertions, + target, + ...canonicalInput + } = input; + const { authorityScope: _authorityScope, ...stableTarget } = target ?? {}; + return digest({ operation, ...canonicalInput, target: stableTarget }); } function assertPropertyTypeAssertions( @@ -1187,7 +1191,6 @@ export async function createDatabaseRow( input: CreateDatabaseRowMutationInput, ): Promise { const initial = await loadContext(input.target, "editor"); - assertPropertyTypeAssertions(initial, input.propertyTypeAssertions); const inputDigest = databaseMutationPayloadDigest("create", input); const replay = await replayReceipt( initial, @@ -1195,6 +1198,7 @@ export async function createDatabaseRow( inputDigest, ); if (replay) return replay; + assertPropertyTypeAssertions(initial, input.propertyTypeAssertions); assertSchema(initial, input.expectedSchemaRevision); const values = await normalizePatch(initial, input.propertyValues); const result = await withMutationLocks(initial.database, () => @@ -1215,6 +1219,7 @@ export async function createDatabaseRow( tx as unknown as Db, ); if (lockedReplay) return lockedReplay; + assertPropertyTypeAssertions(locked, input.propertyTypeAssertions); assertSchema(locked, input.expectedSchemaRevision); await touchContentDatabase( tx as unknown as Db, @@ -1256,7 +1261,6 @@ export async function updateDatabaseRow( input: UpdateDatabaseRowMutationInput, ): Promise { const initial = await loadContext(input.target, "editor"); - assertPropertyTypeAssertions(initial, input.propertyTypeAssertions); await assertAccess("document", input.documentId, "editor"); const inputDigest = databaseMutationPayloadDigest("update", input); const replay = await replayReceipt( @@ -1265,6 +1269,7 @@ export async function updateDatabaseRow( inputDigest, ); if (replay) return replay; + assertPropertyTypeAssertions(initial, input.propertyTypeAssertions); assertSchema(initial, input.expectedSchemaRevision); const values = await normalizePatch(initial, input.propertyValues); const result = await withMutationLocks(initial.database, () => @@ -1285,6 +1290,7 @@ export async function updateDatabaseRow( tx as unknown as Db, ); if (lockedReplay) return lockedReplay; + assertPropertyTypeAssertions(locked, input.propertyTypeAssertions); assertSchema(locked, input.expectedSchemaRevision); const updated = await updateInsideTransaction( tx as unknown as Db, @@ -1337,7 +1343,6 @@ export async function upsertDatabaseRow( input: UpsertDatabaseRowMutationInput, ): Promise { const initial = await loadContext(input.target, "editor"); - assertPropertyTypeAssertions(initial, input.propertyTypeAssertions); const replayKeyPropertyId = initial.database.naturalKeyPropertyId; const replayKeyDefinition = replayKeyPropertyId ? initial.definitions.find( @@ -1365,6 +1370,7 @@ export async function upsertDatabaseRow( inputDigest, ); if (replay) return replay; + assertPropertyTypeAssertions(initial, input.propertyTypeAssertions); assertSchema(initial, input.expectedSchemaRevision); const keyPropertyId = initial.database.naturalKeyPropertyId; if (!keyPropertyId) { @@ -1425,6 +1431,7 @@ export async function upsertDatabaseRow( tx as unknown as Db, ); if (lockedReplay) return lockedReplay; + assertPropertyTypeAssertions(locked, input.propertyTypeAssertions); assertSchema(locked, input.expectedSchemaRevision); if (locked.database.naturalKeyPropertyId !== keyPropertyId) { conflict("SCHEMA_REVISION_CONFLICT", "The natural key changed."); diff --git a/templates/content/parity/run-database-create-property-preservation.ts b/templates/content/parity/run-database-create-property-preservation.ts index 1713a9fd52..ac1f4bc755 100644 --- a/templates/content/parity/run-database-create-property-preservation.ts +++ b/templates/content/parity/run-database-create-property-preservation.ts @@ -1,6 +1,8 @@ import { createAgentRunner, runEvals } from "@agent-native/core/eval"; import addDatabaseItem from "../actions/add-database-item.ts"; +import getContentDatabase from "../actions/get-content-database.ts"; +import listContentDatabases from "../actions/list-content-databases.ts"; import { parityEvalScenarios } from "./eval-scenarios.ts"; import { scenarioToEval } from "./scenario-to-eval.ts"; @@ -11,6 +13,12 @@ if (!scenario) { throw new Error("Missing database create property preservation scenario."); } +const FIXTURE_SPACE_ID = "fixture_personal_space"; +const FIXTURE_DATABASE_ID = "fixture_feedback_database"; +const FIXTURE_DOCUMENT_ID = "fixture_feedback_document"; +const FIXTURE_SCHEMA_REVISION = "fixture_schema_revision"; +const FIXTURE_DATABASE_TITLE = "PR #3314 feedback"; + const evalCase = scenarioToEval(scenario); evalCase.scorers = evalCase.scorers.filter( (scorer) => @@ -20,13 +28,100 @@ evalCase.scorers = evalCase.scorers.filter( const runner = await createAgentRunner({ actions: { + "list-content-databases": { + ...listContentDatabases, + run: async () => ({ + databases: [ + { + databaseId: FIXTURE_DATABASE_ID, + documentId: FIXTURE_DOCUMENT_ID, + spaceId: FIXTURE_SPACE_ID, + title: FIXTURE_DATABASE_TITLE, + description: "", + }, + ], + pagination: { + offset: 0, + limit: 50, + totalItems: 1, + returnedItems: 1, + hasMore: false, + nextOffset: null, + }, + }), + }, + "get-content-database": { + ...getContentDatabase, + run: async () => ({ + database: { + id: FIXTURE_DATABASE_ID, + documentId: FIXTURE_DOCUMENT_ID, + spaceId: FIXTURE_SPACE_ID, + title: FIXTURE_DATABASE_TITLE, + naturalKeyPropertyId: null, + viewConfig: { + activeViewId: "fixture_view", + views: [], + sorts: [], + filters: [], + columnWidths: {}, + }, + createdAt: new Date(0).toISOString(), + updatedAt: new Date(0).toISOString(), + }, + properties: [], + items: [], + source: null, + mutationContract: { + target: { + authorityScope: { + kind: "personal", + id: "fixture-owner@example.com", + }, + spaceId: FIXTURE_SPACE_ID, + databaseId: FIXTURE_DATABASE_ID, + databaseDocumentId: FIXTURE_DOCUMENT_ID, + }, + schemaRevision: FIXTURE_SCHEMA_REVISION, + naturalKeyPropertyId: null, + properties: [ + { + id: "fixture_status_property", + name: "Status", + type: "status", + writable: true, + sourceManaged: false, + acceptedShape: null, + options: { + options: [ + { + id: "status-cannot-verify", + name: "Cannot verify", + color: "gray", + }, + ], + }, + }, + { + id: "fixture_evidence_property", + name: "Evidence", + type: "text", + writable: true, + sourceManaged: false, + acceptedShape: null, + options: {}, + }, + ], + }, + }), + }, "add-database-item": { ...addDatabaseItem, run: async (input) => ({ fixtureOnly: true, received: input }), }, }, systemPrompt: - "You are Content's AI document assistant. Use the registered Content action and preserve exact user-supplied target constraints, property IDs, and property values. Never invent fields or claim an action succeeded when it failed.", + "You are Content's AI document assistant. Use the registered Content actions and preserve exact user-supplied target constraints, property IDs, and property values. Never invent fields or claim an action succeeded when it failed.", }); const report = await runEvals([evalCase], runner, { persist: false }); From 55eb6fd82eeb14a34da3c55b585deefe6c505cee Mon Sep 17 00:00:00 2001 From: "Builder.io" Date: Tue, 25 Aug 2026 13:26:23 +0000 Subject: [PATCH 8/9] fix: require successful discovery before accepting the property preservation create call expected_property_values now requires list-content-databases and get-content-database to complete without error and validates their result payloads (discovered database and mutation contract target/schema revision) match the fixture create target, instead of only checking call order and input shape. --- templates/content/parity/scenario-to-eval.ts | 66 ++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/templates/content/parity/scenario-to-eval.ts b/templates/content/parity/scenario-to-eval.ts index 39e985bb59..27ab3697c6 100644 --- a/templates/content/parity/scenario-to-eval.ts +++ b/templates/content/parity/scenario-to-eval.ts @@ -181,6 +181,18 @@ function matchesCreateEnvelope( ); } +function parseToolResultJson(result: string | undefined): unknown { + if (!result) return undefined; + try { + return JSON.parse(result); + } catch { + // coercion-ok: an unparseable tool result fails the same "did not supply + // the expected target/contract" scorer checks below as a missing one — + // both are non-passing evidence, not a distinction the eval needs to make. + return undefined; + } +} + function expectedPropertyValuesScorer( expected: Record, expectedTypes: Record | undefined, @@ -246,6 +258,60 @@ function expectedPropertyValuesScorer( "discovery did not resolve the requested title to the exact create target", ); } + if ( + !listCall?.completed || + listCall.isError || + !inspectCall?.completed || + inspectCall.isError + ) { + invalid.push("database discovery calls did not complete successfully"); + } + if (expectedEnvelope) { + const listResult = parseToolResultJson(listCall?.result) as + | { databases?: Array> } + | undefined; + const discoveredDatabase = listResult?.databases?.find( + (database) => + database.databaseId === expectedEnvelope.target.databaseId, + ); + const discoveredDocumentId = + discoveredDatabase?.documentId ?? + discoveredDatabase?.databaseDocumentId; + if ( + !discoveredDatabase || + discoveredDocumentId !== expectedEnvelope.target.databaseDocumentId || + discoveredDatabase.spaceId !== expectedEnvelope.target.spaceId + ) { + invalid.push( + "list-content-databases result did not supply the expected create target", + ); + } + const inspectResult = parseToolResultJson(inspectCall?.result) as + | { + mutationContract?: { + target?: Record; + schemaRevision?: string; + expectedSchemaRevision?: string; + }; + } + | undefined; + const contractTarget = inspectResult?.mutationContract?.target; + const contractSchemaRevision = + inspectResult?.mutationContract?.schemaRevision ?? + inspectResult?.mutationContract?.expectedSchemaRevision; + if ( + !contractTarget || + contractTarget.spaceId !== expectedEnvelope.target.spaceId || + contractTarget.databaseId !== expectedEnvelope.target.databaseId || + contractTarget.databaseDocumentId !== + expectedEnvelope.target.databaseDocumentId || + contractSchemaRevision !== expectedEnvelope.expectedSchemaRevision + ) { + invalid.push( + "get-content-database result did not supply the expected mutation contract", + ); + } + } for (const [propertyId, expectedType] of Object.entries( expectedTypes ?? {}, )) { From 9e760783396e9e69ff4ec588e6107fd768506cf3 Mon Sep 17 00:00:00 2001 From: Alice Moore <86723305+3mdistal@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:47:10 +0000 Subject: [PATCH 9/9] fix: preserve Content mutation replay compatibility --- packages/core/src/eval/agent-runner.ts | 6 ++ packages/core/src/eval/runner.spec.ts | 4 ++ packages/core/src/eval/types.ts | 2 + .../content/actions/_database-row-mutation.ts | 67 ++++++++++++++++--- .../database-row-property-input.test.ts | 33 ++++++++- .../__tests__/eval-scenario-coverage.test.ts | 18 ++++- templates/content/parity/scenario-to-eval.ts | 33 +++++++++ 7 files changed, 150 insertions(+), 13 deletions(-) diff --git a/packages/core/src/eval/agent-runner.ts b/packages/core/src/eval/agent-runner.ts index c986194a0e..4527c6bb94 100644 --- a/packages/core/src/eval/agent-runner.ts +++ b/packages/core/src/eval/agent-runner.ts @@ -124,6 +124,8 @@ export async function createAgentRunner( name: string; id?: string; input: unknown; + startedAtEventIndex: number; + completedAtEventIndex?: number; completed?: boolean; completedSideEffect?: boolean; isError?: boolean; @@ -131,12 +133,14 @@ export async function createAgentRunner( }> = []; let ok = true; let error: string | undefined; + let eventIndex = 0; const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); const started = Date.now(); const send = (event: AgentChatEvent): void => { + const currentEventIndex = eventIndex++; switch (event.type) { case "text": text += event.text; @@ -147,6 +151,7 @@ export async function createAgentRunner( name: event.tool, id: event.id, input: event.input, + startedAtEventIndex: currentEventIndex, }); break; case "tool_done": { @@ -157,6 +162,7 @@ export async function createAgentRunner( ); if (detail) { detail.completed = true; + detail.completedAtEventIndex = currentEventIndex; detail.completedSideEffect = event.completedSideEffect; detail.isError = event.isError === true; detail.result = event.result; diff --git a/packages/core/src/eval/runner.spec.ts b/packages/core/src/eval/runner.spec.ts index 389844d1dd..b80f197ef8 100644 --- a/packages/core/src/eval/runner.spec.ts +++ b/packages/core/src/eval/runner.spec.ts @@ -328,6 +328,8 @@ describe("createAgentRunner over a mocked runAgentLoop (no real model)", () => { { name: "search", input: {}, + startedAtEventIndex: 1, + completedAtEventIndex: 2, completed: true, completedSideEffect: true, isError: false, @@ -336,6 +338,8 @@ describe("createAgentRunner over a mocked runAgentLoop (no real model)", () => { { name: "update", input: {}, + startedAtEventIndex: 3, + completedAtEventIndex: 4, completed: true, completedSideEffect: false, isError: false, diff --git a/packages/core/src/eval/types.ts b/packages/core/src/eval/types.ts index 81e32e03a8..dd597fb50e 100644 --- a/packages/core/src/eval/types.ts +++ b/packages/core/src/eval/types.ts @@ -35,6 +35,8 @@ export interface AgentRunOutput { readonly toolCallDetails?: readonly { readonly name: string; readonly input: unknown; + readonly startedAtEventIndex?: number; + readonly completedAtEventIndex?: number; readonly completed?: boolean; readonly completedSideEffect?: boolean; readonly isError?: boolean; diff --git a/templates/content/actions/_database-row-mutation.ts b/templates/content/actions/_database-row-mutation.ts index 33f07ab716..e9977486c3 100644 --- a/templates/content/actions/_database-row-mutation.ts +++ b/templates/content/actions/_database-row-mutation.ts @@ -750,6 +750,29 @@ export function databaseMutationPayloadDigest( return digest({ operation, ...canonicalInput, target: stableTarget }); } +export function legacyDatabaseMutationPayloadDigest( + operation: DatabaseRowMutationOperation, + input: + | CreateDatabaseRowMutationInput + | UpdateDatabaseRowMutationInput + | UpsertDatabaseRowMutationInput, + authorityScope = input.target.authorityScope, +) { + const { propertyTypeAssertions: _propertyTypeAssertions, ...legacyInput } = + input; + return digest({ + operation, + ...legacyInput, + target: { ...legacyInput.target, authorityScope }, + }); +} + +function authorityScopeForContext(context: MutationContext) { + return context.database.orgId + ? ({ kind: "organization", id: context.database.orgId } as const) + : ({ kind: "personal", id: context.database.ownerEmail } as const); +} + function assertPropertyTypeAssertions( context: MutationContext, assertions: Record | undefined, @@ -795,9 +818,7 @@ function resultForReceipt( }, ): ContentDatabaseRowMutationResult { const target = { - authorityScope: context.database.orgId - ? ({ kind: "organization", id: context.database.orgId } as const) - : ({ kind: "personal", id: context.database.ownerEmail } as const), + authorityScope: authorityScopeForContext(context), spaceId: context.database.spaceId!, databaseId: context.database.id, databaseDocumentId: context.database.documentId, @@ -844,7 +865,7 @@ function resultForReceipt( async function replayReceipt( context: MutationContext, idempotencyKey: string, - expectedPayloadDigest: string, + expectedPayloadDigests: readonly string[], db: Db = getDb(), ): Promise { const [stored] = await db @@ -863,7 +884,7 @@ async function replayReceipt( ), ); if (!stored) return null; - if (stored.payloadDigest !== expectedPayloadDigest) { + if (!expectedPayloadDigests.includes(stored.payloadDigest)) { conflict( "IDEMPOTENCY_KEY_REUSED", "This idempotency key was already used for a different row mutation.", @@ -1192,10 +1213,18 @@ export async function createDatabaseRow( ): Promise { const initial = await loadContext(input.target, "editor"); const inputDigest = databaseMutationPayloadDigest("create", input); + const replayDigests = [ + inputDigest, + legacyDatabaseMutationPayloadDigest( + "create", + input, + authorityScopeForContext(initial), + ), + ]; const replay = await replayReceipt( initial, input.idempotencyKey, - inputDigest, + replayDigests, ); if (replay) return replay; assertPropertyTypeAssertions(initial, input.propertyTypeAssertions); @@ -1215,7 +1244,7 @@ export async function createDatabaseRow( const lockedReplay = await replayReceipt( locked, input.idempotencyKey, - inputDigest, + replayDigests, tx as unknown as Db, ); if (lockedReplay) return lockedReplay; @@ -1263,10 +1292,18 @@ export async function updateDatabaseRow( const initial = await loadContext(input.target, "editor"); await assertAccess("document", input.documentId, "editor"); const inputDigest = databaseMutationPayloadDigest("update", input); + const replayDigests = [ + inputDigest, + legacyDatabaseMutationPayloadDigest( + "update", + input, + authorityScopeForContext(initial), + ), + ]; const replay = await replayReceipt( initial, input.idempotencyKey, - inputDigest, + replayDigests, ); if (replay) return replay; assertPropertyTypeAssertions(initial, input.propertyTypeAssertions); @@ -1286,7 +1323,7 @@ export async function updateDatabaseRow( const lockedReplay = await replayReceipt( locked, input.idempotencyKey, - inputDigest, + replayDigests, tx as unknown as Db, ); if (lockedReplay) return lockedReplay; @@ -1364,10 +1401,18 @@ export async function upsertDatabaseRow( ); } const inputDigest = databaseMutationPayloadDigest("upsert", input); + const replayDigests = [ + inputDigest, + legacyDatabaseMutationPayloadDigest( + "upsert", + input, + authorityScopeForContext(initial), + ), + ]; const replay = await replayReceipt( initial, input.idempotencyKey, - inputDigest, + replayDigests, ); if (replay) return replay; assertPropertyTypeAssertions(initial, input.propertyTypeAssertions); @@ -1427,7 +1472,7 @@ export async function upsertDatabaseRow( const lockedReplay = await replayReceipt( locked, input.idempotencyKey, - inputDigest, + replayDigests, tx as unknown as Db, ); if (lockedReplay) return lockedReplay; diff --git a/templates/content/parity/__tests__/database-row-property-input.test.ts b/templates/content/parity/__tests__/database-row-property-input.test.ts index 037dc942b9..63e2a41e33 100644 --- a/templates/content/parity/__tests__/database-row-property-input.test.ts +++ b/templates/content/parity/__tests__/database-row-property-input.test.ts @@ -5,7 +5,10 @@ import { databasePropertyEntriesSchema, normalizeDatabasePropertyInput, } from "../../actions/_database-property-input"; -import { databaseMutationPayloadDigest } from "../../actions/_database-row-mutation"; +import { + databaseMutationPayloadDigest, + legacyDatabaseMutationPayloadDigest, +} from "../../actions/_database-row-mutation"; import addDatabaseItem from "../../actions/add-database-item"; import updateDatabaseItem from "../../actions/update-database-item"; import upsertDatabaseItemByKey from "../../actions/upsert-database-item-by-key"; @@ -216,6 +219,34 @@ describe("database row property inputs", () => { ); }); + it("retains the authority-bearing legacy digest for receipt replay", () => { + const input = canonicalizeDatabasePropertyInput({ + idempotencyKey: "existing-receipt", + target: { + authorityScope: { kind: "personal", id: "owner@example.com" }, + spaceId: "space-id", + databaseId: "database-id", + databaseDocumentId: "database-document-id", + }, + propertyValues: { "status-id": "ready" }, + }); + + expect(legacyDatabaseMutationPayloadDigest("create", input)).not.toBe( + databaseMutationPayloadDigest("create", input), + ); + const withoutAuthoredAuthority = { + ...input, + target: { ...input.target, authorityScope: undefined }, + }; + expect( + legacyDatabaseMutationPayloadDigest( + "create", + withoutAuthoredAuthority, + input.target.authorityScope, + ), + ).toBe(legacyDatabaseMutationPayloadDigest("create", input)); + }); + it("includes __proto__ property values in the canonical digest", () => { const withPrototypeNamedProperty = canonicalizeDatabasePropertyInput({ idempotencyKey: "same-intent", diff --git a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts index 4e00f6e9bc..9edfa373c3 100644 --- a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts +++ b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts @@ -17,6 +17,8 @@ function successfulCreateCall( ...scenario.expectedCreateEnvelope, ...propertyInput, }, + startedAtEventIndex: 4, + completedAtEventIndex: 5, completed: true, completedSideEffect: true, isError: false, @@ -43,6 +45,8 @@ function successfulDiscoveryDetails( { name: "list-content-databases", input: { title: "PR #3314 feedback" }, + startedAtEventIndex: 0, + completedAtEventIndex: 1, completed: true, isError: false, result: JSON.stringify({ @@ -54,10 +58,22 @@ function successfulDiscoveryDetails( input: { databaseId: scenario.expectedCreateEnvelope?.target.databaseId, }, + startedAtEventIndex: 2, + completedAtEventIndex: 3, completed: true, isError: false, result: JSON.stringify({ - mutationContract: scenario.expectedCreateEnvelope, + mutationContract: { + ...scenario.expectedCreateEnvelope, + properties: Object.entries(scenario.expectedPropertyTypes ?? {}).map( + ([id, type]) => ({ + id, + type, + writable: true, + sourceManaged: false, + }), + ), + }, }), }, ]; diff --git a/templates/content/parity/scenario-to-eval.ts b/templates/content/parity/scenario-to-eval.ts index 27ab3697c6..12a4aa36c2 100644 --- a/templates/content/parity/scenario-to-eval.ts +++ b/templates/content/parity/scenario-to-eval.ts @@ -245,6 +245,7 @@ function expectedPropertyValuesScorer( const inspectCall = (run.toolCallDetails ?? []).find( (call) => call.name === "get-content-database", ); + const createCall = createCalls[0]; const listInput = listCall?.input as Record | undefined; const inspectInput = inspectCall?.input as | Record @@ -266,6 +267,18 @@ function expectedPropertyValuesScorer( ) { invalid.push("database discovery calls did not complete successfully"); } + if ( + listCall?.completedAtEventIndex === undefined || + inspectCall?.startedAtEventIndex === undefined || + listCall.completedAtEventIndex >= inspectCall.startedAtEventIndex || + inspectCall.completedAtEventIndex === undefined || + createCall?.startedAtEventIndex === undefined || + inspectCall.completedAtEventIndex >= createCall.startedAtEventIndex + ) { + invalid.push( + "database discovery results were not available before dependent calls started", + ); + } if (expectedEnvelope) { const listResult = parseToolResultJson(listCall?.result) as | { databases?: Array> } @@ -292,6 +305,7 @@ function expectedPropertyValuesScorer( target?: Record; schemaRevision?: string; expectedSchemaRevision?: string; + properties?: Array>; }; } | undefined; @@ -311,6 +325,25 @@ function expectedPropertyValuesScorer( "get-content-database result did not supply the expected mutation contract", ); } + const discoveredProperties = new Map( + (inspectResult?.mutationContract?.properties ?? []).map( + (property) => [property.id, property], + ), + ); + for (const [propertyId, expectedType] of Object.entries( + expectedTypes ?? {}, + )) { + const property = discoveredProperties.get(propertyId); + if ( + property?.type !== expectedType || + property.writable !== true || + property.sourceManaged === true + ) { + invalid.push( + `discovery did not supply writable property ${propertyId} with type ${expectedType}`, + ); + } + } } for (const [propertyId, expectedType] of Object.entries( expectedTypes ?? {},