diff --git a/.changeset/calm-tools-preserve-inputs.md b/.changeset/calm-tools-preserve-inputs.md new file mode 100644 index 0000000000..268e397a4a --- /dev/null +++ b/.changeset/calm-tools-preserve-inputs.md @@ -0,0 +1,5 @@ +--- +"@agent-native/core": patch +--- + +Expose model-produced tool inputs to eval scorers so argument-level agent behavior can be verified. diff --git a/packages/core/src/eval/agent-runner.ts b/packages/core/src/eval/agent-runner.ts index b8599bd85b..4527c6bb94 100644 --- a/packages/core/src/eval/agent-runner.ts +++ b/packages/core/src/eval/agent-runner.ts @@ -120,21 +120,55 @@ export async function createAgentRunner( let text = ""; const toolCalls: string[] = []; + const toolCallDetails: Array<{ + name: string; + id?: string; + input: unknown; + startedAtEventIndex: number; + completedAtEventIndex?: number; + completed?: boolean; + completedSideEffect?: boolean; + isError?: boolean; + result?: string; + }> = []; let ok = true; let error: string | undefined; + let eventIndex = 0; const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); const started = Date.now(); const send = (event: AgentChatEvent): void => { + const currentEventIndex = eventIndex++; switch (event.type) { case "text": text += event.text; break; case "tool_start": toolCalls.push(event.tool); + toolCallDetails.push({ + name: event.tool, + id: event.id, + input: event.input, + startedAtEventIndex: currentEventIndex, + }); + break; + case "tool_done": { + const detail = event.id + ? toolCallDetails.find((call) => call.id === event.id) + : toolCallDetails.find( + (call) => call.name === event.tool && !call.completed, + ); + if (detail) { + detail.completed = true; + detail.completedAtEventIndex = currentEventIndex; + detail.completedSideEffect = event.completedSideEffect; + detail.isError = event.isError === true; + detail.result = event.result; + } break; + } case "error": ok = false; error = event.error; @@ -165,6 +199,7 @@ export async function createAgentRunner( return { text, toolCalls, + toolCallDetails: toolCallDetails.map(({ id: _id, ...detail }) => detail), ok, error, runId, diff --git a/packages/core/src/eval/runner.spec.ts b/packages/core/src/eval/runner.spec.ts index fef74b3690..b80f197ef8 100644 --- a/packages/core/src/eval/runner.spec.ts +++ b/packages/core/src/eval/runner.spec.ts @@ -276,7 +276,32 @@ describe("createAgentRunner over a mocked runAgentLoop (no real model)", () => { const runLoop = vi.fn( async (opts: { send: (e: AgentChatEvent) => void }) => { opts.send({ type: "text", text: "Hello " }); - opts.send({ type: "tool_start", tool: "search", input: {} }); + opts.send({ + type: "tool_start", + tool: "search", + id: "search-1", + input: {}, + }); + opts.send({ + type: "tool_done", + tool: "search", + id: "search-1", + result: '{"ok":true}', + completedSideEffect: true, + }); + opts.send({ + type: "tool_start", + tool: "update", + id: "update-1", + input: {}, + }); + opts.send({ + type: "tool_done", + tool: "update", + id: "update-1", + result: '{"ok":false}', + completedSideEffect: false, + }); opts.send({ type: "text", text: "world" }); return { inputTokens: 0, @@ -298,7 +323,29 @@ describe("createAgentRunner over a mocked runAgentLoop (no real model)", () => { const out = await runner.runAgent({ prompt: "hi" }); expect(out.text).toBe("Hello world"); - expect(out.toolCalls).toEqual(["search"]); + expect(out.toolCalls).toEqual(["search", "update"]); + expect(out.toolCallDetails).toEqual([ + { + name: "search", + input: {}, + startedAtEventIndex: 1, + completedAtEventIndex: 2, + completed: true, + completedSideEffect: true, + isError: false, + result: '{"ok":true}', + }, + { + name: "update", + input: {}, + startedAtEventIndex: 3, + completedAtEventIndex: 4, + completed: true, + completedSideEffect: false, + isError: false, + result: '{"ok":false}', + }, + ]); expect(out.ok).toBe(true); // End-to-end: a contains scorer over the real collected text. diff --git a/packages/core/src/eval/types.ts b/packages/core/src/eval/types.ts index 3648416981..dd597fb50e 100644 --- a/packages/core/src/eval/types.ts +++ b/packages/core/src/eval/types.ts @@ -31,6 +31,17 @@ export interface AgentRunOutput { readonly text: string; /** Names of tools/actions the agent invoked, in call order. */ readonly toolCalls: readonly string[]; + /** Tool names, model-produced inputs, and execution outcomes in call order. */ + readonly toolCallDetails?: readonly { + readonly name: string; + readonly input: unknown; + readonly startedAtEventIndex?: number; + readonly completedAtEventIndex?: number; + readonly completed?: boolean; + readonly completedSideEffect?: boolean; + readonly isError?: boolean; + readonly result?: string; + }[]; /** Whether the run completed without a terminal error event. */ readonly ok: boolean; /** Terminal error message, if the run errored. */ diff --git a/templates/content/actions/_database-property-input.ts b/templates/content/actions/_database-property-input.ts new file mode 100644 index 0000000000..4a9a6bf548 --- /dev/null +++ b/templates/content/actions/_database-property-input.ts @@ -0,0 +1,211 @@ +import { ActionContractError } from "@agent-native/core"; +import { z } from "zod"; + +const nullable = (schema: T) => + z.union([schema, z.null()]); + +const propertyIdSchema = z + .string() + .min(1) + .describe("Exact immutable property definition ID"); + +const stringPropertyEntry = ( + propertyType: "text" | "place" | "phone" | "url" | "email", + valueDescription: string, +) => + z + .object({ + propertyId: propertyIdSchema, + propertyType: z.literal(propertyType), + value: nullable(z.string()).describe(valueDescription), + }) + .strict(); + +const optionPropertyEntry = (propertyType: "select" | "status") => + z + .object({ + propertyId: propertyIdSchema, + propertyType: z.literal(propertyType), + value: nullable(z.string()).describe( + "Exact option ID or exact option label from the discovered property contract; null explicitly clears the value", + ), + }) + .strict(); + +export const databasePropertyEntrySchema = z.discriminatedUnion( + "propertyType", + [ + stringPropertyEntry("text", "Text value; null explicitly clears the value"), + stringPropertyEntry( + "place", + "Place text; null explicitly clears the value", + ), + stringPropertyEntry( + "phone", + "Phone text; null explicitly clears the value", + ), + stringPropertyEntry( + "url", + "Absolute http/https URL; null explicitly clears the value", + ), + stringPropertyEntry( + "email", + "Email address; null explicitly clears the value", + ), + z + .object({ + propertyId: propertyIdSchema, + propertyType: z.literal("number"), + value: nullable(z.number().finite()).describe( + "Finite number; use a JSON number rather than numeric text, or null to explicitly clear", + ), + }) + .strict(), + z + .object({ + propertyId: propertyIdSchema, + propertyType: z.literal("checkbox"), + value: nullable(z.boolean()).describe( + "Boolean; use true or false rather than text, or null to explicitly clear", + ), + }) + .strict(), + optionPropertyEntry("select"), + optionPropertyEntry("status"), + z + .object({ + propertyId: propertyIdSchema, + propertyType: z.literal("multi_select"), + value: nullable(z.array(z.string())).describe( + "Option IDs or exact option labels from the discovered property contract; null explicitly clears the value", + ), + }) + .strict(), + z + .object({ + propertyId: propertyIdSchema, + propertyType: z.literal("date"), + value: nullable( + z.union([ + z.string(), + z + .object({ + start: z.string(), + end: z.string().optional(), + includeTime: z.boolean().optional(), + }) + .strict(), + ]), + ).describe( + "ISO date/date-time string or { start, end?, includeTime? }; null explicitly clears the value", + ), + }) + .strict(), + z + .object({ + propertyId: propertyIdSchema, + propertyType: z.literal("person"), + value: nullable(z.array(z.string())).describe( + "Person identifiers from the discovered property contract; null explicitly clears the value", + ), + }) + .strict(), + z + .object({ + propertyId: propertyIdSchema, + propertyType: z.literal("files_media"), + value: nullable(z.array(z.string())).describe( + "Absolute http/https file URLs; null explicitly clears the value", + ), + }) + .strict(), + ], +); + +export type DatabasePropertyEntry = z.infer; + +export const databasePropertyValuesSchema = z + .record(z.string(), z.unknown()) + .optional() + .describe( + "Programmatic property values keyed by exact property definition ID.", + ); + +export const databasePropertyEntriesSchema = z + .array(databasePropertyEntrySchema) + .min(1) + .max(1_000) + .optional() + .describe( + "Typed property values as explicit entries. Copy each propertyType from the discovered mutation contract and include one entry for every writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", + ); + +export function normalizeDatabasePropertyInput(input: { + propertyEntries?: DatabasePropertyEntry[]; + propertyValues?: Record; +}): { + propertyValues: Record | undefined; + propertyTypeAssertions: Record | undefined; +} { + if (input.propertyEntries && input.propertyValues) { + throw new ActionContractError( + "Provide propertyEntries or propertyValues, not both.", + { errorCode: "AMBIGUOUS_PROPERTY_INPUT" }, + ); + } + if (!input.propertyEntries) { + return { + propertyValues: input.propertyValues, + propertyTypeAssertions: undefined, + }; + } + + const values: Record = Object.create(null) as Record< + string, + unknown + >; + const propertyTypes: Record = Object.create(null) as Record< + string, + string + >; + for (const entry of input.propertyEntries) { + if (Object.prototype.hasOwnProperty.call(values, entry.propertyId)) { + throw new ActionContractError( + `Property entry ${entry.propertyId} was provided more than once.`, + { + errorCode: "DUPLICATE_PROPERTY_INPUT", + details: { propertyId: entry.propertyId }, + }, + ); + } + values[entry.propertyId] = entry.value; + propertyTypes[entry.propertyId] = entry.propertyType; + } + return { + propertyValues: values, + propertyTypeAssertions: propertyTypes, + }; +} + +export function canonicalizeDatabasePropertyInput< + T extends { + propertyEntries?: DatabasePropertyEntry[]; + propertyValues?: Record; + }, +>( + input: T, +): Omit & { + propertyValues?: Record; + propertyTypeAssertions?: Record; +} { + const { propertyEntries, propertyValues, ...canonicalInput } = input; + const normalized = normalizeDatabasePropertyInput({ + propertyEntries, + propertyValues, + }); + return { + ...canonicalInput, + propertyValues: normalized.propertyValues, + propertyTypeAssertions: normalized.propertyTypeAssertions, + }; +} diff --git a/templates/content/actions/_database-row-mutation.ts b/templates/content/actions/_database-row-mutation.ts index 5bab2f32e3..e9977486c3 100644 --- a/templates/content/actions/_database-row-mutation.ts +++ b/templates/content/actions/_database-row-mutation.ts @@ -34,11 +34,13 @@ import { } from "./_position-utils.js"; import { nanoid } from "./_property-utils.js"; +const databaseMutationAuthorityScopeSchema = z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("personal"), id: z.string().min(1) }), + z.object({ kind: z.literal("organization"), id: z.string().min(1) }), +]); + export const databaseMutationTargetSchema = z.object({ - authorityScope: z.discriminatedUnion("kind", [ - z.object({ kind: z.literal("personal"), id: z.string().min(1) }), - z.object({ kind: z.literal("organization"), id: z.string().min(1) }), - ]), + authorityScope: databaseMutationAuthorityScopeSchema, spaceId: z.string().min(1).describe("Exact Content space ID"), databaseId: z.string().min(1).describe("Exact Content database ID"), databaseDocumentId: z @@ -47,8 +49,35 @@ export const databaseMutationTargetSchema = z.object({ .describe("Exact page ID backing the Content database"), }); +export const databaseMutationTargetInputSchema = z.object({ + authorityScope: databaseMutationAuthorityScopeSchema + .optional() + .describe( + "Optional legacy assertion only. Agents must omit it; the authenticated server derives authority from the selected database.", + ), + spaceId: z + .string() + .min(1) + .describe("Exact Content space ID returned by database discovery"), + databaseId: z + .string() + .min(1) + .describe( + "Exact Content database ID returned by database discovery; never derive it from a title or number in the request", + ), + databaseDocumentId: z + .string() + .min(1) + .describe( + "Exact page ID backing the database, returned by database discovery", + ), +}); + +export const databaseMutationAgentTargetSchema = + databaseMutationTargetInputSchema.omit({ authorityScope: true }); + export const databaseMutationEnvelopeSchema = z.object({ - target: databaseMutationTargetSchema, + target: databaseMutationTargetInputSchema, expectedSchemaRevision: z .string() .min(1) @@ -59,6 +88,9 @@ export const databaseMutationEnvelopeSchema = z.object({ export type DatabaseMutationTarget = z.infer< typeof databaseMutationTargetSchema >; +export type DatabaseMutationTargetInput = z.infer< + typeof databaseMutationTargetInputSchema +>; type DatabaseRow = typeof schema.contentDatabases.$inferSelect; type DefinitionRow = typeof schema.documentPropertyDefinitions.$inferSelect; @@ -82,11 +114,12 @@ export interface RowSnapshot { export type DatabaseRowMutationOperation = "create" | "update" | "upsert"; export interface CreateDatabaseRowMutationInput { - target: DatabaseMutationTarget; + target: DatabaseMutationTargetInput; expectedSchemaRevision: string; idempotencyKey: string; title?: string; propertyValues?: Record; + propertyTypeAssertions?: Record; } export interface UpdateDatabaseRowMutationInput extends CreateDatabaseRowMutationInput { @@ -211,7 +244,7 @@ function acceptedShape(type: DocumentPropertyType): string { } export async function loadContext( - target: DatabaseMutationTarget, + target: DatabaseMutationTargetInput, role: "viewer" | "editor", db: Db = getDb(), accessAlreadyResolved = false, @@ -260,8 +293,9 @@ export async function loadContext( ? { kind: "organization" as const, id: database.orgId } : { kind: "personal" as const, id: database.ownerEmail }; if ( - target.authorityScope.kind !== authorityScope.kind || - target.authorityScope.id !== authorityScope.id || + (target.authorityScope !== undefined && + (target.authorityScope.kind !== authorityScope.kind || + target.authorityScope.id !== authorityScope.id)) || database.spaceId !== target.spaceId || database.documentId !== target.databaseDocumentId || databaseDocument.spaceId !== target.spaceId || @@ -322,7 +356,12 @@ export async function getDatabaseMutationContract( ); return { target: { - ...target, + authorityScope: context.database.orgId + ? { kind: "organization", id: context.database.orgId } + : { kind: "personal", id: context.database.ownerEmail }, + spaceId: context.database.spaceId!, + databaseId: context.database.id, + databaseDocumentId: context.database.documentId, }, schemaRevision: context.schemaRevision, naturalKeyPropertyId: context.database.naturalKeyPropertyId, @@ -695,14 +734,72 @@ export function revisionPropertyIds(context: MutationContext) { ); } -function payloadDigest( +export function databaseMutationPayloadDigest( operation: DatabaseRowMutationOperation, input: | CreateDatabaseRowMutationInput | UpdateDatabaseRowMutationInput | UpsertDatabaseRowMutationInput, ) { - return digest({ operation, ...input }); + const { + propertyTypeAssertions: _propertyTypeAssertions, + target, + ...canonicalInput + } = input; + const { authorityScope: _authorityScope, ...stableTarget } = target ?? {}; + return digest({ operation, ...canonicalInput, target: stableTarget }); +} + +export function legacyDatabaseMutationPayloadDigest( + operation: DatabaseRowMutationOperation, + input: + | CreateDatabaseRowMutationInput + | UpdateDatabaseRowMutationInput + | UpsertDatabaseRowMutationInput, + authorityScope = input.target.authorityScope, +) { + const { propertyTypeAssertions: _propertyTypeAssertions, ...legacyInput } = + input; + return digest({ + operation, + ...legacyInput, + target: { ...legacyInput.target, authorityScope }, + }); +} + +function authorityScopeForContext(context: MutationContext) { + return context.database.orgId + ? ({ kind: "organization", id: context.database.orgId } as const) + : ({ kind: "personal", id: context.database.ownerEmail } as const); +} + +function assertPropertyTypeAssertions( + context: MutationContext, + assertions: Record | undefined, +) { + if (!assertions) return; + const definitionsById = new Map( + context.definitions.map((definition) => [definition.id, definition]), + ); + for (const [propertyId, assertedType] of Object.entries(assertions)) { + const definition = definitionsById.get(propertyId); + if (!definition) { + throw new ActionContractError( + `Unknown property definition "${propertyId}".`, + { + errorCode: "UNKNOWN_PROPERTY", + details: { propertyId }, + statusCode: 400, + }, + ); + } + if (definition.type !== assertedType) { + invalidProperty( + definition, + `typed entry declared propertyType "${assertedType}" but the discovered property type is "${definition.type}"`, + ); + } + } } function resultForReceipt( @@ -721,9 +818,7 @@ function resultForReceipt( }, ): ContentDatabaseRowMutationResult { const target = { - authorityScope: context.database.orgId - ? ({ kind: "organization", id: context.database.orgId } as const) - : ({ kind: "personal", id: context.database.ownerEmail } as const), + authorityScope: authorityScopeForContext(context), spaceId: context.database.spaceId!, databaseId: context.database.id, databaseDocumentId: context.database.documentId, @@ -770,7 +865,7 @@ function resultForReceipt( async function replayReceipt( context: MutationContext, idempotencyKey: string, - expectedPayloadDigest: string, + expectedPayloadDigests: readonly string[], db: Db = getDb(), ): Promise { const [stored] = await db @@ -789,7 +884,7 @@ async function replayReceipt( ), ); if (!stored) return null; - if (stored.payloadDigest !== expectedPayloadDigest) { + if (!expectedPayloadDigests.includes(stored.payloadDigest)) { conflict( "IDEMPOTENCY_KEY_REUSED", "This idempotency key was already used for a different row mutation.", @@ -1117,13 +1212,22 @@ export async function createDatabaseRow( input: CreateDatabaseRowMutationInput, ): Promise { const initial = await loadContext(input.target, "editor"); - const inputDigest = payloadDigest("create", input); + const inputDigest = databaseMutationPayloadDigest("create", input); + const replayDigests = [ + inputDigest, + legacyDatabaseMutationPayloadDigest( + "create", + input, + authorityScopeForContext(initial), + ), + ]; const replay = await replayReceipt( initial, input.idempotencyKey, - inputDigest, + replayDigests, ); if (replay) return replay; + assertPropertyTypeAssertions(initial, input.propertyTypeAssertions); assertSchema(initial, input.expectedSchemaRevision); const values = await normalizePatch(initial, input.propertyValues); const result = await withMutationLocks(initial.database, () => @@ -1140,10 +1244,11 @@ export async function createDatabaseRow( const lockedReplay = await replayReceipt( locked, input.idempotencyKey, - inputDigest, + replayDigests, tx as unknown as Db, ); if (lockedReplay) return lockedReplay; + assertPropertyTypeAssertions(locked, input.propertyTypeAssertions); assertSchema(locked, input.expectedSchemaRevision); await touchContentDatabase( tx as unknown as Db, @@ -1186,13 +1291,22 @@ export async function updateDatabaseRow( ): Promise { const initial = await loadContext(input.target, "editor"); await assertAccess("document", input.documentId, "editor"); - const inputDigest = payloadDigest("update", input); + const inputDigest = databaseMutationPayloadDigest("update", input); + const replayDigests = [ + inputDigest, + legacyDatabaseMutationPayloadDigest( + "update", + input, + authorityScopeForContext(initial), + ), + ]; const replay = await replayReceipt( initial, input.idempotencyKey, - inputDigest, + replayDigests, ); if (replay) return replay; + assertPropertyTypeAssertions(initial, input.propertyTypeAssertions); assertSchema(initial, input.expectedSchemaRevision); const values = await normalizePatch(initial, input.propertyValues); const result = await withMutationLocks(initial.database, () => @@ -1209,10 +1323,11 @@ export async function updateDatabaseRow( const lockedReplay = await replayReceipt( locked, input.idempotencyKey, - inputDigest, + replayDigests, tx as unknown as Db, ); if (lockedReplay) return lockedReplay; + assertPropertyTypeAssertions(locked, input.propertyTypeAssertions); assertSchema(locked, input.expectedSchemaRevision); const updated = await updateInsideTransaction( tx as unknown as Db, @@ -1285,13 +1400,22 @@ export async function upsertDatabaseRow( "must match the upsert keyValue when provided in propertyValues", ); } - const inputDigest = payloadDigest("upsert", input); + const inputDigest = databaseMutationPayloadDigest("upsert", input); + const replayDigests = [ + inputDigest, + legacyDatabaseMutationPayloadDigest( + "upsert", + input, + authorityScopeForContext(initial), + ), + ]; const replay = await replayReceipt( initial, input.idempotencyKey, - inputDigest, + replayDigests, ); if (replay) return replay; + assertPropertyTypeAssertions(initial, input.propertyTypeAssertions); assertSchema(initial, input.expectedSchemaRevision); const keyPropertyId = initial.database.naturalKeyPropertyId; if (!keyPropertyId) { @@ -1348,10 +1472,11 @@ export async function upsertDatabaseRow( const lockedReplay = await replayReceipt( locked, input.idempotencyKey, - inputDigest, + replayDigests, tx as unknown as Db, ); if (lockedReplay) return lockedReplay; + assertPropertyTypeAssertions(locked, input.propertyTypeAssertions); assertSchema(locked, input.expectedSchemaRevision); if (locked.database.naturalKeyPropertyId !== keyPropertyId) { conflict("SCHEMA_REVISION_CONFLICT", "The natural key changed."); diff --git a/templates/content/actions/add-database-item.ts b/templates/content/actions/add-database-item.ts index 318f9984a2..423d675b0b 100644 --- a/templates/content/actions/add-database-item.ts +++ b/templates/content/actions/add-database-item.ts @@ -3,8 +3,14 @@ import { buildDeepLink } from "@agent-native/core/server"; import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; +import { + canonicalizeDatabasePropertyInput, + databasePropertyEntriesSchema, + databasePropertyValuesSchema, +} from "./_database-property-input.js"; import { createDatabaseRow, + databaseMutationAgentTargetSchema, databaseMutationEnvelopeSchema, } from "./_database-row-mutation.js"; import { getContentDatabaseResponse } from "./_database-utils.js"; @@ -17,15 +23,17 @@ const schema = databaseMutationEnvelopeSchema.extend({ .max(500) .optional() .describe("New row page title"), - propertyValues: z - .record(z.string(), z.unknown()) - .optional() - .describe("Strict property values keyed by property definition ID"), + propertyValues: databasePropertyValuesSchema, + propertyEntries: databasePropertyEntriesSchema, }); +const agentSchema = schema + .extend({ target: databaseMutationAgentTargetSchema }) + .omit({ propertyValues: true }); export default defineAction({ description: "Create one row in an exact ordinary Content database using its discovered schema revision. Strictly validates every non-Blocks property, applies the side effect once per idempotency key, and returns a verified receipt with stable row identity.", + agentInputSchema: agentSchema, publicAgent: { expose: true, readOnly: false, @@ -52,7 +60,9 @@ export default defineAction({ }, }, run: async (args): Promise => { - const result = await createDatabaseRow(args); + const result = await createDatabaseRow( + canonicalizeDatabasePropertyInput(args), + ); const response = await getContentDatabaseResponse( result.receipt.target.databaseId, { diff --git a/templates/content/actions/roadmap-capability-projection.db.test.ts b/templates/content/actions/roadmap-capability-projection.db.test.ts index 648cf0dc71..221bbf1d19 100644 --- a/templates/content/actions/roadmap-capability-projection.db.test.ts +++ b/templates/content/actions/roadmap-capability-projection.db.test.ts @@ -143,6 +143,7 @@ describe("private roadmap capability projection", () => { databaseId, name, type, + naturalKey: name === "Capability ID", }), ); const property = configured.properties.find( @@ -158,28 +159,45 @@ describe("private roadmap capability projection", () => { const firstReceipts = new Map< string, - { itemId: string; documentId: string } + { itemId: string; documentId: string; rowRevision: string } >(); + const discovered = await asUser(OWNER, () => + getDatabase.run({ databaseId, limit: 1, offset: 0 }), + ); + if (!("database" in discovered) || !discovered.mutationContract) + throw new Error("Projection database has no mutation contract."); + const mutationEnvelope = { + target: { + spaceId: discovered.mutationContract.target.spaceId, + databaseId: discovered.mutationContract.target.databaseId, + databaseDocumentId: + discovered.mutationContract.target.databaseDocumentId, + }, + expectedSchemaRevision: discovered.mutationContract.schemaRevision, + }; + const propertyValuesFor = (capability: Capability) => ({ + [keyPropertyId]: capability.id, + [propertyIds.get("State")!]: capability.state, + [propertyIds.get("Publicness")!]: capability.publicness, + [propertyIds.get("User promise")!]: capability.userPromise, + [propertyIds.get("Source revision")!]: SOURCE_REVISION, + }); for (const capability of capabilities) { const receipt = await asUser(OWNER, () => upsert.run({ - databaseId, - keyPropertyId, + ...mutationEnvelope, + idempotencyKey: `roadmap-projection-${capability.id}`, keyValue: capability.id, + expectedRowRevision: null, title: capability.name, - body: capability.body, - propertyValues: { - [propertyIds.get("State")!]: capability.state, - [propertyIds.get("Publicness")!]: capability.publicness, - [propertyIds.get("User promise")!]: capability.userPromise, - [propertyIds.get("Source revision")!]: SOURCE_REVISION, - }, + propertyValues: propertyValuesFor(capability), }), ); - expect(receipt.status).toBe("created"); + expect(receipt.receipt.outcome).toBe("created"); firstReceipts.set(capability.id, { - itemId: receipt.itemId, - documentId: receipt.documentId, + itemId: receipt.receipt.row.itemId, + documentId: receipt.receipt.row.documentId, + rowRevision: receipt.receipt.row.rowRevision, }); } @@ -191,55 +209,53 @@ describe("private roadmap capability projection", () => { throw new Error("Changed Capability is missing its first receipt."); const changedReceipt = await asUser(OWNER, () => upsert.run({ - databaseId, - keyPropertyId, + ...mutationEnvelope, + idempotencyKey: "roadmap-projection-change", keyValue: changedCapability.id, + expectedRowRevision: changedIdentity.rowRevision, title: `${changedCapability.name} — changed`, }), ); - expect(changedReceipt).toMatchObject({ - status: "updated", - ...changedIdentity, + expect(changedReceipt.receipt).toMatchObject({ + outcome: "updated", + row: { + itemId: changedIdentity.itemId, + documentId: changedIdentity.documentId, + }, }); const restoredReceipt = await asUser(OWNER, () => upsert.run({ - databaseId, - keyPropertyId, + ...mutationEnvelope, + idempotencyKey: "roadmap-projection-restore", keyValue: changedCapability.id, + expectedRowRevision: changedReceipt.receipt.row.rowRevision, title: changedCapability.name, - body: changedCapability.body, - propertyValues: { - [propertyIds.get("State")!]: changedCapability.state, - [propertyIds.get("Publicness")!]: changedCapability.publicness, - [propertyIds.get("User promise")!]: changedCapability.userPromise, - [propertyIds.get("Source revision")!]: SOURCE_REVISION, - }, + propertyValues: propertyValuesFor(changedCapability), }), ); - expect(restoredReceipt).toMatchObject({ - status: "updated", - ...changedIdentity, + expect(restoredReceipt.receipt).toMatchObject({ + outcome: "updated", + row: { + itemId: changedIdentity.itemId, + documentId: changedIdentity.documentId, + }, }); for (const capability of capabilities) { const receipt = await asUser(OWNER, () => upsert.run({ - databaseId, - keyPropertyId, + ...mutationEnvelope, + idempotencyKey: `roadmap-projection-${capability.id}`, keyValue: capability.id, + expectedRowRevision: null, title: capability.name, - body: capability.body, - propertyValues: { - [propertyIds.get("State")!]: capability.state, - [propertyIds.get("Publicness")!]: capability.publicness, - [propertyIds.get("User promise")!]: capability.userPromise, - [propertyIds.get("Source revision")!]: SOURCE_REVISION, - }, + propertyValues: propertyValuesFor(capability), }), ); - expect(receipt).toMatchObject({ - status: "unchanged", - ...firstReceipts.get(capability.id), + expect(receipt.receipt).toMatchObject({ + outcome: "created", + idempotency: { result: "replayed" }, + row: firstReceipts.get(capability.id), }); } @@ -282,7 +298,14 @@ describe("private roadmap capability projection", () => { expect(readbackIds).toEqual( new Set(capabilities.map((capability) => capability.id)), ); - expect(readbackIdentity).toEqual(firstReceipts); + expect(readbackIdentity).toEqual( + new Map( + [...firstReceipts].map(([key, { itemId, documentId }]) => [ + key, + { itemId, documentId }, + ]), + ), + ); const uniqueRoot = await asUser(OWNER, () => searchDocuments.run({ diff --git a/templates/content/actions/update-database-item.ts b/templates/content/actions/update-database-item.ts index 889c33c15d..3ceb2e3e65 100644 --- a/templates/content/actions/update-database-item.ts +++ b/templates/content/actions/update-database-item.ts @@ -4,6 +4,12 @@ import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; import { + canonicalizeDatabasePropertyInput, + databasePropertyEntriesSchema, + databasePropertyValuesSchema, +} from "./_database-property-input.js"; +import { + databaseMutationAgentTargetSchema, databaseMutationEnvelopeSchema, updateDatabaseRow, } from "./_database-row-mutation.js"; @@ -16,17 +22,19 @@ const schema = databaseMutationEnvelopeSchema.extend({ .min(1) .describe("Row revision returned by get-content-database"), title: z.string().trim().min(1).max(500).optional(), - propertyValues: z - .record(z.string(), z.unknown()) - .optional() - .describe( - "Sparse strict patch keyed by property definition ID; omitted fields are preserved and explicit null clears a value", - ), + propertyValues: databasePropertyValuesSchema, + propertyEntries: databasePropertyEntriesSchema.describe( + "Sparse typed property patch as explicit entries; omitted fields are preserved and explicit null clears a value. Copy each propertyType from the discovered mutation contract and include one entry for every writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", + ), }); +const agentSchema = schema + .extend({ target: databaseMutationAgentTargetSchema }) + .omit({ propertyValues: true }); export default defineAction({ description: "Sparsely update one exact Content database row by stable item and document IDs. Requires schema and row revisions, validates every non-Blocks property, and returns a verified idempotent receipt.", + agentInputSchema: agentSchema, schema, http: { method: "PUT" }, audit: { @@ -44,7 +52,7 @@ export default defineAction({ : "Updated Content database row"; }, }, - run: updateDatabaseRow, + run: (args) => updateDatabaseRow(canonicalizeDatabasePropertyInput(args)), link: ({ result }) => { const documentId = (result as ContentDatabaseRowMutationResult | null) ?.receipt.row.documentId; diff --git a/templates/content/actions/upsert-database-item-by-key.db.test.ts b/templates/content/actions/upsert-database-item-by-key.db.test.ts index a817f07ffa..b0cc74cf97 100644 --- a/templates/content/actions/upsert-database-item-by-key.db.test.ts +++ b/templates/content/actions/upsert-database-item-by-key.db.test.ts @@ -95,7 +95,6 @@ function envelope( ) { return { target: { - authorityScope: discovered.target.authorityScope, spaceId: discovered.target.spaceId, databaseId: discovered.target.databaseId, databaseDocumentId: discovered.target.databaseDocumentId, @@ -215,6 +214,52 @@ describe("reliable Content database row mutations", () => { ); }); + it("derives authority and validates typed agent property entries against the discovered contract", async () => { + const ids = await fixture(); + const evidenceId = await addProperty({ + ...ids, + name: "Evidence", + type: "text", + }); + const discovered = await contract(ids.databaseId); + const input = { + ...envelope(discovered, "typed-agent-create"), + title: "Typed agent row", + propertyEntries: [ + { + propertyId: evidenceId, + propertyType: "text" as const, + value: "preserve me", + }, + ], + }; + + const created = await asOwner(() => createRow.run(input)); + expect(created.receipt.target.authorityScope).toEqual({ + kind: "personal", + id: OWNER, + }); + expect(created.receipt.readback.propertyValues[evidenceId]).toBe( + "preserve me", + ); + + await expect( + asOwner(() => + createRow.run({ + ...input, + idempotencyKey: "typed-agent-mismatched-type", + propertyEntries: [ + { + propertyId: evidenceId, + propertyType: "number", + value: 3314, + }, + ], + }), + ), + ).rejects.toMatchObject({ errorCode: "INVALID_PROPERTY_VALUE" }); + }); + it("creates every supported non-Blocks value without coercion and returns one durable verified receipt", async () => { const ids = await fixture(); const propertyIds = { diff --git a/templates/content/actions/upsert-database-item-by-key.ts b/templates/content/actions/upsert-database-item-by-key.ts index c9a86fb303..0dd5d4a7e4 100644 --- a/templates/content/actions/upsert-database-item-by-key.ts +++ b/templates/content/actions/upsert-database-item-by-key.ts @@ -4,6 +4,12 @@ import { z } from "zod"; import type { ContentDatabaseRowMutationResult } from "../shared/api.js"; import { + canonicalizeDatabasePropertyInput, + databasePropertyEntriesSchema, + databasePropertyValuesSchema, +} from "./_database-property-input.js"; +import { + databaseMutationAgentTargetSchema, databaseMutationEnvelopeSchema, upsertDatabaseRow, } from "./_database-row-mutation.js"; @@ -18,15 +24,19 @@ const schema = databaseMutationEnvelopeSchema.extend({ "Use null to assert the key is absent and create; use the discovered row revision to update an existing key", ), title: z.string().trim().min(1).max(500).optional(), - propertyValues: z - .record(z.string(), z.unknown()) - .optional() - .describe("Sparse strict values keyed by property definition ID"), + propertyValues: databasePropertyValuesSchema, + propertyEntries: databasePropertyEntriesSchema.describe( + "Sparse typed property values as explicit entries. Copy each propertyType from the discovered mutation contract and include one entry for every writable property value the user requested, using the exact immutable property definition ID. When at least one value was requested, never pass an empty array. Do not invent or clear unmentioned properties.", + ), }); +const agentSchema = schema + .extend({ target: databaseMutationAgentTargetSchema }) + .omit({ propertyValues: true }); export default defineAction({ description: "Create or sparsely update one Content database row by that database's explicitly configured natural key. Requires schema and row compare-and-swap revisions and returns a verified idempotent receipt.", + agentInputSchema: agentSchema, schema, audit: { recordInputs: false, @@ -43,7 +53,7 @@ export default defineAction({ : "Upserted Content database row by natural key"; }, }, - run: upsertDatabaseRow, + run: (args) => upsertDatabaseRow(canonicalizeDatabasePropertyInput(args)), link: ({ result }) => { const documentId = (result as ContentDatabaseRowMutationResult | null) ?.receipt.row.documentId; diff --git a/templates/content/package.json b/templates/content/package.json index af38011c90..b470021575 100644 --- a/templates/content/package.json +++ b/templates/content/package.json @@ -12,6 +12,7 @@ "test:parity": "vitest --run parity", "test:parity-capabilities": "vitest --run parity actions/content-database-lifecycle.db.test.ts actions/bind-content-database-source-field.db.test.ts actions/_local-file-documents.test.ts actions/builder-source-review-gates.db.test.ts", "eval:parity": "agent-native eval parity", + "eval:property-preservation": "tsx parity/run-database-create-property-preservation.ts", "format.fix": "oxfmt --write .", "typecheck": "agent-native typecheck", "migrate:production": "tsx scripts/migrate-production.ts", diff --git a/templates/content/parity/README.md b/templates/content/parity/README.md index 05ac6c5b35..4c84876e57 100644 --- a/templates/content/parity/README.md +++ b/templates/content/parity/README.md @@ -10,7 +10,7 @@ PR 2.2 adds two executable tiers: existing action tests, make no model calls, and require no private provider credentials. - Gated agent evals run through `agent-native eval parity`. They are opt-in via - `CONTENT_PARITY_EVALS=1`, capped to four initial scenarios, and should be + `CONTENT_PARITY_EVALS=1`, kept to a small explicit scenario set, and should be reserved for manual or nightly checks. ## Deterministic Checks @@ -33,14 +33,23 @@ cd templates/content CONTENT_PARITY_EVALS=1 ANTHROPIC_API_KEY=... ./node_modules/.bin/agent-native eval parity ``` +The database-create property-preservation regression has a dedicated +fixture-only runner so it can inspect model-produced arguments without loading +or executing unrelated Content actions: + +```bash +CONTENT_PARITY_EVALS=1 pnpm eval:property-preservation +``` + With `CONTENT_PARITY_EVALS` unset, parity evals return skipped rows and do not call the agent runner. The CLI still exits `0`, but both readable and JSON reports mark each row with `status: "skipped"` and a `skipReason` such as `Skipped because CONTENT_PARITY_EVALS is unset`. -With the gate set, the eval files run the four PR 2.2 scenarios: +With the gate set, the eval files include these scenarios: - `database-source-scope` +- `database-create-property-preservation` - `document-search-edit` - `local-file-source-truth` - `builder-source-review-readonly` diff --git a/templates/content/parity/__tests__/database-row-property-input.test.ts b/templates/content/parity/__tests__/database-row-property-input.test.ts new file mode 100644 index 0000000000..63e2a41e33 --- /dev/null +++ b/templates/content/parity/__tests__/database-row-property-input.test.ts @@ -0,0 +1,270 @@ +import { describe, expect, it } from "vitest"; + +import { + canonicalizeDatabasePropertyInput, + databasePropertyEntriesSchema, + normalizeDatabasePropertyInput, +} from "../../actions/_database-property-input"; +import { + databaseMutationPayloadDigest, + legacyDatabaseMutationPayloadDigest, +} from "../../actions/_database-row-mutation"; +import addDatabaseItem from "../../actions/add-database-item"; +import updateDatabaseItem from "../../actions/update-database-item"; +import upsertDatabaseItemByKey from "../../actions/upsert-database-item-by-key"; + +const rowMutationActions = [ + ["add-database-item", addDatabaseItem], + ["update-database-item", updateDatabaseItem], + ["upsert-database-item-by-key", upsertDatabaseItemByKey], +] as const; + +describe("database row property inputs", () => { + it.each(rowMutationActions)( + "%s tells the agent to preserve explicitly requested writable values", + (_name, action) => { + const properties = action.tool.parameters.properties; + expect(properties).not.toHaveProperty("propertyValues"); + const propertyEntries = properties.propertyEntries; + expect(propertyEntries.type).toBe("array"); + expect(JSON.stringify(propertyEntries.items)).toContain( + "Exact immutable property definition ID", + ); + expect(JSON.stringify(propertyEntries.items)).toContain("propertyType"); + expect(JSON.stringify(propertyEntries.items)).not.toContain('"value":{}'); + expect(propertyEntries.description).toContain( + "include one entry for every writable property value the user requested", + ); + expect(propertyEntries.description).toContain( + "never pass an empty array", + ); + expect(propertyEntries.description).toContain( + "Do not invent or clear unmentioned properties", + ); + expect(properties.target.properties).not.toHaveProperty("authorityScope"); + }, + ); + + it("normalizes model-friendly entries into the strict action contract", () => { + expect( + normalizeDatabasePropertyInput({ + propertyEntries: [ + { + propertyId: "status-id", + propertyType: "status", + value: "ready", + }, + { + propertyId: "evidence-id", + propertyType: "text", + value: "preserve me", + }, + ], + }), + ).toEqual({ + propertyValues: { + "status-id": "ready", + "evidence-id": "preserve me", + }, + propertyTypeAssertions: { + "status-id": "status", + "evidence-id": "text", + }, + }); + }); + + it("rejects duplicate property entries instead of silently overwriting", () => { + expect(() => + normalizeDatabasePropertyInput({ + propertyEntries: [ + { + propertyId: "status-id", + propertyType: "status", + value: "ready", + }, + { + propertyId: "status-id", + propertyType: "status", + value: "changed", + }, + ], + }), + ).toThrow(/provided more than once/); + }); + + it("rejects ambiguous entry and record inputs", () => { + expect(() => + normalizeDatabasePropertyInput({ + propertyEntries: [ + { + propertyId: "status-id", + propertyType: "status", + value: "ready", + }, + ], + propertyValues: { "status-id": "ready" }, + }), + ).toThrow(/not both/); + }); + + it("preserves __proto__ as an ordinary property definition ID", () => { + const propertyEntries = databasePropertyEntriesSchema.parse([ + { + propertyId: "__proto__", + propertyType: "text", + value: "preserve me", + }, + ]); + const normalized = normalizeDatabasePropertyInput({ + propertyEntries, + }); + + expect(Object.getPrototypeOf(normalized.propertyValues)).toBeNull(); + expect(Object.keys(normalized.propertyValues!)).toEqual(["__proto__"]); + expect( + Object.prototype.hasOwnProperty.call( + normalized.propertyValues, + "__proto__", + ), + ).toBe(true); + expect(normalized.propertyValues?.["__proto__"]).toBe("preserve me"); + expect(normalized.propertyTypeAssertions?.["__proto__"]).toBe("text"); + }); + + it("requires a schema-visible type and its matching JSON value shape", () => { + expect(() => + databasePropertyEntriesSchema.parse([ + { propertyId: "count-id", value: 3314 }, + ]), + ).toThrow(/propertyType/); + expect(() => + databasePropertyEntriesSchema.parse([ + { + propertyId: "count-id", + propertyType: "number", + value: "3314", + }, + ]), + ).toThrow(); + expect( + databasePropertyEntriesSchema.parse([ + { + propertyId: "count-id", + propertyType: "number", + value: 3314, + }, + ]), + ).toEqual([ + { propertyId: "count-id", propertyType: "number", value: 3314 }, + ]); + }); + + it("removes the model-only representation before canonical hashing", () => { + const canonical = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyEntries: [ + { + propertyId: "status-id", + propertyType: "status", + value: "ready", + }, + { + propertyId: "evidence-id", + propertyType: "text", + value: "preserve me", + }, + ], + }); + + expect(canonical).toEqual({ + idempotencyKey: "same-intent", + propertyValues: { + "status-id": "ready", + "evidence-id": "preserve me", + }, + propertyTypeAssertions: { + "status-id": "status", + "evidence-id": "text", + }, + }); + expect(canonical).not.toHaveProperty("propertyEntries"); + }); + + it("gives equivalent entry and record inputs the same canonical digest", () => { + const fromEntries = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyEntries: [ + { + propertyId: "status-id", + propertyType: "status", + value: "ready", + }, + { + propertyId: "evidence-id", + propertyType: "text", + value: "preserve me", + }, + ], + }); + const fromRecord = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyValues: { + "evidence-id": "preserve me", + "status-id": "ready", + }, + }); + + expect(databaseMutationPayloadDigest("create", fromEntries)).toBe( + databaseMutationPayloadDigest("create", fromRecord), + ); + }); + + it("retains the authority-bearing legacy digest for receipt replay", () => { + const input = canonicalizeDatabasePropertyInput({ + idempotencyKey: "existing-receipt", + target: { + authorityScope: { kind: "personal", id: "owner@example.com" }, + spaceId: "space-id", + databaseId: "database-id", + databaseDocumentId: "database-document-id", + }, + propertyValues: { "status-id": "ready" }, + }); + + expect(legacyDatabaseMutationPayloadDigest("create", input)).not.toBe( + databaseMutationPayloadDigest("create", input), + ); + const withoutAuthoredAuthority = { + ...input, + target: { ...input.target, authorityScope: undefined }, + }; + expect( + legacyDatabaseMutationPayloadDigest( + "create", + withoutAuthoredAuthority, + input.target.authorityScope, + ), + ).toBe(legacyDatabaseMutationPayloadDigest("create", input)); + }); + + it("includes __proto__ property values in the canonical digest", () => { + const withPrototypeNamedProperty = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyEntries: [ + { + propertyId: "__proto__", + propertyType: "text", + value: "preserve me", + }, + ], + }); + const withoutProperty = canonicalizeDatabasePropertyInput({ + idempotencyKey: "same-intent", + propertyValues: {}, + }); + + expect( + databaseMutationPayloadDigest("create", withPrototypeNamedProperty), + ).not.toBe(databaseMutationPayloadDigest("create", withoutProperty)); + }); +}); diff --git a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts index 65fb938762..9edfa373c3 100644 --- a/templates/content/parity/__tests__/eval-scenario-coverage.test.ts +++ b/templates/content/parity/__tests__/eval-scenario-coverage.test.ts @@ -7,6 +7,78 @@ import { scenarioToEval } from "../scenario-to-eval"; const OLD_GATE = process.env.CONTENT_PARITY_EVALS; +function successfulCreateCall( + scenario: (typeof parityEvalScenarios)[number], + propertyInput: Record, +) { + return { + name: "add-database-item", + input: { + ...scenario.expectedCreateEnvelope, + ...propertyInput, + }, + startedAtEventIndex: 4, + completedAtEventIndex: 5, + completed: true, + completedSideEffect: true, + isError: false, + result: '{"fixtureOnly":true}', + }; +} + +function typedExpectedEntries(scenario: (typeof parityEvalScenarios)[number]) { + return Object.entries(scenario.expectedPropertyValues ?? {}).map( + ([propertyId, value]) => ({ + propertyId, + propertyType: scenario.expectedPropertyTypes?.[propertyId], + value, + }), + ); +} + +const discoveryCalls = ["list-content-databases", "get-content-database"]; + +function successfulDiscoveryDetails( + scenario: (typeof parityEvalScenarios)[number], +) { + return [ + { + name: "list-content-databases", + input: { title: "PR #3314 feedback" }, + startedAtEventIndex: 0, + completedAtEventIndex: 1, + completed: true, + isError: false, + result: JSON.stringify({ + databases: [scenario.expectedCreateEnvelope?.target], + }), + }, + { + name: "get-content-database", + input: { + databaseId: scenario.expectedCreateEnvelope?.target.databaseId, + }, + startedAtEventIndex: 2, + completedAtEventIndex: 3, + completed: true, + isError: false, + result: JSON.stringify({ + mutationContract: { + ...scenario.expectedCreateEnvelope, + properties: Object.entries(scenario.expectedPropertyTypes ?? {}).map( + ([id, type]) => ({ + id, + type, + writable: true, + sourceManaged: false, + }), + ), + }, + }), + }, + ]; +} + afterEach(() => { if (OLD_GATE === undefined) { delete process.env.CONTENT_PARITY_EVALS; @@ -37,10 +109,11 @@ describe("Content parity eval scenarios", () => { expect(invalid).toEqual([]); }); - it("keeps PR 2.5 capped to the five bundled gated scenarios", () => { + it("keeps the bundled gated scenarios explicit", () => { expect(parityEvalScenarios.map((scenario) => scenario.id).sort()).toEqual([ "builder-source-review-readonly", "database-bulk-row-reliability", + "database-create-property-preservation", "database-source-scope", "document-search-edit", "local-file-source-truth", @@ -83,14 +156,16 @@ describe("Content parity eval scenarios", () => { ); expect(report.failed).toBe(0); - expect(report.skipped).toBe(5); + expect(report.skipped).toBe(6); expect(report.results.every((row) => row.status === "skipped")).toBe(true); }); it("runs scorer-backed evals when the gate is set", async () => { process.env.CONTENT_PARITY_EVALS = "1"; - const scenario = parityEvalScenarios[0]; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-source-scope", + )!; const evalCase = scenarioToEval(scenario); const row = await scoreEval(evalCase, { runAgent: vi.fn(async () => ({ @@ -141,4 +216,299 @@ describe("Content parity eval scenarios", () => { expect(expectedToolsScore?.reason).toContain("remove-database-items"); expect(row.status).toBe("failed"); }); + + it("fails when database creation drops explicitly requested properties", async () => { + process.env.CONTENT_PARITY_EVALS = "1"; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", + )!; + const evalCase = scenarioToEval(scenario); + const row = await scoreEval(evalCase, { + runAgent: vi.fn(async () => ({ + text: scenario.successSignals.join("\n"), + toolCalls: [...discoveryCalls, "add-database-item"], + toolCallDetails: [ + ...successfulDiscoveryDetails(scenario), + successfulCreateCall(scenario, { propertyEntries: [] }), + ], + ok: true, + runId: "content-parity:empty-property-values", + durationMs: 1, + })), + engine: {} as never, + model: "test-model", + analyzeContext: vi.fn(), + }); + + expect( + row.scores.find((score) => score.scorer === "expected_property_values"), + ).toMatchObject({ passed: false, score: 0 }); + expect(row.status).toBe("failed"); + }); + + it("accepts exact database creation properties without extras", async () => { + process.env.CONTENT_PARITY_EVALS = "1"; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", + )!; + const evalCase = scenarioToEval(scenario); + const row = await scoreEval(evalCase, { + runAgent: vi.fn(async () => ({ + text: scenario.successSignals.join("\n"), + toolCalls: [...discoveryCalls, "add-database-item"], + toolCallDetails: [ + ...successfulDiscoveryDetails(scenario), + successfulCreateCall(scenario, { + propertyEntries: typedExpectedEntries(scenario), + }), + ], + ok: true, + runId: "content-parity:exact-property-values", + durationMs: 1, + })), + engine: {} as never, + model: "test-model", + analyzeContext: vi.fn(), + }); + + expect( + row.scores.find((score) => score.scorer === "expected_property_values"), + ).toMatchObject({ passed: true, score: 1 }); + expect(row.status).toBe("passed"); + }); + + it.each([ + (scenario: (typeof parityEvalScenarios)[number]) => ({ + name: "duplicate property entries", + toolCallDetails: [ + successfulCreateCall(scenario, { + propertyEntries: [ + { + propertyId: "fixture_evidence_property", + propertyType: "text", + value: "Baseline fixture preserve-me", + }, + { + propertyId: "fixture_evidence_property", + propertyType: "text", + value: "Baseline fixture preserve-me", + }, + { + propertyId: "fixture_status_property", + propertyType: "status", + value: "status-cannot-verify", + }, + ], + }), + ], + }), + (scenario: (typeof parityEvalScenarios)[number]) => ({ + name: "ambiguous property formats", + toolCallDetails: [ + successfulCreateCall(scenario, { + propertyEntries: [ + { + propertyId: "fixture_evidence_property", + propertyType: "text", + value: "Baseline fixture preserve-me", + }, + { + propertyId: "fixture_status_property", + propertyType: "status", + value: "status-cannot-verify", + }, + ], + propertyValues: { + "parity-text-property-id": "preserve me", + "parity-status-property-id": "ready", + }, + }), + ], + }), + (scenario: (typeof parityEvalScenarios)[number]) => ({ + name: "an extra row mutation", + toolCallDetails: [ + successfulCreateCall(scenario, { + propertyEntries: [ + { + propertyId: "fixture_evidence_property", + propertyType: "text", + value: "Baseline fixture preserve-me", + }, + { + propertyId: "fixture_status_property", + propertyType: "status", + value: "status-cannot-verify", + }, + ], + }), + { + name: "update-database-item", + input: {}, + completed: true, + isError: false, + result: "{}", + }, + ], + }), + ])("rejects invalid property behavior", async (buildCase) => { + process.env.CONTENT_PARITY_EVALS = "1"; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", + )!; + const { toolCallDetails } = buildCase(scenario); + const evalCase = scenarioToEval(scenario); + const row = await scoreEval(evalCase, { + runAgent: vi.fn(async () => ({ + text: scenario.successSignals.join("\n"), + toolCalls: [ + ...discoveryCalls, + ...toolCallDetails.map((call) => call.name), + ], + toolCallDetails: [ + ...successfulDiscoveryDetails(scenario), + ...toolCallDetails, + ], + ok: true, + runId: "content-parity:invalid-property-input", + durationMs: 1, + })), + engine: {} as never, + model: "test-model", + analyzeContext: vi.fn(), + }); + + expect( + row.scores.find((score) => score.scorer === "expected_property_values"), + ).toMatchObject({ passed: false, score: 0 }); + expect(row.status).toBe("failed"); + }); + + it.each([ + { + name: "wrong target", + mutate(call: ReturnType) { + return { + ...call, + input: { + ...(call.input as Record), + target: { + ...((call.input as Record).target as Record< + string, + unknown + >), + databaseId: "wrong-database", + }, + }, + }; + }, + }, + { + name: "failed execution", + mutate(call: ReturnType) { + return { ...call, isError: true, result: "fixture rejected" }; + }, + }, + { + name: "skipped side effect", + mutate(call: ReturnType) { + return { ...call, completedSideEffect: false }; + }, + }, + { + name: "an extra top-level field", + mutate(call: ReturnType) { + return { + ...call, + input: { + ...(call.input as Record), + hallucinated: true, + }, + }; + }, + }, + { + name: "an extra target field", + mutate(call: ReturnType) { + const input = call.input as Record; + return { + ...call, + input: { + ...input, + target: { + ...(input.target as Record), + hallucinated: true, + }, + }, + }; + }, + }, + { + name: "an extra authority field", + mutate(call: ReturnType) { + const input = call.input as Record; + const target = input.target as Record; + return { + ...call, + input: { + ...input, + target: { + ...target, + authorityScope: { + ...(target.authorityScope as Record), + hallucinated: true, + }, + }, + }, + }; + }, + }, + { + name: "an extra property-entry field", + mutate(call: ReturnType) { + const input = call.input as Record; + return { + ...call, + input: { + ...input, + propertyEntries: ( + input.propertyEntries as Array> + ).map((entry) => ({ + ...entry, + hallucinated: true, + })), + }, + }; + }, + }, + ])("rejects $name", async ({ mutate }) => { + process.env.CONTENT_PARITY_EVALS = "1"; + const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", + )!; + const call = mutate( + successfulCreateCall(scenario, { + propertyEntries: typedExpectedEntries(scenario), + }), + ); + const evalCase = scenarioToEval(scenario); + const row = await scoreEval(evalCase, { + runAgent: vi.fn(async () => ({ + text: scenario.successSignals.join("\n"), + toolCalls: [...discoveryCalls, "add-database-item"], + toolCallDetails: [...successfulDiscoveryDetails(scenario), call], + ok: true, + runId: "content-parity:rejected-create", + durationMs: 1, + })), + engine: {} as never, + model: "test-model", + analyzeContext: vi.fn(), + }); + + expect( + row.scores.find((score) => score.scorer === "expected_property_values"), + ).toMatchObject({ passed: false, score: 0 }); + expect(row.status).toBe("failed"); + }); }); diff --git a/templates/content/parity/eval-scenarios.ts b/templates/content/parity/eval-scenarios.ts index b8bcfb76dd..d8a91fc765 100644 --- a/templates/content/parity/eval-scenarios.ts +++ b/templates/content/parity/eval-scenarios.ts @@ -8,9 +8,61 @@ export interface ParityEvalScenario { prompt: string; successSignals: string[]; expectedTools?: string[]; + expectedPropertyValues?: Record; + expectedPropertyTypes?: Record; + expectedCreateEnvelope?: { + target: { + spaceId: string; + databaseId: string; + databaseDocumentId: string; + }; + expectedSchemaRevision: string; + idempotencyKey: string; + title: string; + }; } export const parityEvalScenarios: ParityEvalScenario[] = [ + { + id: "database-create-property-preservation", + title: "Database create property preservation", + capabilityIds: ["database.rows"], + gateEnv: "CONTENT_PARITY_EVALS", + defaultState: "skipped", + requiresPrivateCredentials: false, + prompt: + "Find the fixture Content database titled PR #3314 feedback — do not treat 3314 as its database ID. Discover its exact database target and mutation contract through Content actions, then create exactly one row titled [FIXTURE] preserve explicit properties with idempotency key fixture-create-property-preservation-v1. Set Status to status-cannot-verify and Evidence to Baseline fixture preserve-me using the discovered immutable property IDs and property types. Omit authority identity because the authenticated server owns it. No Blocks value was requested. Call add-database-item once, then report its result truthfully.", + successSignals: [ + "Discovers the exact database and mutation contract before creating.", + "Uses add-database-item once for the discovered stable target without authoring authority identity.", + "Preserves both explicitly requested writable property values.", + "Does not invent a Blocks value or another property.", + "Reports an action failure rather than claiming a row was created if the fixture is unavailable.", + ], + expectedTools: [ + "list-content-databases", + "get-content-database", + "add-database-item", + ], + expectedPropertyValues: { + fixture_status_property: "status-cannot-verify", + fixture_evidence_property: "Baseline fixture preserve-me", + }, + expectedPropertyTypes: { + fixture_status_property: "status", + fixture_evidence_property: "text", + }, + expectedCreateEnvelope: { + target: { + spaceId: "fixture_personal_space", + databaseId: "fixture_feedback_database", + databaseDocumentId: "fixture_feedback_document", + }, + expectedSchemaRevision: "fixture_schema_revision", + idempotencyKey: "fixture-create-property-preservation-v1", + title: "[FIXTURE] preserve explicit properties", + }, + }, { id: "database-bulk-row-reliability", title: "Bulk database row reliability", diff --git a/templates/content/parity/run-database-create-property-preservation.ts b/templates/content/parity/run-database-create-property-preservation.ts new file mode 100644 index 0000000000..ac1f4bc755 --- /dev/null +++ b/templates/content/parity/run-database-create-property-preservation.ts @@ -0,0 +1,140 @@ +import { createAgentRunner, runEvals } from "@agent-native/core/eval"; + +import addDatabaseItem from "../actions/add-database-item.ts"; +import getContentDatabase from "../actions/get-content-database.ts"; +import listContentDatabases from "../actions/list-content-databases.ts"; +import { parityEvalScenarios } from "./eval-scenarios.ts"; +import { scenarioToEval } from "./scenario-to-eval.ts"; + +const scenario = parityEvalScenarios.find( + (candidate) => candidate.id === "database-create-property-preservation", +); +if (!scenario) { + throw new Error("Missing database create property preservation scenario."); +} + +const FIXTURE_SPACE_ID = "fixture_personal_space"; +const FIXTURE_DATABASE_ID = "fixture_feedback_database"; +const FIXTURE_DOCUMENT_ID = "fixture_feedback_document"; +const FIXTURE_SCHEMA_REVISION = "fixture_schema_revision"; +const FIXTURE_DATABASE_TITLE = "PR #3314 feedback"; + +const evalCase = scenarioToEval(scenario); +evalCase.scorers = evalCase.scorers.filter( + (scorer) => + scorer.name === "expected_tools" || + scorer.name === "expected_property_values", +); + +const runner = await createAgentRunner({ + actions: { + "list-content-databases": { + ...listContentDatabases, + run: async () => ({ + databases: [ + { + databaseId: FIXTURE_DATABASE_ID, + documentId: FIXTURE_DOCUMENT_ID, + spaceId: FIXTURE_SPACE_ID, + title: FIXTURE_DATABASE_TITLE, + description: "", + }, + ], + pagination: { + offset: 0, + limit: 50, + totalItems: 1, + returnedItems: 1, + hasMore: false, + nextOffset: null, + }, + }), + }, + "get-content-database": { + ...getContentDatabase, + run: async () => ({ + database: { + id: FIXTURE_DATABASE_ID, + documentId: FIXTURE_DOCUMENT_ID, + spaceId: FIXTURE_SPACE_ID, + title: FIXTURE_DATABASE_TITLE, + naturalKeyPropertyId: null, + viewConfig: { + activeViewId: "fixture_view", + views: [], + sorts: [], + filters: [], + columnWidths: {}, + }, + createdAt: new Date(0).toISOString(), + updatedAt: new Date(0).toISOString(), + }, + properties: [], + items: [], + source: null, + mutationContract: { + target: { + authorityScope: { + kind: "personal", + id: "fixture-owner@example.com", + }, + spaceId: FIXTURE_SPACE_ID, + databaseId: FIXTURE_DATABASE_ID, + databaseDocumentId: FIXTURE_DOCUMENT_ID, + }, + schemaRevision: FIXTURE_SCHEMA_REVISION, + naturalKeyPropertyId: null, + properties: [ + { + id: "fixture_status_property", + name: "Status", + type: "status", + writable: true, + sourceManaged: false, + acceptedShape: null, + options: { + options: [ + { + id: "status-cannot-verify", + name: "Cannot verify", + color: "gray", + }, + ], + }, + }, + { + id: "fixture_evidence_property", + name: "Evidence", + type: "text", + writable: true, + sourceManaged: false, + acceptedShape: null, + options: {}, + }, + ], + }, + }), + }, + "add-database-item": { + ...addDatabaseItem, + run: async (input) => ({ fixtureOnly: true, received: input }), + }, + }, + systemPrompt: + "You are Content's AI document assistant. Use the registered Content actions and preserve exact user-supplied target constraints, property IDs, and property values. Never invent fields or claim an action succeeded when it failed.", +}); + +const report = await runEvals([evalCase], runner, { persist: false }); +console.log( + JSON.stringify( + { + engine: runner.engine.name, + model: runner.model, + report, + }, + null, + 2, + ), +); + +process.exitCode = report.failed === 0 ? 0 : 1; diff --git a/templates/content/parity/scenario-to-eval.ts b/templates/content/parity/scenario-to-eval.ts index 6385b1d8c2..12a4aa36c2 100644 --- a/templates/content/parity/scenario-to-eval.ts +++ b/templates/content/parity/scenario-to-eval.ts @@ -30,6 +30,387 @@ function expectedToolScorer(expectedTools: string[]) { }); } +function hasExactKeys( + record: Record, + expectedKeys: readonly string[], +): boolean { + const actualKeys = Object.keys(record).sort(); + const sortedExpectedKeys = [...expectedKeys].sort(); + return ( + actualKeys.length === sortedExpectedKeys.length && + actualKeys.every((key, index) => key === sortedExpectedKeys[index]) + ); +} + +function analyzePropertyValues(input: unknown): { + received: Record; + receivedTypes: Record; + invalid: string[]; +} { + if (!input || typeof input !== "object" || Array.isArray(input)) { + return { + received: {}, + receivedTypes: {}, + invalid: ["tool input is not an object"], + }; + } + const record = input as Record; + const hasEntries = record.propertyEntries !== undefined; + const hasValues = record.propertyValues !== undefined; + if (hasEntries && hasValues) { + return { + received: {}, + receivedTypes: {}, + invalid: ["propertyEntries and propertyValues were both provided"], + }; + } + if (hasValues) { + if ( + !record.propertyValues || + typeof record.propertyValues !== "object" || + Array.isArray(record.propertyValues) + ) { + return { + received: {}, + receivedTypes: {}, + invalid: ["propertyValues is not a record"], + }; + } + return { + received: record.propertyValues as Record, + receivedTypes: {}, + invalid: ["propertyValues bypassed the typed agent input"], + }; + } + if (!hasEntries || !Array.isArray(record.propertyEntries)) { + return { + received: {}, + receivedTypes: {}, + invalid: ["propertyEntries is not an array"], + }; + } + + const received: Record = Object.create(null) as Record< + string, + unknown + >; + const receivedTypes: Record = Object.create(null) as Record< + string, + string + >; + const invalid: string[] = []; + for (const entry of record.propertyEntries) { + if (!entry || typeof entry !== "object" || Array.isArray(entry)) { + invalid.push("propertyEntries contains a non-object entry"); + continue; + } + const { propertyId, propertyType, value } = entry as Record< + string, + unknown + >; + if ( + !hasExactKeys(entry as Record, [ + "propertyId", + "propertyType", + "value", + ]) + ) { + invalid.push( + "propertyEntries contains an entry with unrecognized fields", + ); + continue; + } + if (typeof propertyId !== "string" || propertyId.length === 0) { + invalid.push("propertyEntries contains an invalid propertyId"); + continue; + } + if (typeof propertyType !== "string" || propertyType.length === 0) { + invalid.push( + `propertyEntries contains an invalid type for ${propertyId}`, + ); + continue; + } + if (Object.prototype.hasOwnProperty.call(received, propertyId)) { + invalid.push(`propertyEntries contains duplicate ID ${propertyId}`); + continue; + } + received[propertyId] = value; + receivedTypes[propertyId] = propertyType; + } + return { received, receivedTypes, invalid }; +} + +const databaseRowMutationTools = new Set([ + "add-database-item", + "update-database-item", + "upsert-database-item-by-key", + "duplicate-database-items", + "remove-database-items", +]); + +function matchesCreateEnvelope( + input: Record, + expected: NonNullable, +): boolean { + const target = input.target; + if (!target || typeof target !== "object" || Array.isArray(target)) { + return false; + } + const actualTarget = target as Record; + return ( + hasExactKeys(input, [ + "target", + "expectedSchemaRevision", + "idempotencyKey", + "title", + input.propertyEntries === undefined + ? "propertyValues" + : "propertyEntries", + ]) && + hasExactKeys(actualTarget, [ + "spaceId", + "databaseId", + "databaseDocumentId", + ]) && + actualTarget.spaceId === expected.target.spaceId && + actualTarget.databaseId === expected.target.databaseId && + actualTarget.databaseDocumentId === expected.target.databaseDocumentId && + input.expectedSchemaRevision === expected.expectedSchemaRevision && + input.idempotencyKey === expected.idempotencyKey && + input.title === expected.title + ); +} + +function parseToolResultJson(result: string | undefined): unknown { + if (!result) return undefined; + try { + return JSON.parse(result); + } catch { + // coercion-ok: an unparseable tool result fails the same "did not supply + // the expected target/contract" scorer checks below as a missing one — + // both are non-passing evidence, not a distinction the eval needs to make. + return undefined; + } +} + +function expectedPropertyValuesScorer( + expected: Record, + expectedTypes: Record | undefined, + expectedEnvelope?: ParityEvalScenario["expectedCreateEnvelope"], +) { + return createScorer< + AgentRunOutput, + { + received: Record; + missing: string[]; + unexpected: string[]; + invalid: string[]; + mutationCalls: string[]; + } + >({ + name: "expected_property_values", + analyze(run) { + const mutationCalls = (run.toolCallDetails ?? []) + .filter((call) => databaseRowMutationTools.has(call.name)) + .map((call) => call.name); + const createCalls = (run.toolCallDetails ?? []).filter( + (call) => call.name === "add-database-item", + ); + const analysis = analyzePropertyValues(createCalls[0]?.input); + const invalid = [...analysis.invalid]; + if (createCalls.length !== 1) { + invalid.push( + `expected exactly one add-database-item call, received ${createCalls.length}`, + ); + } + if (mutationCalls.length !== 1) { + invalid.push( + `expected exactly one row mutation, received ${mutationCalls.length}`, + ); + } + const orderedCalls = run.toolCalls; + const listIndex = orderedCalls.indexOf("list-content-databases"); + const inspectIndex = orderedCalls.indexOf("get-content-database"); + const createIndex = orderedCalls.indexOf("add-database-item"); + if ( + listIndex < 0 || + inspectIndex <= listIndex || + createIndex <= inspectIndex + ) { + invalid.push("database discovery did not precede the create mutation"); + } + const listCall = (run.toolCallDetails ?? []).find( + (call) => call.name === "list-content-databases", + ); + const inspectCall = (run.toolCallDetails ?? []).find( + (call) => call.name === "get-content-database", + ); + const createCall = createCalls[0]; + const listInput = listCall?.input as Record | undefined; + const inspectInput = inspectCall?.input as + | Record + | undefined; + if ( + expectedEnvelope && + (listInput?.title !== "PR #3314 feedback" || + inspectInput?.databaseId !== expectedEnvelope.target.databaseId) + ) { + invalid.push( + "discovery did not resolve the requested title to the exact create target", + ); + } + if ( + !listCall?.completed || + listCall.isError || + !inspectCall?.completed || + inspectCall.isError + ) { + invalid.push("database discovery calls did not complete successfully"); + } + if ( + listCall?.completedAtEventIndex === undefined || + inspectCall?.startedAtEventIndex === undefined || + listCall.completedAtEventIndex >= inspectCall.startedAtEventIndex || + inspectCall.completedAtEventIndex === undefined || + createCall?.startedAtEventIndex === undefined || + inspectCall.completedAtEventIndex >= createCall.startedAtEventIndex + ) { + invalid.push( + "database discovery results were not available before dependent calls started", + ); + } + if (expectedEnvelope) { + const listResult = parseToolResultJson(listCall?.result) as + | { databases?: Array> } + | undefined; + const discoveredDatabase = listResult?.databases?.find( + (database) => + database.databaseId === expectedEnvelope.target.databaseId, + ); + const discoveredDocumentId = + discoveredDatabase?.documentId ?? + discoveredDatabase?.databaseDocumentId; + if ( + !discoveredDatabase || + discoveredDocumentId !== expectedEnvelope.target.databaseDocumentId || + discoveredDatabase.spaceId !== expectedEnvelope.target.spaceId + ) { + invalid.push( + "list-content-databases result did not supply the expected create target", + ); + } + const inspectResult = parseToolResultJson(inspectCall?.result) as + | { + mutationContract?: { + target?: Record; + schemaRevision?: string; + expectedSchemaRevision?: string; + properties?: Array>; + }; + } + | undefined; + const contractTarget = inspectResult?.mutationContract?.target; + const contractSchemaRevision = + inspectResult?.mutationContract?.schemaRevision ?? + inspectResult?.mutationContract?.expectedSchemaRevision; + if ( + !contractTarget || + contractTarget.spaceId !== expectedEnvelope.target.spaceId || + contractTarget.databaseId !== expectedEnvelope.target.databaseId || + contractTarget.databaseDocumentId !== + expectedEnvelope.target.databaseDocumentId || + contractSchemaRevision !== expectedEnvelope.expectedSchemaRevision + ) { + invalid.push( + "get-content-database result did not supply the expected mutation contract", + ); + } + const discoveredProperties = new Map( + (inspectResult?.mutationContract?.properties ?? []).map( + (property) => [property.id, property], + ), + ); + for (const [propertyId, expectedType] of Object.entries( + expectedTypes ?? {}, + )) { + const property = discoveredProperties.get(propertyId); + if ( + property?.type !== expectedType || + property.writable !== true || + property.sourceManaged === true + ) { + invalid.push( + `discovery did not supply writable property ${propertyId} with type ${expectedType}`, + ); + } + } + } + for (const [propertyId, expectedType] of Object.entries( + expectedTypes ?? {}, + )) { + if (analysis.receivedTypes[propertyId] !== expectedType) { + invalid.push( + `property ${propertyId} did not declare discovered type ${expectedType}`, + ); + } + } + const createInput = createCalls[0]?.input; + if ( + expectedEnvelope && + (!createInput || + typeof createInput !== "object" || + Array.isArray(createInput) || + !matchesCreateEnvelope( + createInput as Record, + expectedEnvelope, + )) + ) { + invalid.push( + "create target, schema revision, idempotency key, or title did not match the fixture", + ); + } + if ( + !createCalls[0]?.completed || + createCalls[0]?.completedSideEffect !== true || + createCalls[0]?.isError + ) { + invalid.push("add-database-item did not complete successfully"); + } + if (!run.ok) { + invalid.push("agent run did not complete successfully"); + } + const received = analysis.received; + const missing = Object.entries(expected) + .filter(([propertyId, value]) => received[propertyId] !== value) + .map(([propertyId]) => propertyId); + const unexpected = Object.keys(received).filter( + (propertyId) => + !Object.prototype.hasOwnProperty.call(expected, propertyId), + ); + return { received, missing, unexpected, invalid, mutationCalls }; + }, + generateScore({ missing, unexpected, invalid }) { + return missing.length === 0 && + unexpected.length === 0 && + invalid.length === 0 + ? 1 + : 0; + }, + generateReason({ + analysis: { received, missing, unexpected, invalid, mutationCalls }, + }) { + if ( + missing.length === 0 && + unexpected.length === 0 && + invalid.length === 0 + ) { + return "Agent preserved every expected property ID and exact value without inventing another property."; + } + return `Received propertyValues ${JSON.stringify(received)}; mutations: ${mutationCalls.join(", ") || "none"}; missing or changed: ${missing.join(", ") || "none"}; unexpected: ${unexpected.join(", ") || "none"}; invalid: ${invalid.join("; ") || "none"}`; + }, + }); +} + export function scenarioToEval(scenario: ParityEvalScenario): Eval { const name = `content-parity:${scenario.id}`; @@ -52,6 +433,15 @@ export function scenarioToEval(scenario: ParityEvalScenario): Eval { ...(scenario.expectedTools?.length ? [expectedToolScorer(scenario.expectedTools)] : []), + ...(scenario.expectedPropertyValues + ? [ + expectedPropertyValuesScorer( + scenario.expectedPropertyValues, + scenario.expectedPropertyTypes, + scenario.expectedCreateEnvelope, + ), + ] + : []), ], }); }