diff --git a/apps/mobile/src/lib/modelOptions.test.ts b/apps/mobile/src/lib/modelOptions.test.ts index 2f1d1f5d9..a654e1bb9 100644 --- a/apps/mobile/src/lib/modelOptions.test.ts +++ b/apps/mobile/src/lib/modelOptions.test.ts @@ -432,4 +432,27 @@ describe("mobile model options", () => { expect(resolve(null, null, sticky)).toBe(sticky); expect(resolve(null, null, null)).toBe(providerDefault.selection); }); + + it("omits a disabled fallback selection from mobile options", () => { + const fallback = { + instanceId: ProviderInstanceId.make("primeAgent"), + model: "default", + }; + const config = { + providers: [ + { + instanceId: "primeAgent", + driver: "primeAgent", + displayName: "Prime Agent", + enabled: false, + installed: true, + status: "disabled", + auth: { status: "authenticated" }, + models: [], + }, + ], + } as unknown as ServerConfig; + + expect(buildModelOptions(config, fallback)).toEqual([]); + }); }); diff --git a/apps/mobile/src/lib/modelOptions.ts b/apps/mobile/src/lib/modelOptions.ts index e43f36e0e..10b30d6b8 100644 --- a/apps/mobile/src/lib/modelOptions.ts +++ b/apps/mobile/src/lib/modelOptions.ts @@ -297,7 +297,14 @@ export function buildModelOptions( const provider = config?.providers.find( (candidate) => candidate.instanceId === fallbackModelSelection.instanceId, ); - if (getProviderUnavailablePresentation(provider) === null) { + if ( + provider !== undefined && + getProviderAdmissionAvailability({ + provider, + instanceId: String(fallbackModelSelection.instanceId), + providerSnapshotKnown: true, + }).status === "available" + ) { const providerLabel = provider ? providerDisplayLabel(provider) : fallbackModelSelection.instanceId; diff --git a/apps/server/src/atomicWrite.ts b/apps/server/src/atomicWrite.ts index fcd0345fc..7e06dbb4c 100644 --- a/apps/server/src/atomicWrite.ts +++ b/apps/server/src/atomicWrite.ts @@ -5,6 +5,8 @@ import * as Path from "effect/Path"; export const writeFileStringAtomically = (input: { readonly filePath: string; readonly contents: string; + /** Optional process-local fence checked immediately before the atomic rename. */ + readonly commitGuard?: Effect.Effect; }) => Effect.scoped( Effect.gen(function* () { @@ -20,6 +22,7 @@ export const writeFileStringAtomically = (input: { const tempPath = path.join(tempDirectory, "contents.tmp"); yield* fs.writeFileString(tempPath, input.contents); + if (input.commitGuard !== undefined && !(yield* input.commitGuard)) return; yield* fs.rename(tempPath, input.filePath); }), ); diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index 355e22e3e..5e7f59acd 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -62,6 +62,7 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.serverRemoveKeybinding]: AuthOrchestrationOperateScope, [WS_METHODS.serverGetSettings]: AuthOrchestrationReadScope, [WS_METHODS.serverUpdateSettings]: AuthOrchestrationOperateScope, + [WS_METHODS.serverMutateProviderInstances]: AuthOrchestrationOperateScope, [WS_METHODS.serverDiscoverSourceControl]: AuthOrchestrationReadScope, [WS_METHODS.serverGetTraceDiagnostics]: AuthOrchestrationReadScope, [WS_METHODS.serverGetProcessDiagnostics]: AuthOrchestrationReadScope, diff --git a/apps/server/src/mcp/McpProviderSession.ts b/apps/server/src/mcp/McpProviderSession.ts index d5dc58204..791adb0b8 100644 --- a/apps/server/src/mcp/McpProviderSession.ts +++ b/apps/server/src/mcp/McpProviderSession.ts @@ -1,4 +1,5 @@ import type { EnvironmentId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import type { ProviderRuntimeFence } from "../provider/ProviderDriver.ts"; export interface McpProviderSessionConfig { readonly environmentId: EnvironmentId; @@ -10,19 +11,40 @@ export interface McpProviderSessionConfig { } const sessionsByThread = new Map(); +const generationsByThread = new Map(); -export function setMcpProviderSession(config: McpProviderSessionConfig): void { +export function setMcpProviderSession( + config: McpProviderSessionConfig, + runtimeFence?: ProviderRuntimeFence, +): void { sessionsByThread.set(config.threadId, config); + if (runtimeFence === undefined) generationsByThread.delete(config.threadId); + else generationsByThread.set(config.threadId, runtimeFence.generation); } export function readMcpProviderSession(threadId: ThreadId): McpProviderSessionConfig | undefined { return sessionsByThread.get(threadId); } -export function clearMcpProviderSession(threadId: ThreadId): void { - sessionsByThread.delete(threadId); +export function isMcpProviderSessionOwnedByGeneration( + threadId: ThreadId, + runtimeFence: ProviderRuntimeFence, +): boolean { + return generationsByThread.get(threadId) === runtimeFence.generation; +} + +export function clearMcpProviderSession( + threadId: ThreadId, + runtimeFence?: ProviderRuntimeFence, +): boolean { + if (runtimeFence !== undefined && generationsByThread.get(threadId) !== runtimeFence.generation) { + return false; + } + generationsByThread.delete(threadId); + return sessionsByThread.delete(threadId); } export function clearAllMcpProviderSessions(): void { sessionsByThread.clear(); + generationsByThread.clear(); } diff --git a/apps/server/src/mcp/McpSessionRegistry.test.ts b/apps/server/src/mcp/McpSessionRegistry.test.ts index 1d8aead99..a09a1fcec 100644 --- a/apps/server/src/mcp/McpSessionRegistry.test.ts +++ b/apps/server/src/mcp/McpSessionRegistry.test.ts @@ -127,3 +127,25 @@ it.effect("does not keep credentials of other threads alive", () => expect(yield* registry.resolve(token)).toBeUndefined(); }), ); + +it.effect("keeps the current exact credential when retired issue and cleanup arrive late", () => + Effect.gen(function* () { + const registry = yield* makeRegistry(() => 1_000); + const threadId = ThreadId.make("thread-generation-fence"); + const request = { + threadId, + providerInstanceId: ProviderInstanceId.make("primeAgent"), + }; + const first = yield* registry.issue(request); + const firstToken = first.config.authorizationHeader.replace(/^Bearer\s+/, ""); + const replacement = yield* registry.issueIfCurrent(request, Effect.succeed(true)); + expect(replacement).toBeDefined(); + const replacementToken = replacement!.config.authorizationHeader.replace(/^Bearer\s+/, ""); + expect(yield* registry.resolve(firstToken)).toBeUndefined(); + + const retiredIssue = yield* registry.issueIfCurrent(request, Effect.succeed(false)); + expect(retiredIssue).toBeUndefined(); + yield* registry.revokeProviderSession(first.config.providerSessionId); + expect((yield* registry.resolve(replacementToken))?.threadId).toBe(threadId); + }), +); diff --git a/apps/server/src/mcp/McpSessionRegistry.ts b/apps/server/src/mcp/McpSessionRegistry.ts index f19a4f4e8..c878c7016 100644 --- a/apps/server/src/mcp/McpSessionRegistry.ts +++ b/apps/server/src/mcp/McpSessionRegistry.ts @@ -22,6 +22,11 @@ export interface McpIssuedCredential { export interface McpSessionRegistryShape { readonly issue: (request: McpCredentialRequest) => Effect.Effect; + /** Atomically replace one thread credential only while its provider generation is current. */ + readonly issueIfCurrent: ( + request: McpCredentialRequest, + isCurrent: Effect.Effect, + ) => Effect.Effect; readonly resolve: ( rawToken: string, ) => Effect.Effect; @@ -117,26 +122,26 @@ const makeWithOptions = Effect.fn("McpSessionRegistry.make")(function* ( return next.size === records.size ? records : next; }; - const issue: McpSessionRegistryShape["issue"] = Effect.fn("McpSessionRegistry.issue")( - function* (request) { - const issuedAt = yield* currentTimeMillis; - const providerSessionId = yield* crypto.randomUUIDv4.pipe(Effect.orDie); - const rawToken = yield* crypto.randomBytes(32).pipe(Effect.map(tokenFromBytes), Effect.orDie); - const tokenHash = yield* hashToken(rawToken); - const scope: McpInvocationContext.McpInvocationScope = { - environmentId, - threadId: ThreadId.make(request.threadId), - providerSessionId, - providerInstanceId: ProviderInstanceId.make(request.providerInstanceId), - capabilities: new Set(["preview"]), - issuedAt, - }; - yield* SynchronizedRef.update(state, ({ records }) => { - const next = new Map(pruneDead(records, issuedAt)); - next.set(tokenHash, { tokenHash, scope, lastAliveAt: issuedAt }); - return { records: next }; - }); - return { + const prepareCredential = Effect.fn("McpSessionRegistry.prepareCredential")(function* ( + request: McpCredentialRequest, + ) { + const issuedAt = yield* currentTimeMillis; + const providerSessionId = yield* crypto.randomUUIDv4.pipe(Effect.orDie); + const rawToken = yield* crypto.randomBytes(32).pipe(Effect.map(tokenFromBytes), Effect.orDie); + const tokenHash = yield* hashToken(rawToken); + const scope: McpInvocationContext.McpInvocationScope = { + environmentId, + threadId: ThreadId.make(request.threadId), + providerSessionId, + providerInstanceId: ProviderInstanceId.make(request.providerInstanceId), + capabilities: new Set(["preview"]), + issuedAt, + }; + return { + issuedAt, + tokenHash, + scope, + credential: { config: { environmentId, threadId: scope.threadId, @@ -145,10 +150,50 @@ const makeWithOptions = Effect.fn("McpSessionRegistry.make")(function* ( endpoint, authorizationHeader: `Bearer ${rawToken}`, }, - }; + } satisfies McpIssuedCredential, + }; + }); + + const issue: McpSessionRegistryShape["issue"] = Effect.fn("McpSessionRegistry.issue")( + function* (request) { + const prepared = yield* prepareCredential(request); + yield* SynchronizedRef.update(state, ({ records }) => { + const next = new Map(pruneDead(records, prepared.issuedAt)); + next.set(prepared.tokenHash, { + tokenHash: prepared.tokenHash, + scope: prepared.scope, + lastAliveAt: prepared.issuedAt, + }); + return { records: next }; + }); + return prepared.credential; }, ); + const issueIfCurrent: McpSessionRegistryShape["issueIfCurrent"] = Effect.fn( + "McpSessionRegistry.issueIfCurrent", + )(function* (request, isCurrent) { + const prepared = yield* prepareCredential(request); + return yield* SynchronizedRef.modifyEffect(state, ({ records }) => + Effect.gen(function* () { + // The generation check and replacement share the registry's single mutation permit. + if (!(yield* isCurrent)) return [undefined, { records }] as const; + const current = pruneDead(records, prepared.issuedAt); + const next = new Map( + Array.from(current).filter( + ([, record]) => record.scope.threadId !== prepared.scope.threadId, + ), + ); + next.set(prepared.tokenHash, { + tokenHash: prepared.tokenHash, + scope: prepared.scope, + lastAliveAt: prepared.issuedAt, + }); + return [prepared.credential, { records: next }] as const; + }), + ); + }); + const resolve: McpSessionRegistryShape["resolve"] = Effect.fn("McpSessionRegistry.resolve")( function* (rawToken) { if (rawToken.length === 0) return undefined; @@ -188,6 +233,7 @@ const makeWithOptions = Effect.fn("McpSessionRegistry.make")(function* ( return McpSessionRegistry.of({ issue, + issueIfCurrent, resolve, touch, revokeProviderSession: Effect.fn("McpSessionRegistry.revokeProviderSession")( @@ -224,13 +270,17 @@ export const layer = Layer.effect(McpSessionRegistry, make); export const issueActiveMcpCredential = ( request: McpCredentialRequest, + isCurrent: Effect.Effect = Effect.succeed(true), ): Effect.Effect => activeMcpSessionRegistry - ? activeMcpSessionRegistry - .revokeThread(request.threadId) - .pipe(Effect.andThen(activeMcpSessionRegistry.issue(request))) + ? activeMcpSessionRegistry.issueIfCurrent(request, isCurrent) : Effect.sync((): McpIssuedCredential | undefined => undefined); +export const revokeActiveMcpProviderSession = (providerSessionId: string): Effect.Effect => + activeMcpSessionRegistry + ? activeMcpSessionRegistry.revokeProviderSession(providerSessionId) + : Effect.void; + /** * Refreshes the liveness of a thread's MCP credential. Called on every provider * turn so an active session is never mistaken for an abandoned one. diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts index 1b714b5a4..0aab5d7ca 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts @@ -28,6 +28,8 @@ import { import * as Cache from "effect/Cache"; import * as Cause from "effect/Cause"; import * as Crypto from "effect/Crypto"; +import * as Context from "effect/Context"; +import * as Data from "effect/Data"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -37,6 +39,8 @@ import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; import { ProviderRegistry } from "../../provider/Services/ProviderRegistry.ts"; import { ProviderService } from "../../provider/Services/ProviderService.ts"; +import { readProviderRuntimeEventFence } from "../../provider/providerRuntimeFenceMetadata.ts"; +import type { ProviderRuntimeFence } from "../../provider/ProviderDriver.ts"; import { PrimeAgentRecoveryLedger } from "../../provider/prime/PrimeAgentRecoveryLedger.ts"; import { rateLimitFromRuntimeEventPayload, @@ -58,6 +62,15 @@ import { forkParked } from "../../serverActivation.ts"; import { ServerSettingsService } from "../../serverSettings.ts"; import { canReplaceThreadTitle } from "../threadTitles.ts"; +class PrivateProviderRuntimeEventFence extends Context.Service< + PrivateProviderRuntimeEventFence, + ProviderRuntimeFence +>()("t3/orchestration/Layers/ProviderRuntimeIngestion/PrivateProviderRuntimeEventFence") {} + +class PrivateRetiredProviderRuntimeEvent extends Data.TaggedError( + "PrivateRetiredProviderRuntimeEvent", +)<{}> {} + /** * Thread activities are durable and replicated to every authenticated client * connected to the environment. Keep the prompt shape needed to close pending @@ -1505,7 +1518,21 @@ const make = Effect.gen(function* () { const threadBackgroundLiveness = yield* ThreadBackgroundLivenessService; const threadPlanProgress = yield* ThreadPlanProgressService; const crypto = yield* Crypto.Crypto; - const orchestrationEngine = yield* OrchestrationEngineService; + const rawOrchestrationEngine = yield* OrchestrationEngineService; + const requireRuntimeEventCurrent = Effect.gen(function* () { + const runtimeFence = Option.getOrUndefined( + yield* Effect.serviceOption(PrivateProviderRuntimeEventFence), + ); + if (runtimeFence === undefined || (yield* runtimeFence.isCurrent)) return; + return yield* new PrivateRetiredProviderRuntimeEvent(); + }); + const commitRuntimeMutation = (effect: Effect.Effect) => + requireRuntimeEventCurrent.pipe(Effect.andThen(effect)); + const orchestrationEngine = { + ...rawOrchestrationEngine, + dispatch: (command: Parameters[0]) => + requireRuntimeEventCurrent.pipe(Effect.andThen(rawOrchestrationEngine.dispatch(command))), + }; const projectionSnapshotQuery = yield* ProjectionSnapshotQuery; const providerService = yield* ProviderService; const recoveryLedger = Option.getOrUndefined( @@ -1516,8 +1543,18 @@ const make = Effect.gen(function* () { const settleRecoveryTerminalProjection = (threadId: ThreadId, updatedAt: string) => recoveryLedger === undefined ? Effect.void - : recoveryLedger.markTerminalProjected({ threadId, updatedAt }).pipe( - Effect.andThen(recoveryLedger.deleteIfSettled(threadId)), + : Effect.gen(function* () { + yield* requireRuntimeEventCurrent; + const runtimeFence = Option.getOrUndefined( + yield* Effect.serviceOption(PrivateProviderRuntimeEventFence), + ); + const commitOptions = + runtimeFence === undefined ? undefined : { commitGuard: runtimeFence.isCurrent }; + yield* recoveryLedger.markTerminalProjected({ threadId, updatedAt }, commitOptions); + yield* requireRuntimeEventCurrent; + yield* recoveryLedger.deleteIfSettled(threadId, commitOptions); + }).pipe( + Effect.catchTag("PrivateRetiredProviderRuntimeEvent", () => Effect.void), Effect.catchCause(() => Effect.logWarning("failed to settle Prime Agent terminal projection proof", { threadId, @@ -1567,7 +1604,9 @@ const make = Effect.gen(function* () { }); const rememberTaskDescription = (threadId: ThreadId, taskId: string, description: string) => - Cache.set(taskDescriptionByTaskKey, providerTaskKey(threadId, taskId), description); + commitRuntimeMutation( + Cache.set(taskDescriptionByTaskKey, providerTaskKey(threadId, taskId), description), + ); // Entries are left in place after completion so replayed or duplicate // terminal events stay titled; TTL, capacity, and the session-exit sweep @@ -1597,17 +1636,19 @@ const make = Effect.gen(function* () { const rememberAssistantMessageId = (threadId: ThreadId, turnId: TurnId, messageId: MessageId) => Cache.getOption(turnMessageIdsByTurnKey, providerTurnKey(threadId, turnId)).pipe( Effect.flatMap((existingIds) => - Cache.set( - turnMessageIdsByTurnKey, - providerTurnKey(threadId, turnId), - Option.match(existingIds, { - onNone: () => new Set([messageId]), - onSome: (ids) => { - const nextIds = new Set(ids); - nextIds.add(messageId); - return nextIds; - }, - }), + commitRuntimeMutation( + Cache.set( + turnMessageIdsByTurnKey, + providerTurnKey(threadId, turnId), + Option.match(existingIds, { + onNone: () => new Set([messageId]), + onSome: (ids) => { + const nextIds = new Set(ids); + nextIds.add(messageId); + return nextIds; + }, + }), + ), ), ), ); @@ -1621,9 +1662,13 @@ const make = Effect.gen(function* () { const nextIds = new Set(ids); nextIds.delete(messageId); if (nextIds.size === 0) { - return Cache.invalidate(turnMessageIdsByTurnKey, providerTurnKey(threadId, turnId)); + return commitRuntimeMutation( + Cache.invalidate(turnMessageIdsByTurnKey, providerTurnKey(threadId, turnId)), + ); } - return Cache.set(turnMessageIdsByTurnKey, providerTurnKey(threadId, turnId), nextIds); + return commitRuntimeMutation( + Cache.set(turnMessageIdsByTurnKey, providerTurnKey(threadId, turnId), nextIds), + ); }, }), ), @@ -1637,7 +1682,9 @@ const make = Effect.gen(function* () { ); const clearAssistantMessageIdsForTurn = (threadId: ThreadId, turnId: TurnId) => - Cache.invalidate(turnMessageIdsByTurnKey, providerTurnKey(threadId, turnId)); + commitRuntimeMutation( + Cache.invalidate(turnMessageIdsByTurnKey, providerTurnKey(threadId, turnId)), + ); const getAssistantSegmentStateForTurn = (threadId: ThreadId, turnId: TurnId) => Cache.getOption(assistantSegmentStateByTurnKey, providerTurnKey(threadId, turnId)); @@ -1646,10 +1693,15 @@ const make = Effect.gen(function* () { threadId: ThreadId, turnId: TurnId, state: AssistantSegmentState, - ) => Cache.set(assistantSegmentStateByTurnKey, providerTurnKey(threadId, turnId), state); + ) => + commitRuntimeMutation( + Cache.set(assistantSegmentStateByTurnKey, providerTurnKey(threadId, turnId), state), + ); const clearAssistantSegmentStateForTurn = (threadId: ThreadId, turnId: TurnId) => - Cache.invalidate(assistantSegmentStateByTurnKey, providerTurnKey(threadId, turnId)); + commitRuntimeMutation( + Cache.invalidate(assistantSegmentStateByTurnKey, providerTurnKey(threadId, turnId)), + ); const getActiveAssistantMessageIdForTurn = (threadId: ThreadId, turnId: TurnId) => getAssistantSegmentStateForTurn(threadId, turnId).pipe( @@ -1724,12 +1776,16 @@ const make = Effect.gen(function* () { onSome: (text) => `${text}${delta}`, }); if (nextText.length <= MAX_BUFFERED_ASSISTANT_CHARS) { - yield* Cache.set(bufferedAssistantTextByMessageId, messageId, nextText); + yield* commitRuntimeMutation( + Cache.set(bufferedAssistantTextByMessageId, messageId, nextText), + ); return ""; } // Safety valve: flush full buffered text as an assistant delta to cap memory. - yield* Cache.invalidate(bufferedAssistantTextByMessageId, messageId); + yield* commitRuntimeMutation( + Cache.invalidate(bufferedAssistantTextByMessageId, messageId), + ); return nextText; }), ), @@ -1738,38 +1794,40 @@ const make = Effect.gen(function* () { const takeBufferedAssistantText = (messageId: MessageId) => Cache.getOption(bufferedAssistantTextByMessageId, messageId).pipe( Effect.flatMap((existingText) => - Cache.invalidate(bufferedAssistantTextByMessageId, messageId).pipe( + commitRuntimeMutation(Cache.invalidate(bufferedAssistantTextByMessageId, messageId)).pipe( Effect.as(Option.getOrElse(existingText, () => "")), ), ), ); const clearBufferedAssistantText = (messageId: MessageId) => - Cache.invalidate(bufferedAssistantTextByMessageId, messageId); + commitRuntimeMutation(Cache.invalidate(bufferedAssistantTextByMessageId, messageId)); const appendBufferedProposedPlan = (planId: string, delta: string, createdAt: string) => Cache.getOption(bufferedProposedPlanById, planId).pipe( Effect.flatMap((existingEntry) => { const existing = Option.getOrUndefined(existingEntry); - return Cache.set(bufferedProposedPlanById, planId, { - text: `${existing?.text ?? ""}${delta}`, - createdAt: - existing?.createdAt && existing.createdAt.length > 0 ? existing.createdAt : createdAt, - }); + return commitRuntimeMutation( + Cache.set(bufferedProposedPlanById, planId, { + text: `${existing?.text ?? ""}${delta}`, + createdAt: + existing?.createdAt && existing.createdAt.length > 0 ? existing.createdAt : createdAt, + }), + ); }), ); const takeBufferedProposedPlan = (planId: string) => Cache.getOption(bufferedProposedPlanById, planId).pipe( Effect.flatMap((existingEntry) => - Cache.invalidate(bufferedProposedPlanById, planId).pipe( + commitRuntimeMutation(Cache.invalidate(bufferedProposedPlanById, planId)).pipe( Effect.as(Option.getOrUndefined(existingEntry)), ), ), ); const clearBufferedProposedPlan = (planId: string) => - Cache.invalidate(bufferedProposedPlanById, planId); + commitRuntimeMutation(Cache.invalidate(bufferedProposedPlanById, planId)); const clearAssistantMessageState = (messageId: MessageId) => clearBufferedAssistantText(messageId); @@ -2022,7 +2080,7 @@ const make = Effect.gen(function* () { }).pipe(Effect.asVoid); } - yield* Cache.invalidate(turnMessageIdsByTurnKey, key); + yield* commitRuntimeMutation(Cache.invalidate(turnMessageIdsByTurnKey, key)); }), { concurrency: 1 }, ).pipe(Effect.asVoid); @@ -2030,7 +2088,7 @@ const make = Effect.gen(function* () { assistantSegmentKeys, (key) => key.startsWith(prefix) - ? Cache.invalidate(assistantSegmentStateByTurnKey, key) + ? commitRuntimeMutation(Cache.invalidate(assistantSegmentStateByTurnKey, key)) : Effect.void, { concurrency: 1 }, ).pipe(Effect.asVoid); @@ -2038,14 +2096,16 @@ const make = Effect.gen(function* () { proposedPlanKeys, (key) => key.startsWith(proposedPlanPrefix) - ? Cache.invalidate(bufferedProposedPlanById, key) + ? commitRuntimeMutation(Cache.invalidate(bufferedProposedPlanById, key)) : Effect.void, { concurrency: 1 }, ).pipe(Effect.asVoid); yield* Effect.forEach( taskDescriptionKeys, (key) => - key.startsWith(prefix) ? Cache.invalidate(taskDescriptionByTaskKey, key) : Effect.void, + key.startsWith(prefix) + ? commitRuntimeMutation(Cache.invalidate(taskDescriptionByTaskKey, key)) + : Effect.void, { concurrency: 1 }, ).pipe(Effect.asVoid); }); @@ -2144,11 +2204,14 @@ const make = Effect.gen(function* () { event: Extract, ) { if (event.providerInstanceId === undefined) return; + const runtimeFence = readProviderRuntimeEventFence(event); + if (runtimeFence !== undefined && !(yield* runtimeFence.isCurrent)) return; const state = rateLimitFromRuntimeEventPayload(event.payload, event.createdAt); if (state) { yield* providerRegistry.setProviderRateLimitState({ instanceId: event.providerInstanceId, state, + runtimeFence, }); } const usage = usageWindowsFromRuntimeEventPayload(event.payload); @@ -2158,12 +2221,15 @@ const make = Effect.gen(function* () { source: usage.source, observedAt: event.createdAt, windows: usage.windows, + runtimeFence, }); } }); - const processRuntimeEvent = (event: ProviderRuntimeEvent) => - Effect.gen(function* () { + const processRuntimeEvent = (event: ProviderRuntimeEvent) => { + const runtimeFence = readProviderRuntimeEventFence(event); + const process = Effect.gen(function* () { + if (runtimeFence !== undefined && !(yield* runtimeFence.isCurrent)) return; // Non-assistant deltas cannot change any projection this path derives. if (event.type === "content.delta" && event.payload.streamKind !== "assistant_text") { return; @@ -2177,7 +2243,10 @@ const make = Effect.gen(function* () { // on purpose — the account is shared by every thread — and cheap: the // registry floors it per instance and runs it off this path. if (event.type === "turn.completed" && event.providerInstanceId !== undefined) { - yield* providerRegistry.refreshProviderCapacity(event.providerInstanceId); + yield* providerRegistry.refreshProviderCapacity( + event.providerInstanceId, + readProviderRuntimeEventFence(event), + ); } let thread = yield* resolveThreadShell(event.threadId); @@ -2926,12 +2995,14 @@ const make = Effect.gen(function* () { } } + yield* requireRuntimeEventCurrent; if (event.type === "task.started" || event.type === "task.progress") { const description = event.payload.description?.trim(); if (description) { yield* rememberTaskDescription(thread.id, event.payload.taskId, description); } } + yield* requireRuntimeEventCurrent; // Working-indicator plan progress: current step while the turn runs, // cleared on settle so a finished plan never lingers as stale UI. // Events carrying a turn id that conflicts with the active turn are @@ -2993,6 +3064,7 @@ const make = Effect.gen(function* () { } } + yield* requireRuntimeEventCurrent; const activities = runtimeEventToActivities(event, taskTitle); yield* Effect.forEach(activities, (activity) => providerCommandId(event, "thread-activity-append").pipe( @@ -3008,6 +3080,14 @@ const make = Effect.gen(function* () { ), ).pipe(Effect.asVoid); }); + const scopedProcess = + runtimeFence === undefined + ? process + : process.pipe(Effect.provideService(PrivateProviderRuntimeEventFence, runtimeFence)); + return scopedProcess.pipe( + Effect.catchTag("PrivateRetiredProviderRuntimeEvent", () => Effect.void), + ); + }; const processDomainEvent = (_event: TurnStartRequestedDomainEvent) => Effect.void; diff --git a/apps/server/src/provider/Drivers/PrimeAgentDriver.test.ts b/apps/server/src/provider/Drivers/PrimeAgentDriver.test.ts index 58223aa3c..dbebde99c 100644 --- a/apps/server/src/provider/Drivers/PrimeAgentDriver.test.ts +++ b/apps/server/src/provider/Drivers/PrimeAgentDriver.test.ts @@ -1,14 +1,20 @@ import { describe, expect, it } from "vite-plus/test"; import { BUILT_IN_DRIVERS } from "../builtInDrivers.ts"; -import { isPrimeAgentProviderPlatformSupported, PrimeAgentDriver } from "./PrimeAgentDriver.ts"; +import { + getPrimeAgentMultipleInstanceCapability, + isPrimeAgentProviderPlatformSupported, + primeAgentAcpFallbackHasNativeTeardownProof, + PrimeAgentDriver, +} from "./PrimeAgentDriver.ts"; describe("PrimeAgentDriver", () => { it("registers one global Prime Agent driver with contract defaults", () => { expect(PrimeAgentDriver.driverKind).toBe("primeAgent"); - expect(PrimeAgentDriver.metadata).toEqual({ + expect(PrimeAgentDriver.metadata).toMatchObject({ displayName: "Prime Agent", supportsMultipleInstances: false, + multipleInstancesUnavailableReason: expect.stringMatching(/N=1\/2\/4/u), }); expect(PrimeAgentDriver.defaultConfig()).toEqual({ enabled: true, @@ -22,6 +28,57 @@ describe("PrimeAgentDriver", () => { ]); }); + it("keeps multiple instances disabled until the graduation proof passes", () => { + for (const input of [ + { runtime: "daemon" as const, managedArtifactProved: true }, + { runtime: "daemon" as const, managedArtifactProved: false }, + { runtime: "acp" as const, managedArtifactProved: true }, + ]) { + const capability = getPrimeAgentMultipleInstanceCapability(input); + expect(capability.supportsMultipleInstances).toBe(false); + expect(capability.multipleInstancesUnavailableReason).toMatch(/N=1\/2\/4|managed|ACP/u); + expect(capability.multipleInstancesUnavailableReason).not.toContain("/private/"); + } + }); + + it("allows N=1 ACP only after native ownership teardown is empty", () => { + expect( + primeAgentAcpFallbackHasNativeTeardownProof({ + receipts: [], + quarantinedHomes: [], + quarantinedHomeDigests: [], + corrupt: false, + }), + ).toBe(true); + expect( + primeAgentAcpFallbackHasNativeTeardownProof({ + receipts: [], + quarantinedHomes: [], + quarantinedHomeDigests: [], + corrupt: true, + }), + ).toBe(false); + expect( + primeAgentAcpFallbackHasNativeTeardownProof({ + corrupt: false, + quarantinedHomes: [], + quarantinedHomeDigests: [], + receipts: [ + { + version: 1, + state: "pending", + attemptId: "opaque-attempt", + instanceId: "primeAgent", + creationConfigRevision: "old", + currentConfigRevision: "old", + effectiveHome: "/private/home", + ownerProcessId: "opaque-owner", + }, + ], + }), + ).toBe(false); + }); + it("supports macOS, Linux, and WSL2's Linux runtime only", () => { expect(isPrimeAgentProviderPlatformSupported("darwin")).toBe(true); expect(isPrimeAgentProviderPlatformSupported("linux")).toBe(true); diff --git a/apps/server/src/provider/Drivers/PrimeAgentDriver.ts b/apps/server/src/provider/Drivers/PrimeAgentDriver.ts index 19b3d33bb..6eaf327da 100644 --- a/apps/server/src/provider/Drivers/PrimeAgentDriver.ts +++ b/apps/server/src/provider/Drivers/PrimeAgentDriver.ts @@ -1,6 +1,7 @@ import { PrimeAgentSettings, ProviderDriverKind, + type ProviderInstanceId, type ServerProvider, type ServerProviderDistribution, } from "@t3tools/contracts"; @@ -10,6 +11,7 @@ import * as Crypto from "effect/Crypto"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; +import * as Option from "effect/Option"; import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import { HttpClient } from "effect/unstable/http"; @@ -45,14 +47,38 @@ import { makePrimeDistributionNetworkDependencies, } from "../prime/PrimeAgentDistributionVerifier.ts"; import { makePrimeAgentDaemonManager } from "../prime/PrimeAgentDaemonManager.ts"; +import { fencePrimeAgentAdapter } from "../prime/PrimeAgentGenerationFence.ts"; import { resolvePrimeManagedBuildReceiptTarget } from "../prime/PrimeAgentManagedToolStore.ts"; +import { + PrimeAgentOwnershipReceiptStore, + primeAgentOwnershipHomeLockDigest, + primeAgentOwnershipHomesOverlap, + primeAgentOwnershipReceiptIsSafeLive, + type PrimeAgentAcquiredOwnershipReceipt, + type PrimeAgentOwnershipReceiptScan, +} from "../prime/PrimeAgentOwnershipReceipt.ts"; +import { + PrimeAgentRecoveryLedger, + primeAgentRecoveryOwnershipIdentities, +} from "../prime/PrimeAgentRecoveryLedger.ts"; import { defaultProviderContinuationIdentity, type ProviderDriver, + type ProviderDriverPreflightResult, type ProviderInstance, } from "../ProviderDriver.ts"; import type { ServerProviderDraft } from "../providerSnapshot.ts"; -import { mergeProviderInstanceEnvironment } from "../ProviderInstanceEnvironment.ts"; +import { + bindPrimeAgentRuntimeContext, + materializePrimeAgentIdentities, + PRIME_AGENT_AUTHORITATIVE_CLEANUP_CAPABILITY, + PRIME_AGENT_CALLER_OWNED_SESSION_FEATURE, + type PrimeAgentMaterializedIdentity, +} from "../prime/PrimeAgentRuntimeContext.ts"; +import { + PRIME_AGENT_MULTIPLE_INSTANCES_GRADUATION_REASON, + PRIME_AGENT_SUPPORTED_INSTANCE_LIMIT, +} from "../providerInstanceSettingsValidation.ts"; import { makeManualOnlyProviderMaintenanceCapabilities, makeStaticProviderMaintenanceResolver, @@ -66,6 +92,106 @@ import { const decodePrimeAgentSettings = Schema.decodeSync(PrimeAgentSettings); const DRIVER_KIND = ProviderDriverKind.make("primeAgent"); +const PRIME_AGENT_NATIVE_QUARANTINE_MESSAGE = + "Prime Agent native execution is quarantined until server-owned cleanup proves the prior owned session settled."; +const PRIME_AGENT_NATIVE_MULTI_MANAGED_BUILD_MESSAGE = + "Multiple Prime Agent instances are native-only, but this instance is not a verified Pylon-managed Prime build. Install or select a Pylon-managed build, or reduce the enabled set to one before using ACP compatibility."; +const PRIME_AGENT_NATIVE_MULTI_ACP_DISABLED_MESSAGE = + "Multiple Prime Agent instances are native-only. ACP compatibility is disabled while more than one Prime instance is enabled."; +const PRIME_AGENT_SINGLE_INSTANCE_ACP_REASON = + "This instance is using ACP compatibility, which uses Prime's one per-user default daemon. Multiple Prime instances require the Pylon-managed native path."; +export function primeAgentAcpFallbackHasNativeTeardownProof( + scan: PrimeAgentOwnershipReceiptScan, +): boolean { + return ( + !scan.corrupt && + scan.receipts.length === 0 && + scan.quarantinedHomes.length === 0 && + scan.quarantinedHomeDigests.length === 0 + ); +} + +function ownershipLockQuarantinesHome( + scan: PrimeAgentOwnershipReceiptScan, + effectiveHome: string, + platform: NodeJS.Platform, +): boolean { + return ( + scan.quarantinedHomeDigests.includes(primeAgentOwnershipHomeLockDigest(effectiveHome)) || + scan.quarantinedHomes.some((home) => + primeAgentOwnershipHomesOverlap(home, effectiveHome, platform), + ) + ); +} + +export function getPrimeAgentMultipleInstanceCapability(input: { + readonly runtime: "daemon" | "acp"; + readonly managedArtifactProved: boolean; +}): { + readonly supportsMultipleInstances: boolean; + readonly multipleInstancesUnavailableReason?: string; +} { + return { + supportsMultipleInstances: false, + multipleInstancesUnavailableReason: + input.runtime === "acp" + ? PRIME_AGENT_SINGLE_INSTANCE_ACP_REASON + : input.managedArtifactProved + ? PRIME_AGENT_MULTIPLE_INSTANCES_GRADUATION_REASON + : PRIME_AGENT_NATIVE_MULTI_MANAGED_BUILD_MESSAGE, + }; +} + +const sameStringRecord = ( + left: Readonly>, + right: Readonly>, +): boolean => { + const leftEntries = Object.entries(left); + const rightEntries = Object.entries(right); + return ( + leftEntries.length === rightEntries.length && + leftEntries.every(([name, value]) => right[name] === value) + ); +}; + +const receiptMatchesRecoveryAuthority = ( + receipt: PrimeAgentAcquiredOwnershipReceipt, + identity: PrimeAgentMaterializedIdentity, + authority: import("../prime/PrimeAgentRecoveryLedger.ts").PrimeAgentRecoveryAuthority, +): boolean => + receipt.instanceId === identity.instanceId && + receipt.effectiveHome === identity.effectiveHome && + receipt.activeSessionId === authority.activeSessionId && + receipt.nativeSessionId === authority.nativeSessionId && + receipt.attachProof.daemon.protocolName === authority.protocolName && + receipt.attachProof.daemon.protocolVersion === authority.protocolVersion && + receipt.attachProof.daemon.schemaRevision === authority.schemaRevision && + receipt.attachProof.daemon.supervisorGeneration === authority.supervisorGeneration && + receipt.recovery !== undefined && + primeAgentRecoveryOwnershipIdentities(authority).some( + (accepted) => + receipt.recovery?.threadId === accepted.threadId && + receipt.recovery.sessionIncarnationId === accepted.sessionIncarnationId && + receipt.recovery.admissionRequestId === accepted.admissionRequestId && + receipt.recovery.recoveryHandle === accepted.recoveryHandle && + receipt.recovery.ownershipGeneration === accepted.ownershipGeneration, + ) && + authority.providerInstanceId === identity.instanceId && + (authority.state === "active" || authority.state === "adopting") && + sameStringRecord(authority.launchEnvironment, identity.launchEnv); + +const sameOwnershipReceipt = ( + left: PrimeAgentAcquiredOwnershipReceipt, + right: PrimeAgentAcquiredOwnershipReceipt, +): boolean => + left.attemptId === right.attemptId && + left.instanceId === right.instanceId && + left.creationConfigRevision === right.creationConfigRevision && + left.currentConfigRevision === right.currentConfigRevision && + left.effectiveHome === right.effectiveHome && + left.activeSessionId === right.activeSessionId && + left.nativeSessionId === right.nativeSessionId; + export const PRIME_AGENT_NATIVE_WINDOWS_UNAVAILABLE_MESSAGE = "Prime Agent is unavailable because this Pylon server is running on native Windows. Run the Pylon server and Prime Agent in WSL2, or connect this client to a Pylon server running in WSL2 or another remote environment."; @@ -113,18 +239,143 @@ const withInstanceIdentity = continuation: { groupKey: input.continuationGroupKey }, }); -export const PrimeAgentDriver: ProviderDriver = { +export const PrimeAgentDriver: ProviderDriver< + PrimeAgentSettings, + PrimeAgentDriverEnv, + PrimeAgentMaterializedIdentity +> = { driverKind: DRIVER_KIND, metadata: { displayName: "Prime Agent", supportsMultipleInstances: false, + multipleInstancesUnavailableReason: PRIME_AGENT_MULTIPLE_INSTANCES_GRADUATION_REASON, }, configSchema: PrimeAgentSettings, defaultConfig: (): PrimeAgentSettings => decodePrimeAgentSettings({}), - create: ({ instanceId, displayName, accentColor, environment, enabled, config }) => + preflight: (inputs) => + Effect.gen(function* () { + // Native quarantine is resolved before provider identities are materialized. + // A dirty receipt must prevent all provider/session construction for its home. + const platform = yield* HostProcessPlatform; + if (!isPrimeAgentProviderPlatformSupported(platform)) { + return new Map( + inputs.map((input) => [ + input.instanceId, + { + kind: "unavailable" as const, + error: new ProviderDriverError({ + driver: DRIVER_KIND, + instanceId: input.instanceId, + detail: unsupportedPlatformMessage(platform), + }), + }, + ]), + ); + } + if (inputs.filter((input) => input.enabled).length > PRIME_AGENT_SUPPORTED_INSTANCE_LIMIT) { + const detail = `Prime Agent supports at most ${PRIME_AGENT_SUPPORTED_INSTANCE_LIMIT} enabled instances on one Pylon server.`; + return new Map( + inputs.map((input) => [ + input.instanceId, + { + kind: "unavailable" as const, + error: new ProviderDriverError({ + driver: DRIVER_KIND, + instanceId: input.instanceId, + detail, + }), + }, + ]), + ); + } + const serverConfig = yield* ServerConfig; + const store = new PrimeAgentOwnershipReceiptStore(serverConfig.stateDir, { platform }); + const scan = yield* Effect.tryPromise(() => store.scan()).pipe( + Effect.orElseSucceed( + () => + ({ + receipts: [], + quarantinedHomes: [], + quarantinedHomeDigests: [], + corrupt: true, + }) as const, + ), + ); + const ledger = Option.getOrUndefined(yield* Effect.serviceOption(PrimeAgentRecoveryLedger)); + const authorities = + ledger === undefined ? [] : yield* ledger.listActive().pipe(Effect.orElseSucceed(() => [])); + const materialized = yield* materializePrimeAgentIdentities(inputs); + const results = new Map< + ProviderInstanceId, + ProviderDriverPreflightResult + >(); + + for (const [instanceId, result] of materialized) { + if (result.kind === "unavailable") { + results.set(instanceId, { kind: "unavailable", error: result.error }); + continue; + } + const input = inputs.find((candidate) => candidate.instanceId === instanceId); + const overlapping = scan.receipts.filter((receipt) => + primeAgentOwnershipHomesOverlap( + receipt.effectiveHome, + result.identity.effectiveHome, + platform, + ), + ); + const adoptableReceipts = overlapping.filter( + (receipt): receipt is PrimeAgentAcquiredOwnershipReceipt => + receipt.state === "acquired" && + authorities.some((authority) => + receiptMatchesRecoveryAuthority(receipt, result.identity, authority), + ), + ); + const dirtyBlockingReceipt = overlapping.some( + (receipt) => + !primeAgentOwnershipReceiptIsSafeLive(receipt) && + !( + receipt.state === "acquired" && + adoptableReceipts.some((adoptable) => sameOwnershipReceipt(receipt, adoptable)) + ), + ); + const quarantined = + input?.enabled === true && + (scan.corrupt || + dirtyBlockingReceipt || + ownershipLockQuarantinesHome(scan, result.identity.effectiveHome, platform)); + if (quarantined) { + results.set(instanceId, { + kind: "unavailable", + error: new ProviderDriverError({ + driver: DRIVER_KIND, + instanceId, + detail: PRIME_AGENT_NATIVE_QUARANTINE_MESSAGE, + }), + }); + continue; + } + const identity: PrimeAgentMaterializedIdentity = Object.freeze({ + ...result.identity, + nativeOwnership: Object.freeze({ + store, + adoptableReceipts: Object.freeze([...adoptableReceipts]), + }), + }); + results.set(instanceId, { + kind: "ready", + preparation: identity, + generation: identity.generation, + configRevision: identity.configRevision, + // Prime must never expose a replacement adapter until the exact old + // native/ACP scope has finished its bounded teardown. + teardownBeforeCreate: true, + }); + } + return results; + }), + create: ({ instanceId, displayName, accentColor, enabled, runtimeFence }, identity) => Effect.gen(function* () { const hostPlatform = yield* HostProcessPlatform; - const hostArchitecture = yield* HostProcessArchitecture; if (!isPrimeAgentProviderPlatformSupported(hostPlatform)) { return yield* new ProviderDriverError({ driver: DRIVER_KIND, @@ -132,14 +383,81 @@ export const PrimeAgentDriver: ProviderDriver + identity.nativeOwnership!.store.scan(), + ).pipe( + Effect.orElseSucceed( + () => + ({ + receipts: [], + quarantinedHomes: [], + quarantinedHomeDigests: [], + corrupt: true, + }) as const, + ), + ); + const blockingOwnershipReceipt = ownershipScan.receipts.some((receipt) => { + if ( + !primeAgentOwnershipHomesOverlap( + receipt.effectiveHome, + identity.effectiveHome, + hostPlatform, + ) + ) { + return false; + } + if ( + receipt.state === "acquired" && + identity.nativeOwnership!.adoptableReceipts.some((adoptable) => + sameOwnershipReceipt(receipt, adoptable), + ) + ) { + return false; + } + return true; + }); + if ( + enabled && + (ownershipScan.corrupt || + blockingOwnershipReceipt || + ownershipLockQuarantinesHome(ownershipScan, identity.effectiveHome, hostPlatform)) + ) { + return yield* new ProviderDriverError({ + driver: DRIVER_KIND, + instanceId, + detail: PRIME_AGENT_NATIVE_QUARANTINE_MESSAGE, + }); + } const eventLoggers = yield* ProviderEventLoggers; - const processEnv = mergeProviderInstanceEnvironment(environment); + const processEnv = identity.launchEnv; const continuationIdentity = defaultProviderContinuationIdentity({ driverKind: DRIVER_KIND, instanceId, @@ -150,7 +468,7 @@ export const PrimeAgentDriver: ProviderDriver @@ -270,8 +594,58 @@ export const PrimeAgentDriver: ProviderDriver - stampPrimeAgentBackendSnapshot( + if (backend.runtime === "unavailable") { + return yield* new ProviderDriverError({ + driver: DRIVER_KIND, + instanceId, + detail: `${backend.message} ${PRIME_AGENT_NATIVE_MULTI_ACP_DISABLED_MESSAGE}`, + }); + } + if (backend.runtime === "acp" && identity.nativeOwnership !== undefined) { + const ownershipAfterTeardown = yield* Effect.tryPromise(() => + identity.nativeOwnership!.store.scan(), + ).pipe( + Effect.orElseSucceed( + () => + ({ + receipts: [], + quarantinedHomes: [], + quarantinedHomeDigests: [], + corrupt: true, + }) as const, + ), + ); + if (!primeAgentAcpFallbackHasNativeTeardownProof(ownershipAfterTeardown)) { + return yield* new ProviderDriverError({ + driver: DRIVER_KIND, + instanceId, + detail: PRIME_AGENT_NATIVE_QUARANTINE_MESSAGE, + }); + } + } + const runtimeContext = bindPrimeAgentRuntimeContext( + identity, + backend.runtime === "daemon" + ? { + kind: "daemon", + proof: { + sdkFeatures: Object.freeze([...(backend.manager.bridge.sdkFeatures ?? [])]), + requiredServerCapabilities: [ + PRIME_AGENT_CALLER_OWNED_SESSION_FEATURE, + PRIME_AGENT_AUTHORITATIVE_CLEANUP_CAPABILITY, + ], + }, + } + : { + kind: "acp", + ...(backend.fallbackCategory === undefined + ? {} + : { fallbackCategory: backend.fallbackCategory }), + }, + runtimeFence, + ); + const stampBackendSnapshot = (snapshot: ServerProviderDraft) => { + const stamped = stampPrimeAgentBackendSnapshot( snapshot, backend.runtime === "daemon" ? { @@ -330,6 +704,14 @@ export const PrimeAgentDriver: ProviderDriver stampIdentity(stampBackendSnapshot(snapshot)); @@ -353,7 +735,12 @@ export const PrimeAgentDriver: ProviderDriver >({ maintenanceCapabilities, + commitGuard: runtimeFence.isCurrent, getSettings: snapshotSettings.getSettings, streamSettings: snapshotSettings.streamSettings, haveSettingsChanged: haveProviderSnapshotSettingsChanged, @@ -415,12 +810,13 @@ export const PrimeAgentDriver: ProviderDriver { } }), ); + it.effect("drops buffered records whose private generation retires before flush", () => + Effect.gen(function* () { + const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); + const basePath = NodePath.join(tempDir, "provider-native.ndjson"); + let current = true; + + try { + const logger = yield* makeEventNdjsonLogger(basePath, { + stream: "native", + batchWindowMs: 60_000, + }); + assert.notEqual(logger, undefined); + if (!logger) return; + + yield* logger.write( + { id: "retired-before-flush" }, + ThreadId.make("thread-retired"), + Effect.sync(() => current), + ); + current = false; + yield* logger.close(); + + assert.isFalse(NodeFS.existsSync(ownedLogPath(basePath, "thread-retired"))); + } finally { + NodeFS.rmSync(tempDir, { recursive: true, force: true }); + } + }), + ); }); diff --git a/apps/server/src/provider/Layers/EventNdjsonLogger.ts b/apps/server/src/provider/Layers/EventNdjsonLogger.ts index 241eddb3b..db37ec2b3 100644 --- a/apps/server/src/provider/Layers/EventNdjsonLogger.ts +++ b/apps/server/src/provider/Layers/EventNdjsonLogger.ts @@ -61,7 +61,11 @@ export type EventNdjsonStream = "native" | "canonical" | "orchestration"; export interface EventNdjsonLogger { readonly filePath: string; - readonly write: (event: unknown, threadId: ThreadId | null) => Effect.Effect; + readonly write: ( + event: unknown, + threadId: ThreadId | null, + commitGuard?: Effect.Effect, + ) => Effect.Effect; readonly close: () => Effect.Effect; } @@ -134,6 +138,7 @@ export interface PendingRecord { readonly threadSegment: string; readonly line: string; readonly bytes: number; + readonly commitGuard?: Effect.Effect | undefined; } interface StoreState { @@ -540,17 +545,34 @@ export const makeEventNdjsonLogStore = Effect.fnUntraced(function* ( const flush = Effect.fnUntraced(function* (timerFired: boolean, close: boolean) { const startedAt = yield* Clock.currentTimeMillis; const result = yield* SynchronizedRef.modifyEffect(stateRef, (state) => - Effect.sync(() => - drainPending({ + Effect.gen(function* () { + const currentRecords = yield* Effect.forEach(state.pending, (record) => + Effect.map(record.commitGuard ?? Effect.succeed(true), (current) => ({ + record, + current, + })), + ); + const pending = currentRecords + .filter((entry) => entry.current) + .map((entry) => entry.record); + const currentState = + pending.length === state.pending.length + ? state + : { + ...state, + pending, + pendingBytes: pending.reduce((total, record) => total + record.bytes, 0), + }; + return drainPending({ directory, options: resolved, - state, + state: currentState, filePrefix, now: startedAt, timerFired, close, - }), - ), + }); + }), ); for (const failure of result.failures) { @@ -603,7 +625,11 @@ export const makeEventNdjsonLogStore = Effect.fnUntraced(function* ( const existing = loggerViews.get(stream); if (existing) return existing; - const write = Effect.fnUntraced(function* (event: unknown, threadId: ThreadId | null) { + const write = Effect.fnUntraced(function* ( + event: unknown, + threadId: ThreadId | null, + commitGuard?: Effect.Effect, + ) { if (!shouldPersist(stream, event)) return; const payload = yield* serializeEvent(event); if (payload === undefined) return; @@ -611,28 +637,36 @@ export const makeEventNdjsonLogStore = Effect.fnUntraced(function* ( const observedAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)); const line = `[${observedAt}] ${resolveStreamLabel(stream)}: ${payload}\n`; const bytes = Buffer.byteLength(line); - const action = yield* SynchronizedRef.modifyEffect(stateRef, (state) => { - if (state.closed) { - return Effect.succeed([{ flush: false }, state] as const); - } - const pending = state.pending; - pending.push({ stream, threadSegment: resolveThreadSegment(threadId), line, bytes }); - const pendingBytes = state.pendingBytes + bytes; - const flush = - resolved.batchWindowMs === 0 || - pending.length >= resolved.maxBufferedRecords || - pendingBytes >= resolved.maxBufferedBytes; - const schedule = !flush && !state.flushScheduled; - const nextState = { - ...state, - pending, - pendingBytes, - flushScheduled: state.flushScheduled || schedule, - }; - return (schedule ? scheduleFlush() : Effect.void).pipe( - Effect.as([{ flush }, nextState] as const), - ); - }).pipe(Effect.uninterruptible); + const action = yield* SynchronizedRef.modifyEffect(stateRef, (state) => + Effect.gen(function* () { + if (state.closed || (commitGuard !== undefined && !(yield* commitGuard))) { + return [{ flush: false }, state] as const; + } + const pending = state.pending; + pending.push({ + stream, + threadSegment: resolveThreadSegment(threadId), + line, + bytes, + ...(commitGuard === undefined ? {} : { commitGuard }), + }); + const pendingBytes = state.pendingBytes + bytes; + const flush = + resolved.batchWindowMs === 0 || + pending.length >= resolved.maxBufferedRecords || + pendingBytes >= resolved.maxBufferedBytes; + const schedule = !flush && !state.flushScheduled; + const nextState = { + ...state, + pending, + pendingBytes, + flushScheduled: state.flushScheduled || schedule, + }; + return yield* (schedule ? scheduleFlush() : Effect.void).pipe( + Effect.as([{ flush }, nextState] as const), + ); + }), + ).pipe(Effect.uninterruptible); if (action.flush) { yield* flush(false, false); diff --git a/apps/server/src/provider/Layers/PrimeAgentAdapter.test.ts b/apps/server/src/provider/Layers/PrimeAgentAdapter.test.ts index 6f4c00d52..99e0bd1c8 100644 --- a/apps/server/src/provider/Layers/PrimeAgentAdapter.test.ts +++ b/apps/server/src/provider/Layers/PrimeAgentAdapter.test.ts @@ -70,6 +70,51 @@ it("keeps every branded session id inside its Prime Agent storage root", () => { } }); +it.effect("rejects an MCP route owned by another provider instance before ACP launch", () => + Effect.scoped( + Effect.gen(function* () { + const boundInstanceId = ProviderInstanceId.make("prime-acp-bound"); + const threadId = ThreadId.make("prime-acp-mcp-mismatch"); + yield* Effect.acquireRelease( + Effect.sync(() => + McpProviderSession.setMcpProviderSession({ + providerSessionId: "provider-session-mismatch", + threadId, + environmentId: EnvironmentId.make("environment-mismatch"), + providerInstanceId: ProviderInstanceId.make("prime-acp-other"), + endpoint: "http://127.0.0.1:4321/mcp/mismatch", + authorizationHeader: "Bearer must-not-route", + }), + ), + () => Effect.sync(() => McpProviderSession.clearMcpProviderSession(threadId)), + ); + const adapter = yield* makePrimeAgentAdapter(decodeSettings({}), { + instanceId: boundInstanceId, + }); + + const result = yield* adapter + .startSession({ + threadId, + provider: ProviderDriverKind.make("primeAgent"), + cwd: process.cwd(), + runtimeMode: "full-access", + }) + .pipe(Effect.result); + + assert.equal(result._tag, "Failure"); + if (result._tag === "Failure") { + assert.equal(result.failure._tag, "ProviderAdapterValidationError"); + if (result.failure._tag === "ProviderAdapterValidationError") { + assert.equal( + result.failure.issue, + "The MCP route does not belong to this provider instance.", + ); + } + } + }), + ).pipe(Effect.provide(testLayer)), +); + it.effect("validates Prime Agent session constraints and owns continuation storage", () => Effect.gen(function* () { const tempDir = yield* Effect.promise(() => diff --git a/apps/server/src/provider/Layers/PrimeAgentAdapter.ts b/apps/server/src/provider/Layers/PrimeAgentAdapter.ts index 35dc4becd..f55f6e5b4 100644 --- a/apps/server/src/provider/Layers/PrimeAgentAdapter.ts +++ b/apps/server/src/provider/Layers/PrimeAgentAdapter.ts @@ -52,6 +52,7 @@ import { import type { PrimeAgentAdapterShape } from "../Services/PrimeAgentAdapter.ts"; import { BUILT_IN_ADAPTER_CONVERSATION_ROLLBACK_MODES } from "../Services/ProviderAdapter.ts"; import { canonicalPrimeToolItemId } from "../prime/PrimeAgentDaemonRuntimeEvents.ts"; +import type { PrimeAgentRuntimeContext } from "../prime/PrimeAgentRuntimeContext.ts"; import { makePrimeAgentEventPubSub, shutdownPrimeAgentEventPubSub, @@ -75,6 +76,7 @@ export interface PrimeAgentAdapterLiveOptions { readonly nativeEventLogPath?: string; readonly nativeEventLogger?: EventNdjsonLogger; readonly instanceId?: ProviderInstanceId; + readonly runtimeContext?: PrimeAgentRuntimeContext; /** Generic, user-visible explanation when this adapter is an explicit compatibility fallback. */ readonly startupWarning?: string; } @@ -197,6 +199,30 @@ export function makePrimeAgentAdapter( ) { return Effect.gen(function* () { const boundInstanceId = options?.instanceId ?? ProviderInstanceId.make("primeAgent"); + const primeRuntimeContext = options?.runtimeContext; + if ( + primeRuntimeContext !== undefined && + (primeRuntimeContext.backendKind !== "acp" || + primeRuntimeContext.instanceId !== boundInstanceId) + ) { + return yield* Effect.die( + new Error("The Prime Agent runtime context does not own this ACP adapter."), + ); + } + const effectiveSettings = primeRuntimeContext?.settings ?? primeAgentSettings; + const commitGuard = primeRuntimeContext?.runtimeFence?.isCurrent ?? Effect.succeed(true); + const requireCurrentGeneration = (operation: string) => + Effect.flatMap(commitGuard, (current) => + current + ? Effect.void + : Effect.fail( + new ProviderAdapterRequestError({ + provider: PROVIDER, + method: operation, + detail: "The Prime Agent runtime was replaced while this operation was pending.", + }), + ), + ); const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; @@ -235,7 +261,9 @@ export function makePrimeAgentAdapter( ) => { const stampedEvent = sessionIncarnationId === undefined ? event : { ...event, sessionIncarnationId }; - return PubSub.publish(runtimeEventPubSub, stampedEvent).pipe( + return Effect.flatMap(commitGuard, (current) => + current ? PubSub.publish(runtimeEventPubSub, stampedEvent) : Effect.succeed(false), + ).pipe( Effect.flatMap((accepted) => accepted ? Effect.void @@ -285,6 +313,7 @@ export function makePrimeAgentAdapter( }, }, threadId, + commitGuard, ); }).pipe( Effect.catchCause((cause) => @@ -502,6 +531,7 @@ export function makePrimeAgentAdapter( const modelSelection = input.modelSelection?.instanceId === boundInstanceId ? input.modelSelection : undefined; const model = modelSelection?.model?.trim() || "default"; + yield* requireCurrentGeneration("startSession"); const sessionDir = primeAgentSessionDirectory({ stateDir: serverConfig.stateDir, instanceId: boundInstanceId, @@ -520,6 +550,7 @@ export function makePrimeAgentAdapter( }), ), ); + yield* requireCurrentGeneration("startSession"); const sessionScope = yield* Scope.make("sequential"); let sessionContext: PrimeAgentSessionContext | undefined; @@ -533,9 +564,28 @@ export function makePrimeAgentAdapter( threadId: input.threadId, }); const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); + if ( + mcpSession !== undefined && + (mcpSession.providerInstanceId !== boundInstanceId || + (primeRuntimeContext?.runtimeFence !== undefined && + !McpProviderSession.isMcpProviderSessionOwnedByGeneration( + input.threadId, + primeRuntimeContext.runtimeFence, + ))) + ) { + return yield* new ProviderAdapterValidationError({ + provider: PROVIDER, + operation: "startSession", + issue: "The MCP route does not belong to this provider instance.", + }); + } const acp = yield* makePrimeAgentAcpRuntime({ - primeAgentSettings, - ...(options?.environment ? { environment: options.environment } : {}), + primeAgentSettings: effectiveSettings, + ...(primeRuntimeContext !== undefined + ? { environment: primeRuntimeContext.launchEnv } + : options?.environment + ? { environment: options.environment } + : {}), childProcessSpawner, cwd, sessionDir, @@ -732,6 +782,7 @@ export function makePrimeAgentAdapter( Effect.forkChild, ); ctx.notificationFiber = notificationFiber; + yield* requireCurrentGeneration("startSession"); sessions.set(input.threadId, ctx); sessionScopeTransferred = true; diff --git a/apps/server/src/provider/Layers/PrimeAgentProvider.test.ts b/apps/server/src/provider/Layers/PrimeAgentProvider.test.ts index 654e3bb0c..41d750c43 100644 --- a/apps/server/src/provider/Layers/PrimeAgentProvider.test.ts +++ b/apps/server/src/provider/Layers/PrimeAgentProvider.test.ts @@ -21,7 +21,7 @@ import * as Stream from "effect/Stream"; import { HttpClient, HttpClientResponse } from "effect/unstable/http"; import { ChildProcessSpawner } from "effect/unstable/process"; -import { primeAgentCredentialFingerprint, readPrimeAgentBackends } from "../primeAgentBackends.ts"; +import { readPrimeAgentBackends } from "../primeAgentBackends.ts"; import { acquireSharedUsageLock, readSharedUsageEntry, @@ -765,7 +765,12 @@ it.layer(NodeServices.layer)("checkPrimeAgentProviderStatus", (it) => { ); const readBackends = readPrimeAgentBackends( { agentHomePath: home }, - { sharedCacheDir: cacheDir, freshForMs: 0 }, + { + sharedCacheDir: cacheDir, + freshForMs: 0, + instanceId: "primeAgent", + configRevision: "test-revision", + }, ).pipe( Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), @@ -796,15 +801,11 @@ it.layer(NodeServices.layer)("checkPrimeAgentProviderStatus", (it) => { yield* Deferred.await(codexFinalized); const anthropicEntry = yield* readSharedUsageEntry( cacheDir, - sharedUsageReadKey([ - "prime-anthropic", - home, - primeAgentCredentialFingerprint("anthropic-secret"), - ]), + sharedUsageReadKey(["prime", "primeAgent", "anthropic", "test-revision"]), ); const codexEntry = yield* readSharedUsageEntry( cacheDir, - sharedUsageReadKey(["prime-codex", home, "acct_123"]), + sharedUsageReadKey(["prime", "primeAgent", "openai-codex", "test-revision"]), ); expect(snapshot.status).toBe("ready"); @@ -851,6 +852,8 @@ it.layer(NodeServices.layer)("checkPrimeAgentProviderStatus", (it) => { { sharedCacheDir: cacheDir, freshForMs: 0, + instanceId: "primeAgent", + configRevision: "test-revision", readCodexWindows: () => Effect.succeed({ rateLimits: { @@ -879,11 +882,7 @@ it.layer(NodeServices.layer)("checkPrimeAgentProviderStatus", (it) => { const snapshot = yield* Fiber.join(provider); const anthropicEntry = yield* readSharedUsageEntry( cacheDir, - sharedUsageReadKey([ - "prime-anthropic", - home, - primeAgentCredentialFingerprint("anthropic-secret"), - ]), + sharedUsageReadKey(["prime", "primeAgent", "anthropic", "test-revision"]), ); expect(snapshot.backends).toEqual([ @@ -941,6 +940,8 @@ it.layer(NodeServices.layer)("checkPrimeAgentProviderStatus", (it) => { { sharedCacheDir: cacheDir, freshForMs: 0, + instanceId: "primeAgent", + configRevision: "test-revision", readCodexWindows: () => Deferred.succeed(codexStarted, undefined).pipe( Effect.andThen(Effect.never), @@ -971,11 +972,12 @@ it.layer(NodeServices.layer)("checkPrimeAgentProviderStatus", (it) => { expect(Exit.isFailure(exit) && Cause.hasInterrupts(exit.cause)).toBe(true); const anthropicKey = sharedUsageReadKey([ - "prime-anthropic", - home, - primeAgentCredentialFingerprint("anthropic-secret"), + "prime", + "primeAgent", + "anthropic", + "test-revision", ]); - const codexKey = sharedUsageReadKey(["prime-codex", home, "acct_123"]); + const codexKey = sharedUsageReadKey(["prime", "primeAgent", "openai-codex", "test-revision"]); expect(yield* readSharedUsageEntry(cacheDir, anthropicKey)).toBeUndefined(); expect(yield* readSharedUsageEntry(cacheDir, codexKey)).toBeUndefined(); diff --git a/apps/server/src/provider/Layers/ProviderInstanceRegistryLive.test.ts b/apps/server/src/provider/Layers/ProviderInstanceRegistryLive.test.ts index 70584723d..be0df965f 100644 --- a/apps/server/src/provider/Layers/ProviderInstanceRegistryLive.test.ts +++ b/apps/server/src/provider/Layers/ProviderInstanceRegistryLive.test.ts @@ -1,3 +1,4 @@ +// @effect-diagnostics nodeBuiltinImport:off /** * Multi-instance validation slices for `ProviderInstanceRegistryLive`. * @@ -22,6 +23,10 @@ * binaries. That keeps the assertions focused on registry routing * behaviour rather than the runtime details of each provider. */ +import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + import { describe, expect, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { @@ -40,6 +45,7 @@ import * as DateTime from "effect/DateTime"; import { createModelSelection } from "@t3tools/shared/model"; import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Stream from "effect/Stream"; import { HttpClient, HttpClientResponse } from "effect/unstable/http"; @@ -315,6 +321,122 @@ describe("ProviderInstanceRegistryLive — multi-instance codex slice", () => { }).pipe(Effect.provide(testLayer)), ); + it.live("rejects multiple Prime instances at the graduation gate before any Prime work", () => { + let processCalls = 0; + let networkCalls = 0; + const trackingSpawner = ChildProcessSpawner.make(() => + Effect.sync(() => { + processCalls += 1; + throw new Error("overlap preflight must run before process work"); + }), + ); + const trackingHttpClient = HttpClient.make((request) => + Effect.sync(() => { + networkCalls += 1; + return HttpClientResponse.fromWeb(request, Response.json({})); + }), + ); + + return Effect.scoped( + Effect.gen(function* () { + const root = yield* Effect.acquireRelease( + Effect.promise(() => NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "prime-registry-"))), + (directory) => + Effect.promise(() => NodeFSP.rm(directory, { recursive: true, force: true })), + ); + const shared = NodePath.join(root, "shared"); + yield* Effect.promise(() => NodeFSP.mkdir(shared, { recursive: true })); + const parentId = ProviderInstanceId.make("prime-parent"); + const childId = ProviderInstanceId.make("prime-child"); + const { registry } = yield* makeProviderInstanceRegistry({ + drivers: [PrimeAgentDriver], + configMap: { + [parentId]: { + driver: ProviderDriverKind.make("primeAgent"), + enabled: true, + config: makePrimeAgentConfig({ agentHomePath: shared }), + }, + [childId]: { + driver: ProviderDriverKind.make("primeAgent"), + enabled: true, + config: makePrimeAgentConfig({ agentHomePath: NodePath.join(shared, "nested") }), + }, + }, + }); + + expect(yield* registry.listInstances).toEqual([]); + const unavailable = yield* registry.listUnavailable; + expect(unavailable.map((snapshot) => snapshot.instanceId).toSorted()).toEqual( + [parentId, childId].toSorted(), + ); + expect( + unavailable.every((snapshot) => snapshot.unavailableReason?.includes("N=1/2/4")), + ).toBe(true); + expect(processCalls).toBe(0); + expect(networkCalls).toBe(0); + }), + ).pipe( + Effect.provideService(HostProcessPlatform, "linux"), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, trackingSpawner), + Effect.provideService(HttpClient.HttpClient, trackingHttpClient), + Effect.provide(testLayer), + ); + }); + + it.live("rotates Prime generation only for material changes", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "prime-generation-fence-" }); + const instanceId = ProviderInstanceId.make("primeAgent"); + const entry = { + driver: ProviderDriverKind.make("primeAgent"), + displayName: "Prime A", + enabled: false, + config: makePrimeAgentConfig({ + enabled: false, + binaryPath: "/opt/prime-a", + agentHomePath: root, + }), + } as const; + const { registry, mutator } = yield* makeProviderInstanceRegistry({ + drivers: [PrimeAgentDriver], + configMap: { [instanceId]: entry }, + }); + const first = yield* registry.getInstance(instanceId); + expect((yield* first!.snapshot.getSnapshot).supportsMultipleInstances).toBe(false); + expect((yield* first!.snapshot.getSnapshot).multipleInstancesUnavailableReason).toContain( + "ACP compatibility", + ); + expect(first?.runtimeFence).toBeDefined(); + expect(yield* first!.runtimeFence!.isCurrent).toBe(true); + + yield* mutator.reconcile({ + [instanceId]: { ...entry, displayName: "Prime presentation only" }, + }); + const presentationOnly = yield* registry.getInstance(instanceId); + expect(presentationOnly?.adapter).toBe(first?.adapter); + expect(presentationOnly?.runtimeFence).toBe(first?.runtimeFence); + expect(yield* first!.runtimeFence!.isCurrent).toBe(true); + + yield* mutator.reconcile({ + [instanceId]: { + ...entry, + config: { ...entry.config, binaryPath: "/opt/prime-b" }, + }, + }); + const replacement = yield* registry.getInstance(instanceId); + expect(replacement?.adapter).not.toBe(first?.adapter); + expect(replacement?.runtimeFence?.generation).not.toBe(first?.runtimeFence?.generation); + expect(replacement?.runtimeFence?.configRevision).not.toBe( + first?.runtimeFence?.configRevision, + ); + expect(yield* first!.runtimeFence!.isCurrent).toBe(false); + expect(yield* replacement!.runtimeFence!.isCurrent).toBe(true); + }), + ).pipe(Effect.provideService(HostProcessPlatform, "linux"), Effect.provide(testLayer)), + ); + it.live("fails native Windows Prime closed across every routed entry point", () => { let processCalls = 0; let networkProbeCalls = 0; @@ -384,6 +506,8 @@ describe("ProviderInstanceRegistryLive — multi-instance codex slice", () => { installed: false, status: "disabled", models: [], + supportsMultipleInstances: false, + multipleInstancesUnavailableReason: PRIME_AGENT_NATIVE_WINDOWS_UNAVAILABLE_MESSAGE, }); expect(unavailable[0]!.message).toContain(PRIME_AGENT_NATIVE_WINDOWS_UNAVAILABLE_MESSAGE); expect(unavailable[0]!.unavailableReason).toContain( diff --git a/apps/server/src/provider/Layers/ProviderInstanceRegistryLive.ts b/apps/server/src/provider/Layers/ProviderInstanceRegistryLive.ts index cbd7f9a68..944ab026c 100644 --- a/apps/server/src/provider/Layers/ProviderInstanceRegistryLive.ts +++ b/apps/server/src/provider/Layers/ProviderInstanceRegistryLive.ts @@ -50,9 +50,11 @@ import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import { buildUnavailableProviderSnapshot } from "../unavailableProviderSnapshot.ts"; +import { ProviderDriverError } from "../Errors.ts"; import { ProviderInstanceRegistry, type ProviderInstanceRegistryShape, @@ -61,7 +63,14 @@ import { ProviderInstanceRegistryMutator, type ProviderInstanceRegistryMutatorShape, } from "../Services/ProviderInstanceRegistryMutator.ts"; -import type { AnyProviderDriver, ProviderInstance } from "../ProviderDriver.ts"; +import type { + AnyProviderDriver, + ProviderDriverCreateInput, + ProviderDriverMetadata, + ProviderDriverPreflightResult, + ProviderInstance, + ProviderRuntimeFence, +} from "../ProviderDriver.ts"; /** * Live registry entry: the materialized `ProviderInstance` + the fresh @@ -81,7 +90,15 @@ interface LiveEntry { interface RegistryState { readonly entries: Ref.Ref>; readonly unavailable: Ref.Ref>; + readonly generations: Ref.Ref< + ReadonlyMap< + ProviderInstanceId, + { readonly driver: ProviderDriverKind; readonly generation: object } + > + >; readonly changes: PubSub.PubSub; + readonly configured: Ref.Ref; + readonly reconcileSemaphore: Semaphore.Semaphore; } /** @@ -94,6 +111,70 @@ interface RegistryState { const entryEqual = (a: ProviderInstanceConfig, b: ProviderInstanceConfig): boolean => Equal.equals(a, b); +/** Labels are presentation-only. They update the snapshot without rematerializing the runtime. */ +const materialEntryEqual = (a: ProviderInstanceConfig, b: ProviderInstanceConfig): boolean => + Equal.equals( + { + driver: a.driver, + enabled: a.enabled, + environment: a.environment, + config: a.config, + }, + { + driver: b.driver, + enabled: b.enabled, + environment: b.environment, + config: b.config, + }, + ); + +const applyEntryPresentation = ( + provider: ServerProvider, + entry: ProviderInstanceConfig, + metadata: ProviderDriverMetadata, +): ServerProvider => { + const { + displayName: _displayName, + accentColor: _accentColor, + ...providerWithoutPresentation + } = provider; + const supportsMultipleInstances = + provider.supportsMultipleInstances ?? metadata.supportsMultipleInstances === true; + const multipleInstancesUnavailableReason = + provider.multipleInstancesUnavailableReason ?? metadata.multipleInstancesUnavailableReason; + return { + ...providerWithoutPresentation, + ...(entry.displayName ? { displayName: entry.displayName } : {}), + ...(entry.accentColor ? { accentColor: entry.accentColor } : {}), + supportsMultipleInstances, + ...(supportsMultipleInstances || !multipleInstancesUnavailableReason + ? {} + : { multipleInstancesUnavailableReason }), + }; +}; + +const withEntryPresentation = ( + instance: ProviderInstance, + entry: ProviderInstanceConfig, + metadata: ProviderDriverMetadata, +): ProviderInstance => ({ + ...instance, + displayName: entry.displayName, + accentColor: entry.accentColor, + snapshot: { + maintenanceCapabilities: instance.snapshot.maintenanceCapabilities, + getSnapshot: instance.snapshot.getSnapshot.pipe( + Effect.map((provider) => applyEntryPresentation(provider, entry, metadata)), + ), + refresh: instance.snapshot.refresh.pipe( + Effect.map((provider) => applyEntryPresentation(provider, entry, metadata)), + ), + streamChanges: instance.snapshot.streamChanges.pipe( + Stream.map((provider) => applyEntryPresentation(provider, entry, metadata)), + ), + }, +}); + /** * Resolve an entry's enabled state. An explicit false on either the * envelope or the raw config blob wins (most restrictive) — old settings @@ -121,6 +202,8 @@ const buildEntry = (input: { readonly instanceId: ProviderInstanceId; readonly rawInstanceId: string; readonly entry: ProviderInstanceConfig; + readonly generations: RegistryState["generations"]; + readonly preflight?: ReadonlyMap>; }): Effect.Effect< | { readonly kind: "live"; readonly live: LiveEntry } | { readonly kind: "unavailable"; readonly snapshot: ServerProvider }, @@ -128,7 +211,8 @@ const buildEntry = (input: { R > => Effect.gen(function* () { - const { driversById, parentScope, instanceId, rawInstanceId, entry } = input; + const { driversById, parentScope, instanceId, rawInstanceId, entry, generations, preflight } = + input; const driver = driversById.get(entry.driver); if (!driver) { return { @@ -166,6 +250,52 @@ const buildEntry = (input: { } const typedConfig = decodeResult.success; + const preparation = preflight?.get(instanceId); + if (preflight !== undefined && preparation?.kind !== "ready") { + const detail = + preparation?.kind === "unavailable" + ? preparation.error.detail + : "Provider instance identity preflight did not return a result."; + return { + kind: "unavailable" as const, + snapshot: yield* buildUnavailableProviderSnapshot({ + driverKind: entry.driver, + instanceId, + displayName: entry.displayName, + accentColor: entry.accentColor, + reason: detail, + }), + }; + } + + const runtimeFence: ProviderRuntimeFence | undefined = + preparation?.kind === "ready" && preparation.generation !== undefined + ? Object.freeze({ + generation: preparation.generation, + ...(preparation.configRevision === undefined + ? {} + : { configRevision: preparation.configRevision }), + isCurrent: Ref.get(generations).pipe( + Effect.map((current) => { + const published = current.get(instanceId); + return ( + published?.driver === entry.driver && + published.generation === preparation.generation + ); + }), + ), + }) + : undefined; + const createInput = { + instanceId, + displayName: entry.displayName, + accentColor: entry.accentColor, + environment: entry.environment ?? [], + enabled: resolveEntryEnabled(entry, typedConfig), + config: typedConfig, + ...(runtimeFence === undefined ? {} : { runtimeFence }), + } satisfies ProviderDriverCreateInput; + const childScope = yield* Scope.make(); // Attach the child scope to the registry's parent scope: if the // registry scope closes, each surviving instance's child scope is @@ -175,14 +305,7 @@ const buildEntry = (input: { yield* Scope.addFinalizer(parentScope, Scope.close(childScope, Exit.void).pipe(Effect.ignore)); const createResult = yield* driver - .create({ - instanceId, - displayName: entry.displayName, - accentColor: entry.accentColor, - environment: entry.environment ?? [], - enabled: resolveEntryEnabled(entry, typedConfig), - config: typedConfig, - }) + .create(createInput, preparation?.kind === "ready" ? preparation.preparation : undefined) .pipe(Effect.provideService(Scope.Scope, childScope), Effect.result); if (createResult._tag === "Failure") { yield* Effect.logError("Failed to create provider instance", { @@ -206,7 +329,13 @@ const buildEntry = (input: { return { kind: "live" as const, live: { - instance: createResult.success, + instance: withEntryPresentation( + runtimeFence === undefined + ? createResult.success + : { ...createResult.success, runtimeFence }, + entry, + driver.metadata, + ), scope: childScope, entry, }, @@ -217,6 +346,12 @@ const buildEntry = (input: { * Reconcile-only implementation of the mutator. Exposed to the hydration * layer; never called directly by the rest of the server. */ +const driverConfigEntries = ( + configMap: ProviderInstanceConfigMap, + driverKind: ProviderDriverKind, +): ReadonlyArray => + Object.entries(configMap).filter(([, entry]) => entry.driver === driverKind); + const makeReconcile = (input: { readonly state: RegistryState; readonly driversById: ReadonlyMap>; @@ -224,100 +359,274 @@ const makeReconcile = (input: { }): ((configMap: ProviderInstanceConfigMap) => Effect.Effect) => { const { state, driversById, parentScope } = input; return (configMap: ProviderInstanceConfigMap) => - Effect.gen(function* () { - const previousEntries = yield* Ref.get(state.entries); - const previousUnavailable = yield* Ref.get(state.unavailable); - const nextRaw = Object.entries(configMap); - const nextKeys = new Set( - nextRaw.map(([raw]) => ProviderInstanceId.make(raw)), - ); - - // 1. Close scopes for instances that disappeared or whose config - // changed. Do this BEFORE creating replacements so ids map 1-to-1 - // to live scopes at all times. - const removedIds: Array = []; - const replacedIds = new Set(); - for (const [instanceId, live] of previousEntries) { - if (!nextKeys.has(instanceId)) { - removedIds.push(instanceId); - continue; - } - const nextEntry = configMap[instanceId]; - if (nextEntry !== undefined && !entryEqual(live.entry, nextEntry)) { - replacedIds.add(instanceId); + state.reconcileSemaphore.withPermit( + Effect.gen(function* () { + const previousEntries = yield* Ref.get(state.entries); + const previousUnavailable = yield* Ref.get(state.unavailable); + const previousConfigMap = yield* Ref.get(state.configured); + const nextRaw = Object.entries(configMap); + const nextKeys = new Set( + nextRaw.map(([raw]) => ProviderInstanceId.make(raw)), + ); + const changedPreflightDrivers = new Set(); + const preflightByDriver = new Map< + ProviderDriverKind, + ReadonlyMap> + >(); + + // A missing capability is fail-closed. Decode the driver's complete + // desired set before any process launch so a false/missing driver can + // never materialize a second enabled instance, including after a + // remote multi-client settings race or direct settings-file edit. + for (const [driverKind, driver] of driversById) { + if (driver.metadata.supportsMultipleInstances === true) continue; + const entries = driverConfigEntries(configMap, driverKind); + const decoder = Schema.decodeUnknownEffect(driver.configSchema); + const enabledIds: ProviderInstanceId[] = []; + for (const [rawInstanceId, entry] of entries) { + const decoded = yield* decoder(entry.config ?? driver.defaultConfig()).pipe( + Effect.result, + ); + if (decoded._tag === "Success" && resolveEntryEnabled(entry, decoded.success)) { + enabledIds.push(ProviderInstanceId.make(rawInstanceId)); + } + } + if (enabledIds.length < 2) continue; + changedPreflightDrivers.add(driverKind); + const detail = + driver.metadata.multipleInstancesUnavailableReason ?? + `Driver '${driverKind}' supports only one enabled instance.`; + preflightByDriver.set( + driverKind, + new Map( + enabledIds.map((instanceId) => [ + instanceId, + { + kind: "unavailable" as const, + error: new ProviderDriverError({ driver: driverKind, instanceId, detail }), + }, + ]), + ), + ); } - } - for (const id of [...removedIds, ...replacedIds]) { - const live = previousEntries.get(id); - if (live) { - yield* Scope.close(live.scope, Exit.void).pipe(Effect.ignore); + + // Preflight the complete desired set before closing an old instance or + // starting a new one. An unchanged live instance is retained unless the + // set-wide result makes that instance unavailable. + for (const [driverKind, driver] of driversById) { + if (preflightByDriver.has(driverKind) || driver.preflight === undefined) continue; + const previousDriverEntries = driverConfigEntries(previousConfigMap, driverKind); + const nextDriverEntries = driverConfigEntries(configMap, driverKind); + const materialEntries = ( + entries: ReadonlyArray, + ) => + entries.map( + ([id, entry]) => + [ + id, + { + driver: entry.driver, + enabled: entry.enabled, + environment: entry.environment, + config: entry.config, + }, + ] as const, + ); + if ( + Equal.equals(materialEntries(previousDriverEntries), materialEntries(nextDriverEntries)) + ) { + continue; + } + changedPreflightDrivers.add(driverKind); + + const decoder = Schema.decodeUnknownEffect(driver.configSchema); + const decodedInputs: Array> = []; + for (const [rawInstanceId, entry] of nextDriverEntries) { + const decoded = yield* decoder(entry.config ?? driver.defaultConfig()).pipe( + Effect.result, + ); + if (decoded._tag === "Failure") continue; + decodedInputs.push({ + instanceId: ProviderInstanceId.make(rawInstanceId), + displayName: entry.displayName, + accentColor: entry.accentColor, + environment: entry.environment ?? [], + enabled: resolveEntryEnabled(entry, decoded.success), + config: decoded.success, + }); + } + preflightByDriver.set(driverKind, yield* driver.preflight(decodedInputs)); } - } - - // 2. Build additions and replacements. Walk `nextRaw` so the final - // entry order follows settings-author order. - const builtEntries = new Map(); - const builtUnavailable = new Map(); - let orderChanged = false; - const previousOrder = [...previousEntries.keys()]; - const nextOrder: Array = []; - - for (const [rawInstanceId, entry] of nextRaw) { - const instanceId = ProviderInstanceId.make(rawInstanceId); - nextOrder.push(instanceId); - - const existing = previousEntries.get(instanceId); - if (existing !== undefined && !replacedIds.has(instanceId)) { - // No-op update: keep the existing live entry and scope. - builtEntries.set(instanceId, existing); - continue; + + // 1. Identify instances whose material runtime changed. Their generations + // are retired before replacement construction, but their scopes close only + // after the exact replacement is published. + const removedIds: Array = []; + const replacedIds = new Set(); + const presentationChangedIds = new Set(); + for (const [instanceId, live] of previousEntries) { + if (!nextKeys.has(instanceId)) { + removedIds.push(instanceId); + continue; + } + const nextEntry = configMap[instanceId]; + const preflightResult = preflightByDriver.get(live.instance.driverKind)?.get(instanceId); + if ( + (nextEntry !== undefined && !materialEntryEqual(live.entry, nextEntry)) || + preflightResult?.kind === "unavailable" || + (preflightResult?.kind === "ready" && + preflightResult.generation !== undefined && + preflightResult.generation !== live.instance.runtimeFence?.generation) + ) { + replacedIds.add(instanceId); + } else if (nextEntry !== undefined && !entryEqual(live.entry, nextEntry)) { + presentationChangedIds.add(instanceId); + } } - const result = yield* buildEntry({ - driversById, - parentScope, - instanceId, - rawInstanceId, - entry, + // Publish the replacement generation before old fibers and scopes begin cleanup. + // An old callback becomes stale at this single atomic Ref update even if it runs late. + yield* Ref.update(state.generations, (previous) => { + const next = new Map(previous); + for (const id of [...removedIds, ...replacedIds]) next.delete(id); + for (const [rawInstanceId, entry] of nextRaw) { + const instanceId = ProviderInstanceId.make(rawInstanceId); + if (previousEntries.has(instanceId) && !replacedIds.has(instanceId)) continue; + const preparation = preflightByDriver.get(entry.driver)?.get(instanceId); + if (preparation?.kind === "ready" && preparation.generation !== undefined) { + next.set(instanceId, { driver: entry.driver, generation: preparation.generation }); + } + } + return next; }); - if (result.kind === "live") { - builtEntries.set(instanceId, result.live); - } else { - builtUnavailable.set(instanceId, result.snapshot); + + const driversRequiringTeardownBeforeCreate = new Set(); + for (const [driverKind, results] of preflightByDriver) { + if ( + [...results.values()].some( + (result) => result.kind === "ready" && result.teardownBeforeCreate === true, + ) + ) { + driversRequiringTeardownBeforeCreate.add(driverKind); + } + } + const closedBeforeCreate = new Set(); + for (const id of [...removedIds, ...replacedIds]) { + const live = previousEntries.get(id); + if ( + live !== undefined && + driversRequiringTeardownBeforeCreate.has(live.instance.driverKind) + ) { + yield* Scope.close(live.scope, Exit.void).pipe(Effect.ignore); + closedBeforeCreate.add(id); + } } - } - if (previousOrder.length === nextOrder.length) { - for (let i = 0; i < previousOrder.length; i++) { - if (previousOrder[i] !== nextOrder[i]) { - orderChanged = true; - break; + // 2. Build additions and replacements. Drivers with an exact teardown + // barrier have already closed their retired scopes; other drivers keep + // the gap-free publish-before-close behavior. + // This avoids an observable registry gap and makes all late old callbacks inert. Walk `nextRaw` so the final + // entry order follows settings-author order. + const builtEntries = new Map(); + const builtUnavailable = new Map(); + let orderChanged = false; + const previousOrder = [...previousEntries.keys()]; + const nextOrder: Array = []; + + for (const [rawInstanceId, entry] of nextRaw) { + const instanceId = ProviderInstanceId.make(rawInstanceId); + nextOrder.push(instanceId); + + const existing = previousEntries.get(instanceId); + if (existing !== undefined && !replacedIds.has(instanceId)) { + // Presentation-only changes keep the exact runtime, generation, and config revision. + builtEntries.set( + instanceId, + presentationChangedIds.has(instanceId) + ? { + ...existing, + instance: withEntryPresentation( + existing.instance, + entry, + driversById.get(entry.driver)!.metadata, + ), + entry, + } + : existing, + ); + continue; + } + const driver = driversById.get(entry.driver); + const previousShadow = previousUnavailable.get(instanceId); + if ( + previousShadow !== undefined && + driver?.preflight !== undefined && + !changedPreflightDrivers.has(entry.driver) + ) { + builtUnavailable.set(instanceId, previousShadow); + continue; } + + const result = yield* buildEntry({ + driversById, + parentScope, + instanceId, + rawInstanceId, + entry, + generations: state.generations, + ...(preflightByDriver.has(entry.driver) + ? { preflight: preflightByDriver.get(entry.driver)! } + : {}), + }); + if (result.kind === "live") { + builtEntries.set(instanceId, result.live); + } else { + builtUnavailable.set(instanceId, result.snapshot); + } + } + + if (previousOrder.length === nextOrder.length) { + for (let i = 0; i < previousOrder.length; i++) { + if (previousOrder[i] !== nextOrder[i]) { + orderChanged = true; + break; + } + } + } else { + orderChanged = true; } - } else { - orderChanged = true; - } - - const entriesChanged = - orderChanged || - removedIds.length > 0 || - replacedIds.size > 0 || - builtEntries.size !== previousEntries.size; - const unavailableChanged = - builtUnavailable.size !== previousUnavailable.size || - [...builtUnavailable].some(([id, snapshot]) => { - const prev = previousUnavailable.get(id); - return prev === undefined || !Equal.equals(prev, snapshot); - }) || - [...previousUnavailable].some(([id]) => !builtUnavailable.has(id)); - - yield* Ref.set(state.entries, builtEntries); - yield* Ref.set(state.unavailable, builtUnavailable); - - if (entriesChanged || unavailableChanged) { - yield* PubSub.publish(state.changes, undefined); - } - }); + + const entriesChanged = + orderChanged || + removedIds.length > 0 || + replacedIds.size > 0 || + presentationChangedIds.size > 0 || + builtEntries.size !== previousEntries.size; + const unavailableChanged = + builtUnavailable.size !== previousUnavailable.size || + [...builtUnavailable].some(([id, snapshot]) => { + const prev = previousUnavailable.get(id); + return prev === undefined || !Equal.equals(prev, snapshot); + }) || + [...previousUnavailable].some(([id]) => !builtUnavailable.has(id)); + + // Publish the exact replacement before closing its predecessor. Consumers either see + // the old fenced instance or the new current instance, never a missing middle state. + yield* Ref.set(state.entries, builtEntries); + yield* Ref.set(state.unavailable, builtUnavailable); + yield* Ref.set(state.configured, configMap); + + for (const id of [...removedIds, ...replacedIds]) { + if (closedBeforeCreate.has(id)) continue; + const live = previousEntries.get(id); + if (live) yield* Scope.close(live.scope, Exit.void).pipe(Effect.ignore); + } + + if (entriesChanged || unavailableChanged) { + yield* PubSub.publish(state.changes, undefined); + } + }), + ); }; /** @@ -366,10 +675,25 @@ export const makeProviderInstanceRegistry = (input: { const entries = yield* Ref.make>(new Map()); const unavailable = yield* Ref.make>(new Map()); + const generations = yield* Ref.make< + ReadonlyMap< + ProviderInstanceId, + { readonly driver: ProviderDriverKind; readonly generation: object } + > + >(new Map()); + const configured = yield* Ref.make({}); const changes = yield* PubSub.unbounded(); + const reconcileSemaphore = yield* Semaphore.make(1); yield* Effect.addFinalizer(() => PubSub.shutdown(changes)); - const state: RegistryState = { entries, unavailable, changes }; + const state: RegistryState = { + entries, + unavailable, + generations, + changes, + configured, + reconcileSemaphore, + }; const reconcileWithR = makeReconcile({ state, driversById, parentScope }); const reconcile: ProviderInstanceRegistryMutatorShape["reconcile"] = (configMap) => reconcileWithR(configMap).pipe(Effect.provideContext(driverContext)); diff --git a/apps/server/src/provider/Layers/ProviderRegistry.test.ts b/apps/server/src/provider/Layers/ProviderRegistry.test.ts index 83ae1e86d..3f1d79e61 100644 --- a/apps/server/src/provider/Layers/ProviderRegistry.test.ts +++ b/apps/server/src/provider/Layers/ProviderRegistry.test.ts @@ -330,6 +330,7 @@ function makeMutableServerSettingsService( yield* PubSub.publish(changes, next); return next; }), + mutateProviderInstances: () => Effect.die(new Error("unused in this test")), get streamChanges() { return Stream.fromPubSub(changes); }, @@ -344,7 +345,7 @@ function makeMutableServerSettingsService( describe("mergeProviderCapacityRefresh", () => { const checkedAt = "2026-08-04T18:00:00.000Z"; - const retentionIdentity = "openai-codex:fingerprint-a"; + const retentionIdentity = "openai-codex:revision-a"; const usageAt = (at: string, usedPercent: number) => ({ source: "primeAgentCodex", checkedAt: at, @@ -413,7 +414,7 @@ describe("mergeProviderCapacityRefresh", () => { backend: "anthropic", usageLimits: usageAt(checkedAt, 41), }, - retentionIdentity: "anthropic:fingerprint-a", + retentionIdentity: "anthropic:revision-a", }, ], refresh: { @@ -421,7 +422,7 @@ describe("mergeProviderCapacityRefresh", () => { { backend: { backend: "anthropic" }, didReadCapacity: false, - retentionIdentity: "anthropic:fingerprint-b", + retentionIdentity: "anthropic:revision-b", }, ], }, @@ -430,7 +431,7 @@ describe("mergeProviderCapacityRefresh", () => { [ { backend: { backend: "anthropic" }, - retentionIdentity: "anthropic:fingerprint-b", + retentionIdentity: "anthropic:revision-b", }, ], ); @@ -1362,7 +1363,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, ServerSettingsModule.layerTest(), Te } as const satisfies ServerProvider; const changes = yield* PubSub.unbounded(); const reads = yield* Ref.make(0); - const sameAccountIdentity = "anthropic:fingerprint-a"; + const sameAccountIdentity = "anthropic:revision-a"; const instance = { instanceId: primeInstanceId, driverKind: primeDriver, @@ -1520,7 +1521,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, ServerSettingsModule.layerTest(), Te }, }, didReadCapacity: true, - retentionIdentity: "anthropic:fingerprint-b", + retentionIdentity: "anthropic:revision-b", }, ], }); @@ -3186,3 +3187,174 @@ it.layer(Layer.mergeAll(NodeServices.layer, ServerSettingsModule.layerTest(), Te }); }, ); + +it.effect( + "drops deferred Prime snapshot, capacity, and volatile writes after generation B wins", + () => + Effect.scoped( + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("primeAgent"); + const driver = ProviderDriverKind.make("primeAgent"); + const current = yield* Ref.make(true); + const falseChecks = yield* Ref.make(0); + const bothRetiredCommitsChecked = yield* Deferred.make(); + const isCurrent = Ref.get(current).pipe( + Effect.tap((value) => + value + ? Effect.void + : Ref.updateAndGet(falseChecks, (count) => count + 1).pipe( + Effect.flatMap((count) => + count >= 2 + ? Deferred.succeed(bothRetiredCommitsChecked, undefined).pipe(Effect.asVoid) + : Effect.void, + ), + ), + ), + ); + const runtimeFence = { + generation: {}, + configRevision: "revision-a", + isCurrent, + }; + const snapshotStarted = yield* Deferred.make(); + const releaseSnapshot = yield* Deferred.make(); + const capacityStarted = yield* Deferred.make(); + const releaseCapacity = yield* Deferred.make(); + const baseProvider = { + instanceId, + driver, + status: "ready", + enabled: true, + installed: true, + auth: { status: "authenticated" }, + checkedAt: "2026-06-10T00:00:00.000Z", + version: "1.0.0", + models: [], + slashCommands: [], + skills: [], + } as const satisfies ServerProvider; + const staleProvider = { + ...baseProvider, + checkedAt: "2026-06-10T00:01:00.000Z", + models: [ + { + slug: "stale-model", + name: "Stale Model", + isCustom: false, + capabilities: null, + }, + ], + } as const satisfies ServerProvider; + const instance = { + instanceId, + driverKind: driver, + continuationIdentity: { + driverKind: driver, + continuationKey: "primeAgent:instance:primeAgent", + }, + displayName: undefined, + enabled: true, + runtimeFence, + snapshot: { + maintenanceCapabilities: makeManualOnlyProviderMaintenanceCapabilities({ + provider: driver, + packageName: null, + }), + getSnapshot: Effect.succeed(baseProvider), + refresh: Deferred.succeed(snapshotStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseSnapshot)), + Effect.as(staleProvider), + ), + streamChanges: Stream.empty, + }, + adapter: {} as ProviderInstance["adapter"], + textGeneration: {} as ProviderInstance["textGeneration"], + capacity: { + refresh: Deferred.succeed(capacityStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseCapacity)), + Effect.as({ + backends: [ + { + backend: { + backend: "anthropic", + usageLimits: { + source: "stale-capacity", + checkedAt: "2026-06-10T00:01:00.000Z", + windows: [{ label: "Session", usedPercent: 99 }], + }, + }, + didReadCapacity: true, + retentionIdentity: "anthropic:revision-a", + }, + ], + }), + ), + }, + } satisfies ProviderInstance; + const instanceRegistryLayer = Layer.succeed( + ProviderInstanceRegistry.ProviderInstanceRegistry, + { + getInstance: () => Effect.succeed(instance), + listInstances: Effect.succeed([instance]), + listUnavailable: Effect.succeed([]), + streamChanges: Stream.empty, + subscribeChanges: Effect.flatMap(PubSub.unbounded(), PubSub.subscribe), + }, + ); + const scope = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); + const services = yield* Layer.build( + ProviderRegistryLive.pipe( + Layer.provideMerge(instanceRegistryLayer), + Layer.provideMerge( + ServerConfig.layerTest(process.cwd(), { + prefix: "provider-generation-fence-", + }), + ), + Layer.provideMerge(NodeServices.layer), + ), + ).pipe(Scope.provide(scope)); + const registry = yield* ProviderRegistry.ProviderRegistry.pipe(Effect.provide(services)); + + const snapshotFiber = yield* registry.refresh().pipe(Effect.forkChild); + yield* registry.refreshProviderCapacity(instanceId); + yield* Deferred.await(snapshotStarted); + yield* Deferred.await(capacityStarted); + yield* Ref.set(current, false); + yield* Deferred.succeed(releaseSnapshot, undefined); + yield* Deferred.succeed(releaseCapacity, undefined); + yield* Deferred.await(bothRetiredCommitsChecked); + yield* Fiber.join(snapshotFiber); + + yield* registry.setProviderMaintenanceActionState({ + instanceId, + action: "update", + state: { + status: "running", + startedAt: "2026-06-10T00:02:00.000Z", + finishedAt: null, + message: "stale maintenance", + output: null, + }, + runtimeFence, + }); + yield* registry.setProviderRateLimitState({ + instanceId, + state: { + status: "rejected", + observedAt: "2026-06-10T00:02:00.000Z", + }, + runtimeFence, + }); + yield* registry.mergeProviderUsageWindows({ + instanceId, + source: "stale-usage", + observedAt: "2026-06-10T00:02:00.000Z", + windows: [{ label: "Session", usedPercent: 100 }], + runtimeFence, + }); + + assert.deepStrictEqual(yield* registry.getProviders, [baseProvider]); + }), + ), +); diff --git a/apps/server/src/provider/Layers/ProviderRegistry.ts b/apps/server/src/provider/Layers/ProviderRegistry.ts index 61d5a62b9..7e09e7744 100644 --- a/apps/server/src/provider/Layers/ProviderRegistry.ts +++ b/apps/server/src/provider/Layers/ProviderRegistry.ts @@ -38,6 +38,7 @@ import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Equal from "effect/Equal"; import * as FileSystem from "effect/FileSystem"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; @@ -66,6 +67,7 @@ import { capacityRefreshFromProviderBackends, type ProviderCapacityRefresh, type ProviderInstance, + type ProviderRuntimeFence, } from "../ProviderDriver.ts"; import { makeManualOnlyProviderMaintenanceCapabilities } from "../providerMaintenance.ts"; import type { ProviderSnapshotSource } from "../builtInProviderCatalog.ts"; @@ -277,14 +279,22 @@ const snapshotInstanceKey = (provider: ServerProvider): ProviderInstanceId => { // after `ProviderInstanceRegistry` rebuilds an instance (e.g. because // its settings changed), a fresh source rides the new PubSub instead // of a closed one. -const buildSnapshotSource = (instance: ProviderInstance): ProviderSnapshotSource => ({ +type RuntimeProviderSnapshotSource = ProviderSnapshotSource & { + readonly runtimeFence?: ProviderRuntimeFence | undefined; +}; + +const buildSnapshotSource = (instance: ProviderInstance): RuntimeProviderSnapshotSource => ({ instanceId: instance.instanceId, driverKind: instance.driverKind, getSnapshot: instance.snapshot.getSnapshot, refresh: instance.snapshot.refresh, streamChanges: instance.snapshot.streamChanges, + ...(instance.runtimeFence === undefined ? {} : { runtimeFence: instance.runtimeFence }), }); +const sourceIsCurrent = (source: RuntimeProviderSnapshotSource): Effect.Effect => + source.runtimeFence?.isCurrent ?? Effect.succeed(true); + export const ProviderRegistryLive = Layer.effect( ProviderRegistry, Effect.gen(function* () { @@ -336,7 +346,9 @@ export const ProviderRegistryLive = Layer.effect( if (fallbackProvider === undefined) { return undefined; } - return yield* readProviderStatusCache(filePath).pipe( + return yield* readProviderStatusCache(filePath, { + configRevision: source.runtimeFence?.configRevision, + }).pipe( Effect.provideService(FileSystem.FileSystem, fileSystem), Effect.flatMap((cachedProvider) => { if (cachedProvider === undefined) { @@ -402,7 +414,14 @@ export const ProviderRegistryLive = Layer.effect( // One capacity read in flight per instance, and no more than one a minute: // a burst of short turns must cost the backend one request, not one each. const capacityRunsRef = yield* Ref.make< - ReadonlyMap + ReadonlyMap< + ProviderInstanceId, + { + readonly startedAtMs: number; + readonly running: boolean; + readonly generation?: object | undefined; + } + > >(new Map()); const registryScope = yield* Effect.scope; @@ -413,28 +432,36 @@ export const ProviderRegistryLive = Layer.effect( const liveSubsRef = yield* Ref.make>( new Map(), ); + const subscriptionFibersRef = yield* Ref.make< + ReadonlyMap> + >(new Map()); // Serialize `syncLiveSources` so a rapid burst of reconciles doesn't // interleave two passes clobbering each other's fiber bookkeeping. const syncSemaphore = yield* Semaphore.make(1); - const getLiveSources: Effect.Effect> = Ref.get( + const getLiveSources: Effect.Effect> = Ref.get( liveSubsRef, ).pipe(Effect.map((map) => Array.from(map.values(), buildSnapshotSource))); - const persistProvider = (provider: ServerProvider) => + const persistProvider = (provider: ServerProvider, capturedFence?: ProviderRuntimeFence) => Effect.gen(function* () { - // Persist every instance — the file name is the instance id, so - // multi-instance setups (e.g. `codex_personal`, `codex_work`) each - // get their own cache. We resolve the path fresh so snapshots - // produced by newly-added instances post-boot still land on disk - // without the aggregator holding a stale `cachePathByInstance` - // entry. const key = snapshotInstanceKey(provider); + const currentInstance = yield* instanceRegistry.getInstance(key); + if (currentInstance?.driverKind !== provider.driver) return; + const runtimeFence = capturedFence ?? currentInstance.runtimeFence; + if (runtimeFence !== undefined && !(yield* runtimeFence.isCurrent)) return; const filePath = yield* resolveProviderStatusCachePath({ cacheDir: config.providerStatusCacheDir, instanceId: key, }).pipe(Effect.provideService(Path.Path, path)); - yield* writeProviderStatusCache({ filePath, provider }).pipe( + yield* writeProviderStatusCache({ + filePath, + provider, + ...(runtimeFence?.configRevision === undefined + ? {} + : { configRevision: runtimeFence.configRevision }), + ...(runtimeFence === undefined ? {} : { commitGuard: runtimeFence.isCurrent }), + }).pipe( Effect.provideService(FileSystem.FileSystem, fileSystem), Effect.provideService(Path.Path, path), Effect.tapError(Effect.logError), @@ -537,6 +564,7 @@ export const ProviderRegistryLive = Layer.effect( readonly publish?: boolean; readonly persist?: boolean; readonly replace?: boolean; + readonly runtimeFence?: ProviderRuntimeFence | undefined; }, ) { const nextProvidersWithUpdateState = yield* Effect.forEach( @@ -546,6 +574,9 @@ export const ProviderRegistryLive = Layer.effect( concurrency: "unbounded", }, ); + if (options?.runtimeFence !== undefined && !(yield* options.runtimeFence.isCurrent)) { + return yield* Ref.get(providersRef); + } const [previousProviders, providers, providersToPersist] = yield* Ref.modify( providersRef, (previousProviders) => { @@ -573,15 +604,27 @@ export const ProviderRegistryLive = Layer.effect( }, ); + if (options?.runtimeFence !== undefined && !(yield* options.runtimeFence.isCurrent)) { + yield* Ref.update(providersRef, (current) => + current === providers ? previousProviders : current, + ); + return yield* Ref.get(providersRef); + } if (haveProvidersChanged(previousProviders, providers)) { if (options?.persist !== false) { - yield* Effect.forEach(providersToPersist, persistProvider, { - concurrency: "unbounded", - discard: true, - }); + yield* Effect.forEach( + providersToPersist, + (provider) => persistProvider(provider, options?.runtimeFence), + { + concurrency: "unbounded", + discard: true, + }, + ); } if (options?.publish !== false) { - yield* PubSub.publish(changesPubSub, providers); + if (options?.runtimeFence === undefined || (yield* options.runtimeFence.isCurrent)) { + yield* PubSub.publish(changesPubSub, providers); + } } } @@ -592,8 +635,13 @@ export const ProviderRegistryLive = Layer.effect( provider: ServerProvider, options?: { readonly publish?: boolean; + readonly replace?: boolean; + readonly runtimeFence?: ProviderRuntimeFence | undefined; }, ) { + if (options?.runtimeFence !== undefined && !(yield* options.runtimeFence.isCurrent)) { + return yield* Ref.get(providersRef); + } return yield* upsertProviders([provider], options); }); @@ -602,7 +650,11 @@ export const ProviderRegistryLive = Layer.effect( readonly instanceId: ProviderInstanceId; readonly action: "update"; readonly state: ServerProviderUpdateState | null; + readonly runtimeFence?: ProviderRuntimeFence | undefined; }) { + if (input.runtimeFence !== undefined && !(yield* input.runtimeFence.isCurrent)) { + return yield* Ref.get(providersRef); + } yield* Ref.update(maintenanceActionStatesRef, (previous) => { const previousActions = previous.get(input.instanceId); const nextActions = { ...previousActions }; @@ -621,6 +673,9 @@ export const ProviderRegistryLive = Layer.effect( return next; }); + if (input.runtimeFence !== undefined && !(yield* input.runtimeFence.isCurrent)) { + return yield* Ref.get(providersRef); + } const existingProviders = yield* Ref.get(providersRef); const matchingProvider = existingProviders.find( (candidate) => candidate.instanceId === input.instanceId, @@ -632,6 +687,7 @@ export const ProviderRegistryLive = Layer.effect( const nextProvider = yield* applyVolatileProviderState(matchingProvider); return yield* upsertProviders([nextProvider], { persist: false, + runtimeFence: input.runtimeFence, }); }, ); @@ -639,7 +695,11 @@ export const ProviderRegistryLive = Layer.effect( const setProviderRateLimitState = Effect.fn("setProviderRateLimitState")(function* (input: { readonly instanceId: ProviderInstanceId; readonly state: ServerProviderRateLimit | null; + readonly runtimeFence?: ProviderRuntimeFence | undefined; }) { + if (input.runtimeFence !== undefined && !(yield* input.runtimeFence.isCurrent)) { + return yield* Ref.get(providersRef); + } yield* Ref.update(rateLimitStatesRef, (previous) => { const next = new Map(previous); if (input.state === null) { @@ -650,6 +710,9 @@ export const ProviderRegistryLive = Layer.effect( return next; }); + if (input.runtimeFence !== undefined && !(yield* input.runtimeFence.isCurrent)) { + return yield* Ref.get(providersRef); + } const existingProviders = yield* Ref.get(providersRef); const matchingProvider = existingProviders.find( (candidate) => candidate.instanceId === input.instanceId, @@ -663,6 +726,7 @@ export const ProviderRegistryLive = Layer.effect( const nextProvider = yield* applyVolatileProviderState(matchingProvider); return yield* upsertProviders([nextProvider], { persist: false, + runtimeFence: input.runtimeFence, }); }); @@ -671,7 +735,11 @@ export const ProviderRegistryLive = Layer.effect( readonly source: string; readonly observedAt: string; readonly windows: ReadonlyArray; + readonly runtimeFence?: ProviderRuntimeFence | undefined; }) { + if (input.runtimeFence !== undefined && !(yield* input.runtimeFence.isCurrent)) { + return yield* Ref.get(providersRef); + } const pushed = input.windows.map( (window): PushedUsageWindow => ({ window, observedAt: input.observedAt }), ); @@ -687,6 +755,9 @@ export const ProviderRegistryLive = Layer.effect( return next; }); + if (input.runtimeFence !== undefined && !(yield* input.runtimeFence.isCurrent)) { + return yield* Ref.get(providersRef); + } const existingProviders = yield* Ref.get(providersRef); const matchingProvider = existingProviders.find( (candidate) => candidate.instanceId === input.instanceId, @@ -698,6 +769,7 @@ export const ProviderRegistryLive = Layer.effect( const nextProvider = yield* applyVolatileProviderState(matchingProvider); return yield* upsertProviders([nextProvider], { persist: false, + runtimeFence: input.runtimeFence, }); }); @@ -705,27 +777,43 @@ export const ProviderRegistryLive = Layer.effect( const refreshProviderCapacity = Effect.fn("refreshProviderCapacity")(function* ( instanceId: ProviderInstanceId, + capturedFence?: ProviderRuntimeFence, ) { - const capacity = (yield* Ref.get(liveSubsRef)).get(instanceId)?.capacity; + if (capturedFence !== undefined && !(yield* capturedFence.isCurrent)) return; + const instance = (yield* Ref.get(liveSubsRef)).get(instanceId); + const capacity = instance?.capacity; if (!capacity) return; + const runtimeFence = capturedFence ?? instance.runtimeFence; + if (runtimeFence !== undefined && !(yield* runtimeFence.isCurrent)) return; + if ( + capturedFence !== undefined && + instance.runtimeFence?.generation !== capturedFence.generation + ) + return; + const generation = runtimeFence?.generation; const nowMs = yield* Effect.clockWith((clock) => clock.currentTimeMillis); const admitted = yield* Ref.modify(capacityRunsRef, (previous) => { const current = previous.get(instanceId); if ( current && + current.generation === generation && (current.running || nowMs - current.startedAtMs < CAPACITY_REFRESH_FLOOR_MS) ) { return [false, previous] as const; } const next = new Map(previous); - next.set(instanceId, { startedAtMs: nowMs, running: true }); + next.set(instanceId, { + startedAtMs: nowMs, + running: true, + ...(generation === undefined ? {} : { generation }), + }); return [true, next] as const; }); if (!admitted) return; const settle = Ref.update(capacityRunsRef, (previous) => { const current = previous.get(instanceId); - if (!current) return previous; + if (!current || current.generation !== generation) return previous; const next = new Map(previous); next.set(instanceId, { ...current, running: false }); return next; @@ -735,6 +823,7 @@ export const ProviderRegistryLive = Layer.effect( refresh === undefined ? Effect.void : Effect.gen(function* () { + if (runtimeFence !== undefined && !(yield* runtimeFence.isCurrent)) return; const observedAtMs = yield* Effect.clockWith((clock) => clock.currentTimeMillis); const existingProviders = yield* Ref.get(providersRef); const matchingProvider = existingProviders.find( @@ -746,6 +835,7 @@ export const ProviderRegistryLive = Layer.effect( refresh, nowMs: observedAtMs, }); + if (runtimeFence !== undefined && !(yield* runtimeFence.isCurrent)) return; yield* Ref.update(capacityOverlayRef, (previous) => { const next = new Map(previous); next.set(instanceId, { @@ -756,7 +846,11 @@ export const ProviderRegistryLive = Layer.effect( }); if (!matchingProvider) return; const nextProvider = yield* applyVolatileProviderState(matchingProvider); - yield* upsertProviders([nextProvider], { persist: false }); + if (runtimeFence !== undefined && !(yield* runtimeFence.isCurrent)) return; + yield* upsertProviders([nextProvider], { + persist: false, + runtimeFence, + }); }), ), Effect.ensuring(settle), @@ -766,12 +860,20 @@ export const ProviderRegistryLive = Layer.effect( }); const refreshOneSource = Effect.fn("refreshOneSource")(function* ( - providerSource: ProviderSnapshotSource, + providerSource: RuntimeProviderSnapshotSource, ) { return yield* providerSource.refresh.pipe( Effect.flatMap((nextProvider) => - correlateSnapshotWithSource(providerSource, nextProvider).pipe( - Effect.flatMap(syncProvider), + sourceIsCurrent(providerSource).pipe( + Effect.flatMap((current) => + current + ? correlateSnapshotWithSource(providerSource, nextProvider).pipe( + Effect.flatMap((provider) => + syncProvider(provider, { runtimeFence: providerSource.runtimeFence }), + ), + ) + : Ref.get(providersRef), + ), ), ), ); @@ -879,15 +981,73 @@ export const ProviderRegistryLive = Layer.effect( newlyAdded.push([instanceId, instance] as const); } + const retiredFencedIds = new Set(); + for (const [instanceId, previousInstance] of previousSubs) { + if (previousInstance.runtimeFence === undefined) continue; + const nextInstance = nextByInstance.get(instanceId); + if (nextInstance?.runtimeFence?.generation !== previousInstance.runtimeFence.generation) { + retiredFencedIds.add(instanceId); + } + } + if (retiredFencedIds.size > 0) { + const dropRetired = (previous: ReadonlyMap) => { + const next = new Map(previous); + for (const instanceId of retiredFencedIds) next.delete(instanceId); + return next; + }; + yield* Effect.all( + [ + Ref.update(maintenanceActionStatesRef, dropRetired), + Ref.update(rateLimitStatesRef, dropRetired), + Ref.update(pushedUsageRef, dropRetired), + Ref.update(capacityOverlayRef, dropRetired), + Ref.update(capacityRunsRef, dropRetired), + ], + { discard: true }, + ); + const [beforeRetirement, afterRetirement] = yield* Ref.modify( + providersRef, + (previous) => { + const next = previous.filter( + (provider) => !retiredFencedIds.has(snapshotInstanceKey(provider)), + ); + return [[previous, next] as const, next]; + }, + ); + if (haveProvidersChanged(beforeRetirement, afterRetirement)) { + yield* PubSub.publish(changesPubSub, afterRetirement); + } + } + // Fork long-lived subscriptions to each new/rebuilt instance's // change stream before reading its current snapshot. If the // driver's own initial probe finishes during this sync, either // the current read or the active subscriber observes the result. - for (const [, instance] of newlyAdded) { + for (const [instanceId, instance] of newlyAdded) { + const previousFiber = (yield* Ref.get(subscriptionFibersRef)).get(instanceId); + if (previousFiber !== undefined) yield* Fiber.interrupt(previousFiber); const source = buildSnapshotSource(instance); - yield* Stream.runForEach(source.streamChanges, (provider) => - correlateSnapshotWithSource(source, provider).pipe(Effect.flatMap(syncProvider)), + const fiber = yield* Stream.runForEach(source.streamChanges, (provider) => + sourceIsCurrent(source).pipe( + Effect.flatMap((current) => + current + ? correlateSnapshotWithSource(source, provider).pipe( + Effect.flatMap((correlated) => + syncProvider(correlated, { + runtimeFence: source.runtimeFence, + replace: retiredFencedIds.has(instanceId), + }), + ), + ) + : Effect.void, + ), + ), ).pipe(Effect.forkScoped); + yield* Ref.update(subscriptionFibersRef, (previous) => { + const next = new Map(previous); + next.set(instanceId, fiber); + return next; + }); } yield* Effect.yieldNow; @@ -901,8 +1061,14 @@ export const ProviderRegistryLive = Layer.effect( Effect.gen(function* () { const source = buildSnapshotSource(instance); const provider = yield* source.getSnapshot; + if (!(yield* sourceIsCurrent(source))) return; yield* correlateSnapshotWithSource(source, provider).pipe( - Effect.flatMap(syncProvider), + Effect.flatMap((correlated) => + syncProvider(correlated, { + runtimeFence: source.runtimeFence, + replace: retiredFencedIds.has(instance.instanceId), + }), + ), ); }).pipe(Effect.ignoreCause({ log: true })), { concurrency: "unbounded", discard: true }, @@ -917,6 +1083,16 @@ export const ProviderRegistryLive = Layer.effect( nextSubs.set(instanceId, instance); } yield* Ref.set(liveSubsRef, nextSubs); + const subscriptionFibers = yield* Ref.get(subscriptionFibersRef); + for (const [instanceId, fiber] of subscriptionFibers) { + if (nextSubs.has(instanceId)) continue; + yield* Fiber.interrupt(fiber); + yield* Ref.update(subscriptionFibersRef, (previous) => { + const next = new Map(previous); + if (next.get(instanceId) === fiber) next.delete(instanceId); + return next; + }); + } // Drop aggregator state for instances that have disappeared — // otherwise the UI would keep rendering ghosts. @@ -1045,6 +1221,10 @@ export const ProviderRegistryLive = Layer.effect( refreshInstance: (instanceId: ProviderInstanceId) => refreshInstance(instanceId).pipe(Effect.catchCause(recoverRefreshFailure)), getProviderMaintenanceCapabilitiesForInstance, + getProviderRuntimeFence: (instanceId: ProviderInstanceId) => + instanceRegistry + .getInstance(instanceId) + .pipe(Effect.map((instance) => instance?.runtimeFence)), setProviderMaintenanceActionState, setProviderRateLimitState, mergeProviderUsageWindows, diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts index 1f082cd47..ebd9e8b55 100644 --- a/apps/server/src/provider/Layers/ProviderService.test.ts +++ b/apps/server/src/provider/Layers/ProviderService.test.ts @@ -2664,6 +2664,62 @@ fanout.layer("ProviderServiceLive fanout", (it) => { }), ); + it.effect("quarantines a session result released after its Prime generation retires", () => + Effect.gen(function* () { + const provider = yield* ProviderService.ProviderService; + const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory; + const threadId = asThreadId("thread-generation-replaced-start"); + const current = yield* Ref.make(true); + Object.assign(fanout.codex.adapter, { + runtimeFence: { + generation: {}, + configRevision: "private-test-revision", + isCurrent: Ref.get(current), + }, + }); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + delete (fanout.codex.adapter as { runtimeFence?: unknown }).runtimeFence; + }), + ); + const startEntered = yield* Deferred.make(); + const releaseStart = yield* Deferred.make(); + fanout.codex.startSession.mockImplementationOnce((input) => + Deferred.succeed(startEntered, input).pipe(Effect.andThen(Deferred.await(releaseStart))), + ); + + const startFiber = yield* provider + .startSession(threadId, { + provider: CODEX_DRIVER, + providerInstanceId: codexInstanceId, + threadId, + runtimeMode: "full-access", + }) + .pipe(Effect.forkChild); + const startInput = yield* Deferred.await(startEntered); + yield* Ref.set(current, false); + const now = "2026-01-01T00:00:00.000Z"; + yield* Deferred.succeed(releaseStart, { + provider: CODEX_DRIVER, + providerInstanceId: codexInstanceId, + sessionIncarnationId: startInput.sessionIncarnationId, + status: "ready", + runtimeMode: "full-access", + threadId, + cwd: process.cwd(), + createdAt: now, + updatedAt: now, + }); + + assert.equal(Exit.isFailure(yield* Fiber.await(startFiber)), true); + assert.equal(Option.isNone(yield* directory.getBinding(threadId)), true); + assert.deepEqual( + (yield* provider.listSessions()).filter((session) => session.threadId === threadId), + [], + ); + }), + ); + it.effect("keeps Stop authoritative while an account transition is starting", () => Effect.gen(function* () { const provider = yield* ProviderService.ProviderService; diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index c70b6b7b0..a4d7b7203 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -85,6 +85,10 @@ import type { ProviderAdapterShape } from "../Services/ProviderAdapter.ts"; import * as ProviderAdapterRegistry from "../Services/ProviderAdapterRegistry.ts"; import * as ProviderService from "../Services/ProviderService.ts"; import * as ProviderSessionDirectory from "../Services/ProviderSessionDirectory.ts"; +import { + attachProviderRuntimeEventFence, + readProviderRuntimeEventFence, +} from "../providerRuntimeFenceMetadata.ts"; import { type EventNdjsonLogger } from "./EventNdjsonLogger.ts"; import * as ProviderEventLoggers from "./ProviderEventLoggers.ts"; import * as AnalyticsService from "../../telemetry/AnalyticsService.ts"; @@ -429,6 +433,25 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( return next; }).pipe(Effect.tap(() => reportStartReservationCount)); const nowIso = Effect.map(DateTime.now, DateTime.formatIso); + const requireAdapterGenerationCurrent = Effect.fnUntraced(function* ( + adapter: ProviderAdapterShape, + operation: string, + ) { + if (adapter.runtimeFence === undefined || (yield* adapter.runtimeFence.isCurrent)) return; + return yield* toValidationError( + operation, + "The selected provider runtime was replaced while this operation was pending.", + ); + }); + const recordAdapterAnalytics = ( + adapter: ProviderAdapterShape, + record: Effect.Effect, + ) => + adapter.runtimeFence === undefined + ? record + : Effect.flatMap(adapter.runtimeFence.isCurrent, (current) => + current ? record : Effect.void, + ); /** * Attach the `t3-code` MCP server to the session that is about to start. * @@ -455,43 +478,72 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ), ); - const prepareMcpSession = (threadId: ThreadId, providerInstanceId: ProviderInstanceId) => + const prepareMcpSession = ( + threadId: ThreadId, + providerInstanceId: ProviderInstanceId, + adapter: ProviderAdapterShape, + ) => Effect.gen(function* () { + const fence = adapter.runtimeFence; if (!(yield* agentBrowserAccessEnabled)) { - // Revoke as well as clear. Every other prepare path reaches - // `issueActiveMcpCredential`, which revokes the thread first, so - // skipping it here would leave a previously issued bearer token valid - // against `/mcp` for the rest of its liveness window — and later turns - // would keep refreshing it. A session restart (runtime mode, cwd, - // model) re-prepares without stopping, so it relies on this. - yield* revokeMcpCredential(threadId); - yield* Effect.sync(() => McpProviderSession.clearMcpProviderSession(threadId)); + yield* clearMcpSession(threadId, fence); return undefined; } - const credential = yield* issueMcpCredential({ threadId, providerInstanceId }); - if (credential) { - yield* Effect.sync(() => McpProviderSession.setMcpProviderSession(credential.config)); + const credential = yield* issueMcpCredential( + { threadId, providerInstanceId }, + fence?.isCurrent ?? Effect.succeed(true), + ); + if (credential === undefined) return undefined; + if (fence !== undefined && !(yield* fence.isCurrent)) { + yield* McpSessionRegistry.revokeActiveMcpProviderSession( + credential.config.providerSessionId, + ); + return undefined; + } + yield* Effect.sync(() => McpProviderSession.setMcpProviderSession(credential.config, fence)); + if (fence !== undefined && !(yield* fence.isCurrent)) { + yield* McpSessionRegistry.revokeActiveMcpProviderSession( + credential.config.providerSessionId, + ); + yield* Effect.sync(() => McpProviderSession.clearMcpProviderSession(threadId, fence)); + return undefined; } return credential; }); - const clearMcpSession = (threadId: ThreadId) => - revokeMcpCredential(threadId).pipe( - Effect.ensuring(Effect.sync(() => McpProviderSession.clearMcpProviderSession(threadId))), - ); + function clearMcpSession( + threadId: ThreadId, + runtimeFence?: import("../ProviderDriver.ts").ProviderRuntimeFence, + ) { + return Effect.gen(function* () { + if (runtimeFence === undefined) { + yield* revokeMcpCredential(threadId); + yield* Effect.sync(() => McpProviderSession.clearMcpProviderSession(threadId)); + return; + } + const config = McpProviderSession.readMcpProviderSession(threadId); + if ( + config === undefined || + !McpProviderSession.isMcpProviderSessionOwnedByGeneration(threadId, runtimeFence) + ) + return; + yield* McpSessionRegistry.revokeActiveMcpProviderSession(config.providerSessionId); + yield* Effect.sync(() => McpProviderSession.clearMcpProviderSession(threadId, runtimeFence)); + }); + } const clearAllMcpSessions = McpSessionRegistry.revokeAllActiveMcpCredentials().pipe( Effect.ensuring(Effect.sync(() => McpProviderSession.clearAllMcpProviderSessions())), ); const publishRuntimeEvent = (event: ProviderRuntimeEvent): Effect.Effect => - Effect.succeed(event).pipe( - Effect.tap((canonicalEvent) => - canonicalEventLogger - ? canonicalEventLogger.write(canonicalEvent, canonicalEvent.threadId) - : Effect.void, - ), - Effect.flatMap((canonicalEvent) => PubSub.publish(runtimeEventPubSub, canonicalEvent)), - Effect.asVoid, - ); + Effect.gen(function* () { + const runtimeFence = readProviderRuntimeEventFence(event); + if (runtimeFence !== undefined && !(yield* runtimeFence.isCurrent)) return; + if (canonicalEventLogger) { + yield* canonicalEventLogger.write(event, event.threadId, runtimeFence?.isCurrent); + } + if (runtimeFence !== undefined && !(yield* runtimeFence.isCurrent)) return; + yield* PubSub.publish(runtimeEventPubSub, event); + }); const requireBindingInstanceId = ( operation: string, @@ -518,22 +570,40 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( readonly modelSelection?: unknown; readonly lastRuntimeEvent?: string; readonly lastRuntimeEventAt?: string; + readonly runtimeFence?: import("../ProviderDriver.ts").ProviderRuntimeFence | undefined; }, ) => Effect.gen(function* () { + if (extra?.runtimeFence !== undefined && !(yield* extra.runtimeFence.isCurrent)) { + return yield* toValidationError( + "ProviderService.upsertSessionBinding", + "The selected provider runtime was replaced before its session binding committed.", + ); + } const providerInstanceId = yield* requireBindingInstanceId( "ProviderService.upsertSessionBinding", session, ); - yield* directory.upsert({ - threadId, - provider: session.provider, - providerInstanceId, - runtimeMode: session.runtimeMode, - status: toRuntimeStatus(session), - ...(session.resumeCursor !== undefined ? { resumeCursor: session.resumeCursor } : {}), - runtimePayload: toRuntimePayloadFromSession(session, extra), - }); + if (extra?.runtimeFence !== undefined && !(yield* extra.runtimeFence.isCurrent)) { + return yield* toValidationError( + "ProviderService.upsertSessionBinding", + "The selected provider runtime was replaced before its session binding committed.", + ); + } + yield* directory.upsert( + { + threadId, + provider: session.provider, + providerInstanceId, + runtimeMode: session.runtimeMode, + status: toRuntimeStatus(session), + ...(session.resumeCursor !== undefined ? { resumeCursor: session.resumeCursor } : {}), + runtimePayload: toRuntimePayloadFromSession(session, extra), + }, + extra?.runtimeFence === undefined + ? undefined + : { commitGuard: extra.runtimeFence.isCurrent }, + ); }); // `subscribedAdapters` is our source-of-truth for "which instance adapters @@ -621,6 +691,11 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( // lifecycle or output into the replacement session. const currentAdapters = yield* Ref.get(subscribedAdapters); if (currentAdapters.get(source.instanceId) !== source.adapter) return; + if ( + source.adapter.runtimeFence !== undefined && + !(yield* source.adapter.runtimeFence.isCurrent) + ) + return; const currentIncarnation = currentSessionIncarnations.get(event.threadId); if ( @@ -686,6 +761,12 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( .pipe(Effect.orDie); } + canonicalEvent = attachProviderRuntimeEventFence(canonicalEvent, source.adapter.runtimeFence); + if ( + source.adapter.runtimeFence !== undefined && + !(yield* source.adapter.runtimeFence.isCurrent) + ) + return; yield* increment(providerRuntimeEventsTotal, { provider: canonicalEvent.provider, eventType: canonicalEvent.type, @@ -716,7 +797,8 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( } const mcpSession = McpProviderSession.readMcpProviderSession(canonicalEvent.threadId); if (mcpSession?.providerInstanceId !== source.instanceId) return; - if (!stillActive) yield* clearMcpSession(canonicalEvent.threadId); + if (!stillActive) + yield* clearMcpSession(canonicalEvent.threadId, source.adapter.runtimeFence); } }); @@ -730,6 +812,9 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( const previous = yield* Ref.get(subscribedAdapters); const currentIds = yield* registry.listInstances(); const next = new Map>(); + const newSubscriptions: Array< + readonly [ProviderInstanceId, ProviderAdapterShape] + > = []; for (const id of currentIds) { const adapterOption = yield* registry .getByInstance(id) @@ -737,20 +822,32 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( if (Option.isNone(adapterOption)) continue; const adapter = adapterOption.value; next.set(id, adapter); - if (previous.get(id) !== adapter) { - yield* Stream.runForEach(adapter.streamEvents, (event) => - processRuntimeEvent( - { - instanceId: id, - provider: adapter.provider, - adapter, - }, - event, - ), - ).pipe(Effect.forkScoped); + if (previous.get(id) !== adapter) newSubscriptions.push([id, adapter]); + } + + for (const [instanceId, oldAdapter] of previous) { + if (oldAdapter.runtimeFence === undefined || next.get(instanceId) === oldAdapter) continue; + for (const [threadId, incarnation] of currentSessionIncarnations) { + if (incarnation.instanceId !== instanceId || incarnation.adapter !== oldAdapter) continue; + yield* clearMcpSession(threadId, oldAdapter.runtimeFence); + currentSessionIncarnations.delete(threadId); + activeTurnAdmissions.delete(threadId); } } + // Publish the new adapter set before any replacement stream can emit. yield* Ref.set(subscribedAdapters, next); + for (const [id, adapter] of newSubscriptions) { + yield* Stream.runForEach(adapter.streamEvents, (event) => + processRuntimeEvent( + { + instanceId: id, + provider: adapter.provider, + adapter, + }, + event, + ), + ).pipe(Effect.forkScoped); + } }); const instanceChanges = yield* registry.subscribeChanges; @@ -773,6 +870,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( }); return yield* Effect.gen(function* () { const adapter = yield* registry.getByInstance(bindingInstanceId); + yield* requireAdapterGenerationCurrent(adapter, input.operation); const hasResumeCursor = input.binding.resumeCursor !== null && input.binding.resumeCursor !== undefined; const hasActiveSession = yield* adapter.hasSession(input.binding.threadId); @@ -782,6 +880,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( (session) => session.threadId === input.binding.threadId, ); if (existing) { + yield* requireAdapterGenerationCurrent(adapter, input.operation); if (existing.sessionIncarnationId !== undefined) { currentSessionIncarnations.set(input.binding.threadId, { id: existing.sessionIncarnationId, @@ -792,12 +891,31 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( yield* upsertSessionBinding( { ...existing, providerInstanceId: bindingInstanceId }, input.binding.threadId, + { runtimeFence: adapter.runtimeFence }, + ).pipe( + Effect.onError(() => + adapter.stopSession(input.binding.threadId).pipe( + Effect.ignore, + Effect.ensuring(clearMcpSession(input.binding.threadId, adapter.runtimeFence)), + Effect.ensuring( + Effect.sync(() => { + const current = currentSessionIncarnations.get(input.binding.threadId); + if (current?.adapter === adapter) { + currentSessionIncarnations.delete(input.binding.threadId); + } + }), + ), + ), + ), + ); + yield* recordAdapterAnalytics( + adapter, + analytics.record("provider.session.recovered", { + provider: existing.provider, + strategy: "adopt-existing", + hasResumeCursor: existing.resumeCursor !== undefined, + }), ); - yield* analytics.record("provider.session.recovered", { - provider: existing.provider, - strategy: "adopt-existing", - hasResumeCursor: existing.resumeCursor !== undefined, - }); return { adapter, session: existing } as const; } } @@ -824,7 +942,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( instanceId: bindingInstanceId, adapter, }); - yield* prepareMcpSession(input.binding.threadId, bindingInstanceId); + yield* prepareMcpSession(input.binding.threadId, bindingInstanceId, adapter); const resumed = yield* adapter .startSession({ threadId: input.binding.threadId, @@ -838,13 +956,24 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( }) .pipe( Effect.onError(() => - clearMcpSession(input.binding.threadId).pipe( + clearMcpSession(input.binding.threadId, adapter.runtimeFence).pipe( Effect.ensuring(restorePreviousIncarnation), ), ), ); + yield* requireAdapterGenerationCurrent(adapter, input.operation).pipe( + Effect.onError(() => + adapter + .stopSession(input.binding.threadId) + .pipe( + Effect.ignore, + Effect.ensuring(clearMcpSession(input.binding.threadId, adapter.runtimeFence)), + Effect.ensuring(restorePreviousIncarnation), + ), + ), + ); if (resumed.provider !== adapter.provider) { - yield* clearMcpSession(input.binding.threadId); + yield* clearMcpSession(input.binding.threadId, adapter.runtimeFence); yield* restorePreviousIncarnation; return yield* toValidationError( input.operation, @@ -857,12 +986,27 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( sessionIncarnationId, }; - yield* upsertSessionBinding(resumedWithIncarnation, input.binding.threadId); - yield* analytics.record("provider.session.recovered", { - provider: resumed.provider, - strategy: "resume-thread", - hasResumeCursor: resumed.resumeCursor !== undefined, - }); + yield* upsertSessionBinding(resumedWithIncarnation, input.binding.threadId, { + runtimeFence: adapter.runtimeFence, + }).pipe( + Effect.onError(() => + adapter + .stopSession(input.binding.threadId) + .pipe( + Effect.ignore, + Effect.ensuring(clearMcpSession(input.binding.threadId, adapter.runtimeFence)), + Effect.ensuring(restorePreviousIncarnation), + ), + ), + ); + yield* recordAdapterAnalytics( + adapter, + analytics.record("provider.session.recovered", { + provider: resumed.provider, + strategy: "resume-thread", + hasResumeCursor: resumed.resumeCursor !== undefined, + }), + ); return { adapter, session: resumedWithIncarnation } as const; }).pipe( withMetrics({ @@ -961,9 +1105,12 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( yield* adapter.stopSession(input.threadId).pipe( Effect.tap(() => - analytics.record("provider.session.stopped", { - provider: adapter.provider, - }), + recordAdapterAnalytics( + adapter, + analytics.record("provider.session.stopped", { + provider: adapter.provider, + }), + ), ), Effect.catchCause((cause) => Effect.logWarning("provider.session.stop-stale-failed", { @@ -1067,6 +1214,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( "provider.cwd.effective": effectiveCwd ?? "", }); const adapter = yield* registry.getByInstance(resolvedInstanceId); + yield* requireAdapterGenerationCurrent(adapter, "ProviderService.startSession"); const sessionIncarnationId = RuntimeSessionId.make(NodeCrypto.randomUUID()); const previousIncarnation = currentSessionIncarnations.get(threadId); const restorePreviousIncarnation = restorePreviousIncarnationIfLive( @@ -1079,7 +1227,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( instanceId: resolvedInstanceId, adapter, }); - yield* prepareMcpSession(threadId, resolvedInstanceId); + yield* prepareMcpSession(threadId, resolvedInstanceId, adapter); const session = yield* adapter .startSession({ ...input, @@ -1092,12 +1240,14 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( }) .pipe( Effect.onError(() => - clearMcpSession(threadId).pipe(Effect.ensuring(restorePreviousIncarnation)), + clearMcpSession(threadId, adapter.runtimeFence).pipe( + Effect.ensuring(restorePreviousIncarnation), + ), ), ); if (session.provider !== adapter.provider) { - yield* clearMcpSession(threadId); + yield* clearMcpSession(threadId, adapter.runtimeFence); yield* restorePreviousIncarnation; return yield* toValidationError( "ProviderService.startSession", @@ -1110,7 +1260,13 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( sessionIncarnationId, }; const requireCurrentStartReservation = Effect.fnUntraced(function* () { - if (yield* isStartReservationCurrent(threadId, reservation.token)) return; + const reservationCurrent = yield* isStartReservationCurrent( + threadId, + reservation.token, + ); + const generationCurrent = + adapter.runtimeFence === undefined || (yield* adapter.runtimeFence.isCurrent); + if (reservationCurrent && generationCurrent) return; yield* adapter.stopSession(threadId).pipe( Effect.catchCause((cause) => Effect.logWarning("provider.session.stop-superseded-start-failed", { @@ -1120,11 +1276,15 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( }), ), ); - yield* clearMcpSession(threadId); + yield* clearMcpSession(threadId, adapter.runtimeFence); if (currentSessionIncarnations.get(threadId)?.id === sessionIncarnationId) { currentSessionIncarnations.delete(threadId); } - activeTurnAdmissions.delete(threadId); + if ( + activeTurnAdmissions.get(threadId)?.sessionIncarnationId === sessionIncarnationId + ) { + activeTurnAdmissions.delete(threadId); + } // The invalidation can land between the pre-upsert check and the // directory write. Remove only the exact late incarnation; a // newer start may already own the thread and must remain intact. @@ -1147,17 +1307,21 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( yield* requireCurrentStartReservation(); yield* upsertSessionBinding(sessionWithInstance, threadId, { modelSelection: input.modelSelection, - }); + runtimeFence: adapter.runtimeFence, + }).pipe(Effect.onError(() => requireCurrentStartReservation().pipe(Effect.ignore))); yield* requireCurrentStartReservation(); - yield* analytics.record("provider.session.started", { - provider: sessionWithInstance.provider, - runtimeMode: input.runtimeMode, - hasResumeCursor: sessionWithInstance.resumeCursor !== undefined, - hasCwd: typeof effectiveCwd === "string" && effectiveCwd.trim().length > 0, - hasModel: - typeof input.modelSelection?.model === "string" && - input.modelSelection.model.trim().length > 0, - }); + yield* recordAdapterAnalytics( + adapter, + analytics.record("provider.session.started", { + provider: sessionWithInstance.provider, + runtimeMode: input.runtimeMode, + hasResumeCursor: sessionWithInstance.resumeCursor !== undefined, + hasCwd: typeof effectiveCwd === "string" && effectiveCwd.trim().length > 0, + hasModel: + typeof input.modelSelection?.model === "string" && + input.modelSelection.model.trim().length > 0, + }), + ); // Changing runtime mode restarts the session, so the transition is only // observable here, by diffing against the mode the previous session for @@ -1165,11 +1329,14 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( // "started supervised, switched to full access" funnel answerable. const previousRuntimeMode = persistedBinding?.runtimeMode; if (previousRuntimeMode !== undefined && previousRuntimeMode !== input.runtimeMode) { - yield* analytics.record("provider.runtime_mode.changed", { - provider: sessionWithInstance.provider, - from: previousRuntimeMode, - to: input.runtimeMode, - }); + yield* recordAdapterAnalytics( + adapter, + analytics.record("provider.runtime_mode.changed", { + provider: sessionWithInstance.provider, + from: previousRuntimeMode, + to: input.runtimeMode, + }), + ); } yield* requireCurrentStartReservation(); @@ -1262,6 +1429,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( "provider.kind": routed.adapter.provider, ...(input.modelSelection?.model ? { "provider.model": input.modelSelection.model } : {}), }); + yield* requireAdapterGenerationCurrent(routed.adapter, "ProviderService.sendTurn"); // A turn is the clearest sign a session is still alive. The MCP // credential is minted once at session start and cannot be rotated into // an already-spawned agent process, so we keep the existing token valid @@ -1283,21 +1451,26 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( } if (input.admissionRequestId !== undefined && input.sessionIncarnationId !== undefined) { const binding = Option.getOrUndefined(yield* directory.getBinding(input.threadId)); - yield* directory.upsert({ - threadId: input.threadId, - provider: routed.adapter.provider, - providerInstanceId: routed.instanceId, - runtimeMode: routed.runtimeMode ?? binding?.runtimeMode ?? "full-access", - status: "starting", - ...(binding?.resumeCursor !== undefined ? { resumeCursor: binding.resumeCursor } : {}), - runtimePayload: { - ...(typeof binding?.runtimePayload === "object" && binding.runtimePayload !== null - ? binding.runtimePayload - : {}), - admissionRequestId: input.admissionRequestId, - sessionIncarnationId: input.sessionIncarnationId, + yield* directory.upsert( + { + threadId: input.threadId, + provider: routed.adapter.provider, + providerInstanceId: routed.instanceId, + runtimeMode: routed.runtimeMode ?? binding?.runtimeMode ?? "full-access", + status: "starting", + ...(binding?.resumeCursor !== undefined ? { resumeCursor: binding.resumeCursor } : {}), + runtimePayload: { + ...(typeof binding?.runtimePayload === "object" && binding.runtimePayload !== null + ? binding.runtimePayload + : {}), + admissionRequestId: input.admissionRequestId, + sessionIncarnationId: input.sessionIncarnationId, + }, }, - }); + routed.adapter.runtimeFence === undefined + ? undefined + : { commitGuard: routed.adapter.runtimeFence.isCurrent }, + ); } if (routed.adapter.prepareTurnRecovery !== undefined) { turnRecoveryPreparations.add(input.threadId); @@ -1313,7 +1486,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ) { currentSessionIncarnations.delete(input.threadId); } - return clearMcpSession(input.threadId); + return clearMcpSession(input.threadId, routed.adapter.runtimeFence); }), Effect.ignore, ), @@ -1325,6 +1498,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ), ); } + yield* requireAdapterGenerationCurrent(routed.adapter, "ProviderService.sendTurn"); if ( routed.adapter.prepareTurnRecovery !== undefined && input.sessionIncarnationId !== undefined @@ -1349,36 +1523,45 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( } } const turn = yield* routed.adapter.sendTurn(input); - yield* directory.upsert({ - threadId: input.threadId, - provider: routed.adapter.provider, - providerInstanceId: routed.instanceId, - status: "running", - ...(turn.resumeCursor !== undefined ? { resumeCursor: turn.resumeCursor } : {}), - runtimePayload: { - ...(input.modelSelection !== undefined ? { modelSelection: input.modelSelection } : {}), - activeTurnId: turn.turnId, - ...(input.admissionRequestId !== undefined - ? { activeTurnRequestId: input.admissionRequestId } - : {}), - ...(input.sessionIncarnationId !== undefined - ? { sessionIncarnationId: input.sessionIncarnationId } - : {}), - lastRuntimeEvent: "provider.sendTurn", - lastRuntimeEventAt: yield* nowIso, + yield* requireAdapterGenerationCurrent(routed.adapter, "ProviderService.sendTurn"); + yield* directory.upsert( + { + threadId: input.threadId, + provider: routed.adapter.provider, + providerInstanceId: routed.instanceId, + status: "running", + ...(turn.resumeCursor !== undefined ? { resumeCursor: turn.resumeCursor } : {}), + runtimePayload: { + ...(input.modelSelection !== undefined ? { modelSelection: input.modelSelection } : {}), + activeTurnId: turn.turnId, + ...(input.admissionRequestId !== undefined + ? { activeTurnRequestId: input.admissionRequestId } + : {}), + ...(input.sessionIncarnationId !== undefined + ? { sessionIncarnationId: input.sessionIncarnationId } + : {}), + lastRuntimeEvent: "provider.sendTurn", + lastRuntimeEventAt: yield* nowIso, + }, }, - }); - yield* analytics.record("provider.turn.sent", { - provider: routed.adapter.provider, - model: input.modelSelection?.model, - interactionMode: input.interactionMode, - // Session-start events alone skew runtime mode toward users who toggle - // often, since every toggle restarts the session. Recording it per turn - // gives a usage-weighted view and lets it cross with interactionMode. - runtimeMode: routed.runtimeMode, - attachmentCount: attachments.length, - hasInput: typeof input.input === "string" && input.input.trim().length > 0, - }); + routed.adapter.runtimeFence === undefined + ? undefined + : { commitGuard: routed.adapter.runtimeFence.isCurrent }, + ); + yield* recordAdapterAnalytics( + routed.adapter, + analytics.record("provider.turn.sent", { + provider: routed.adapter.provider, + model: input.modelSelection?.model, + interactionMode: input.interactionMode, + // Session-start events alone skew runtime mode toward users who toggle + // often, since every toggle restarts the session. Recording it per turn + // gives a usage-weighted view and lets it cross with interactionMode. + runtimeMode: routed.runtimeMode, + attachmentCount: attachments.length, + hasInput: typeof input.input === "string" && input.input.trim().length > 0, + }), + ); return turn; }).pipe( withMetrics({ @@ -1409,6 +1592,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( binding, ); const adapter = yield* registry.getByInstance(instanceId); + yield* requireAdapterGenerationCurrent(adapter, "ProviderService.recoverRestartSessions"); if ( adapter.recoverSession === undefined || adapter.activateRecoveredSession === undefined @@ -1428,7 +1612,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ) { return; } - yield* prepareMcpSession(binding.threadId, instanceId); + yield* prepareMcpSession(binding.threadId, instanceId, adapter); mcpPrepared = true; const modelSelection = readPersistedModelSelection(binding.runtimePayload); const recovered = yield* adapter.recoverSession({ @@ -1440,8 +1624,9 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ...(modelSelection === undefined ? {} : { modelSelection }), resumeCursor: binding.resumeCursor, }); + yield* requireAdapterGenerationCurrent(adapter, "ProviderService.recoverRestartSessions"); if (recovered === null) { - yield* clearMcpSession(binding.threadId); + yield* clearMcpSession(binding.threadId, adapter.runtimeFence); return; } adoptedAdapter = adapter; @@ -1464,8 +1649,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( { lastRuntimeEvent: "provider.restart-adopted", lastRuntimeEventAt: yield* nowIso, + runtimeFence: adapter.runtimeFence, }, ); + yield* requireAdapterGenerationCurrent(adapter, "ProviderService.recoverRestartSessions"); yield* adapter.activateRecoveredSession(binding.threadId); }).pipe( Effect.catchCause((cause) => @@ -1484,7 +1671,9 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( currentSessionIncarnations.delete(binding.threadId); } } - if (mcpPrepared) yield* clearMcpSession(binding.threadId); + if (mcpPrepared) { + yield* clearMcpSession(binding.threadId, adoptedAdapter?.runtimeFence); + } yield* Effect.logWarning("failed to adopt recoverable provider session", { threadId: binding.threadId, errorTag: causeErrorTag(cause), @@ -1517,9 +1706,12 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( "provider.turn_id": input.turnId, }); yield* routed.adapter.interruptTurn(routed.threadId, input.turnId); - yield* analytics.record("provider.turn.interrupted", { - provider: routed.adapter.provider, - }); + yield* recordAdapterAnalytics( + routed.adapter, + analytics.record("provider.turn.interrupted", { + provider: routed.adapter.provider, + }), + ); }).pipe( withMetrics({ counter: providerTurnsTotal, @@ -1554,10 +1746,13 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( "provider.request_id": input.requestId, }); yield* routed.adapter.respondToRequest(routed.threadId, input.requestId, input.decision); - yield* analytics.record("provider.request.responded", { - provider: routed.adapter.provider, - decision: input.decision, - }); + yield* recordAdapterAnalytics( + routed.adapter, + analytics.record("provider.request.responded", { + provider: routed.adapter.provider, + decision: input.decision, + }), + ); }).pipe( withMetrics({ counter: providerTurnsTotal, @@ -2336,7 +2531,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( !targetIsExact || currentSessionIncarnations.get(input.threadId) === currentIncarnation ) { - yield* clearMcpSession(input.threadId); + yield* clearMcpSession(input.threadId, currentIncarnation?.adapter.runtimeFence); currentSessionIncarnations.delete(input.threadId); } return; @@ -2360,7 +2555,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( if (routed.isActive) { yield* routed.adapter .stopSession(routed.threadId) - .pipe(Effect.ensuring(clearMcpSession(input.threadId))); + .pipe(Effect.ensuring(clearMcpSession(input.threadId, routed.adapter.runtimeFence))); // The adapter may hand off its sole stamped session.exited event through // an asynchronous relay after stopSession returns. Keep this exact // incarnation routable until processRuntimeEvent ingests that exit. A @@ -2372,7 +2567,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ) { currentSessionIncarnations.delete(input.threadId); } - yield* clearMcpSession(input.threadId); + yield* clearMcpSession(input.threadId, routed.adapter.runtimeFence); const latestBinding = Option.getOrUndefined(yield* directory.getBinding(input.threadId)); const latestIncarnationId = @@ -2398,19 +2593,27 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( }); } } else if (!targetIsExact || latestIsTarget) { - yield* directory.upsert({ - threadId: input.threadId, - provider: routed.adapter.provider, - providerInstanceId: routed.instanceId, - status: "stopped", - runtimePayload: { - activeTurnId: null, + yield* directory.upsert( + { + threadId: input.threadId, + provider: routed.adapter.provider, + providerInstanceId: routed.instanceId, + status: "stopped", + runtimePayload: { + activeTurnId: null, + }, }, - }); + routed.adapter.runtimeFence === undefined + ? undefined + : { commitGuard: routed.adapter.runtimeFence.isCurrent }, + ); } - yield* analytics.record("provider.session.stopped", { - provider: routed.adapter.provider, - }); + yield* recordAdapterAnalytics( + routed.adapter, + analytics.record("provider.session.stopped", { + provider: routed.adapter.provider, + }), + ); }).pipe( withMetrics({ counter: providerSessionsTotal, @@ -2674,10 +2877,13 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( "provider.rollback_turns": input.numTurns, }); yield* routed.adapter.rollbackThread(routed.threadId, input.numTurns); - yield* analytics.record("provider.conversation.rolled_back", { - provider: routed.adapter.provider, - turns: input.numTurns, - }); + yield* recordAdapterAnalytics( + routed.adapter, + analytics.record("provider.conversation.rolled_back", { + provider: routed.adapter.provider, + turns: input.numTurns, + }), + ); }).pipe( withMetrics({ counter: providerTurnsTotal, @@ -2808,26 +3014,35 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ); yield* adapter.stopAll().pipe( Effect.ensuring( - Effect.forEach(activeSessions, (session) => clearMcpSession(session.threadId), { - discard: true, - }), + Effect.forEach( + activeSessions, + (session) => clearMcpSession(session.threadId, adapter.runtimeFence), + { + discard: true, + }, + ), ), ); yield* Effect.forEach( bindings.filter((binding) => binding.providerInstanceId === instanceId), (binding) => Effect.flatMap(nowIso, (lastRuntimeEventAt) => - directory.upsert({ - threadId: binding.threadId, - provider: binding.provider, - providerInstanceId: instanceId, - status: "stopped", - runtimePayload: { - activeTurnId: null, - lastRuntimeEvent: "provider.stopAll", - lastRuntimeEventAt, + directory.upsert( + { + threadId: binding.threadId, + provider: binding.provider, + providerInstanceId: instanceId, + status: "stopped", + runtimePayload: { + activeTurnId: null, + lastRuntimeEvent: "provider.stopAll", + lastRuntimeEventAt, + }, }, - }), + adapter.runtimeFence === undefined + ? undefined + : { commitGuard: adapter.runtimeFence.isCurrent }, + ), ), { discard: true }, ); diff --git a/apps/server/src/provider/Layers/ProviderSessionDirectory.test.ts b/apps/server/src/provider/Layers/ProviderSessionDirectory.test.ts index 079b7f10e..134424a62 100644 --- a/apps/server/src/provider/Layers/ProviderSessionDirectory.test.ts +++ b/apps/server/src/provider/Layers/ProviderSessionDirectory.test.ts @@ -4,7 +4,7 @@ import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { ProviderDriverKind, ThreadId } from "@t3tools/contracts"; +import { ProviderDriverKind, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; import { it, assert } from "@effect/vitest"; import { assertSome } from "@effect/vitest/utils"; import * as Effect from "effect/Effect"; @@ -268,4 +268,20 @@ it.layer(makeDirectoryLayer(SqlitePersistenceMemory))("ProviderSessionDirectoryL NodeFS.rmSync(tempDir, { recursive: true, force: true }); })); + it("skips a session binding when its private commit guard retires", () => + Effect.gen(function* () { + const directory = yield* ProviderSessionDirectory; + const threadId = ThreadId.make("thread-retired-generation"); + + yield* directory.upsert( + { + provider: ProviderDriverKind.make("primeAgent"), + providerInstanceId: ProviderInstanceId.make("primeAgent"), + threadId, + }, + { commitGuard: Effect.succeed(false) }, + ); + + assert.isTrue(Option.isNone(yield* directory.getBinding(threadId))); + })); }); diff --git a/apps/server/src/provider/Layers/ProviderSessionDirectory.ts b/apps/server/src/provider/Layers/ProviderSessionDirectory.ts index 70b004702..b2bffae3b 100644 --- a/apps/server/src/provider/Layers/ProviderSessionDirectory.ts +++ b/apps/server/src/provider/Layers/ProviderSessionDirectory.ts @@ -102,7 +102,7 @@ const makeProviderSessionDirectory = Effect.gen(function* () { ); const mutationPermit = Semaphore.makeUnsafe(1); - const upsert: ProviderSessionDirectoryShape["upsert"] = Effect.fn(function* (binding) { + const upsert: ProviderSessionDirectoryShape["upsert"] = Effect.fn(function* (binding, options) { return yield* mutationPermit.withPermit( Effect.gen(function* () { const existing = yield* repository @@ -132,6 +132,7 @@ const makeProviderSessionDirectory = Effect.gen(function* () { issue: "providerInstanceId is required for provider session runtime bindings.", }); } + if (options?.commitGuard !== undefined && !(yield* options.commitGuard)) return; yield* repository .upsert({ threadId: resolvedThreadId, diff --git a/apps/server/src/provider/ProviderDriver.ts b/apps/server/src/provider/ProviderDriver.ts index a1fab1575..a3524717d 100644 --- a/apps/server/src/provider/ProviderDriver.ts +++ b/apps/server/src/provider/ProviderDriver.ts @@ -46,11 +46,12 @@ export interface ProviderDriverMetadata { readonly displayName: string; /** * Whether the driver may be instantiated more than once concurrently. - * Defaults to `true`. Set to `false` for drivers that wrap a global - * resource (e.g. a single desktop app socket) — the registry then - * rejects multi-instance configurations with a clear error. + * Missing is fail-closed. Drivers must publish `true` only after their + * per-instance process, credential, state, and cleanup boundaries are proved. */ readonly supportsMultipleInstances?: boolean; + /** Actionable explanation when multiple enabled instances are unavailable. */ + readonly multipleInstancesUnavailableReason?: string | undefined; } /** @@ -62,6 +63,15 @@ export interface ProviderDriverMetadata { * instance of the same driver does not reach into the first instance's * state. */ +export interface ProviderRuntimeFence { + /** Opaque process-local identity. It must never cross a public or observability boundary. */ + readonly generation: object; + /** Private disk-cache correlation only. It must never enter a provider snapshot. */ + readonly configRevision?: string | undefined; + /** Re-read by every async result immediately before it commits side effects. */ + readonly isCurrent: Effect.Effect; +} + export interface ProviderInstance { readonly instanceId: ProviderInstanceId; readonly driverKind: ProviderDriverKind; @@ -80,6 +90,8 @@ export interface ProviderInstance { * drivers that use Pylon's configured accounts have nothing to read. */ readonly capacity?: ProviderCapacitySource | undefined; + /** Server-private replacement fence. Prime is the first fenced driver. */ + readonly runtimeFence?: ProviderRuntimeFence | undefined; } export interface ProviderCapacitySource { @@ -98,7 +110,7 @@ export interface ProviderCapacityRefresh { export interface ProviderBackendCapacityRead { readonly backend: ServerProviderBackend; readonly didReadCapacity: boolean; - /** Secret-safe identity used only to retain a failed read for the same account. */ + /** Private runtime-revision identity. Never derive this value from account or secret material. */ readonly retentionIdentity?: string | undefined; } @@ -159,6 +171,8 @@ export interface ProviderDriverCreateInput { readonly environment: ProviderInstanceEnvironment; readonly enabled: boolean; readonly config: Config; + /** Present only when preflight published a process-local replacement generation. */ + readonly runtimeFence?: ProviderRuntimeFence | undefined; } /** @@ -174,7 +188,20 @@ export interface ProviderDriverCreateInput { * scope closes. Two calls to `create` with different `instanceId` / * `config` MUST yield instances with no shared mutable state. */ -export interface ProviderDriver { +export type ProviderDriverPreflightResult = + | { + readonly kind: "ready"; + readonly preparation: Preparation; + /** Opaque process-local identity, published before an old scope is closed. */ + readonly generation?: object | undefined; + /** Random server-owned correlation for private disk caches only. */ + readonly configRevision?: string | undefined; + /** Close the retired driver set before any replacement can become reachable. */ + readonly teardownBeforeCreate?: boolean | undefined; + } + | { readonly kind: "unavailable"; readonly error: ProviderDriverError }; + +export interface ProviderDriver { readonly driverKind: ProviderDriverKind; readonly metadata: ProviderDriverMetadata; /** @@ -203,15 +230,31 @@ export interface ProviderDriver { * to construct a "blank" instance. */ readonly defaultConfig: () => Config; + /** + * Optional driver-wide identity preflight. The registry runs this for the + * complete driver set before it closes or creates any affected instance. + * The registry must apply unavailable set-wide results to already-live + * participants before it starts replacements or additions. + */ + readonly preflight?: ( + inputs: ReadonlyArray>, + ) => Effect.Effect< + ReadonlyMap>, + never, + R + >; /** * Materialize one instance. The returned effect runs in a scope owned * by the registry; closing that scope releases every resource the * driver opened. Failures become unavailable shadow snapshots — the * driver MUST NOT throw defects. */ - readonly create: ( - input: ProviderDriverCreateInput, - ) => Effect.Effect; + readonly create: { + bivarianceHack( + input: ProviderDriverCreateInput, + preparation?: Preparation, + ): Effect.Effect; + }["bivarianceHack"]; } /** @@ -224,4 +267,4 @@ export interface ProviderDriver { // needs the original `Config` type. Using `unknown` instead would force // `create` callers into casts since `unknown` is not assignable to a // concrete `Config` from inside the driver body. -export type AnyProviderDriver = ProviderDriver; +export type AnyProviderDriver = ProviderDriver; diff --git a/apps/server/src/provider/Services/ProviderAdapter.ts b/apps/server/src/provider/Services/ProviderAdapter.ts index 087725b5a..c7317c686 100644 --- a/apps/server/src/provider/Services/ProviderAdapter.ts +++ b/apps/server/src/provider/Services/ProviderAdapter.ts @@ -42,6 +42,7 @@ import type { } from "@t3tools/contracts"; import type * as Effect from "effect/Effect"; import type * as Stream from "effect/Stream"; +import type { ProviderRuntimeFence } from "../ProviderDriver.ts"; export type ProviderSessionModelSwitchMode = "in-session" | "unsupported"; export type ProviderConversationRollbackMode = "absolute" | "relative" | "unsupported"; @@ -84,6 +85,8 @@ export interface ProviderAdapterShape { */ readonly provider: ProviderDriverKind; readonly capabilities: ProviderAdapterCapabilities; + /** Server-private materialization fence. It never crosses provider contracts. */ + readonly runtimeFence?: ProviderRuntimeFence | undefined; /** * Start a provider-backed session. diff --git a/apps/server/src/provider/Services/ProviderRegistry.ts b/apps/server/src/provider/Services/ProviderRegistry.ts index 347f1df6b..5af169037 100644 --- a/apps/server/src/provider/Services/ProviderRegistry.ts +++ b/apps/server/src/provider/Services/ProviderRegistry.ts @@ -18,6 +18,7 @@ import * as Context from "effect/Context"; import type * as Effect from "effect/Effect"; import type * as Stream from "effect/Stream"; import type { ProviderMaintenanceCapabilities } from "../providerMaintenance.ts"; +import type { ProviderRuntimeFence } from "../ProviderDriver.ts"; export type ProviderMaintenanceActionKind = "update"; @@ -59,6 +60,11 @@ export interface ProviderRegistryShape { provider: ProviderDriverKind, ) => Effect.Effect; + /** Private process-local fence capture for long-running maintenance work. */ + readonly getProviderRuntimeFence?: ( + instanceId: ProviderInstanceId, + ) => Effect.Effect; + /** * Apply volatile maintenance-action state to one configured instance. * This state is never persisted to disk. Today only update actions are @@ -69,6 +75,7 @@ export interface ProviderRegistryShape { readonly instanceId: ProviderInstanceId; readonly action: ProviderMaintenanceActionKind; readonly state: ServerProviderUpdateState | null; + readonly runtimeFence?: ProviderRuntimeFence | undefined; }) => Effect.Effect>; /** @@ -84,6 +91,7 @@ export interface ProviderRegistryShape { readonly setProviderRateLimitState: (input: { readonly instanceId: ProviderInstanceId; readonly state: ServerProviderRateLimit | null; + readonly runtimeFence?: ProviderRuntimeFence | undefined; }) => Effect.Effect>; /** @@ -104,6 +112,7 @@ export interface ProviderRegistryShape { readonly source: string; readonly observedAt: string; readonly windows: ReadonlyArray; + readonly runtimeFence?: ProviderRuntimeFence | undefined; }) => Effect.Effect>; /** @@ -116,7 +125,10 @@ export interface ProviderRegistryShape { * a burst of short turns costs one read. Unknown instances and instances * without a capacity source are no-ops. */ - readonly refreshProviderCapacity: (instanceId: ProviderInstanceId) => Effect.Effect; + readonly refreshProviderCapacity: ( + instanceId: ProviderInstanceId, + runtimeFence?: ProviderRuntimeFence, + ) => Effect.Effect; /** * Stream of provider snapshot updates — one emission per aggregated diff --git a/apps/server/src/provider/Services/ProviderSessionDirectory.ts b/apps/server/src/provider/Services/ProviderSessionDirectory.ts index 766633634..b995a1257 100644 --- a/apps/server/src/provider/Services/ProviderSessionDirectory.ts +++ b/apps/server/src/provider/Services/ProviderSessionDirectory.ts @@ -44,6 +44,7 @@ export type ProviderSessionDirectoryWriteError = export interface ProviderSessionDirectoryShape { readonly upsert: ( binding: ProviderRuntimeBinding, + options?: { readonly commitGuard?: Effect.Effect | undefined }, ) => Effect.Effect; readonly getProvider: ( diff --git a/apps/server/src/provider/claudeOAuthUsage.ts b/apps/server/src/provider/claudeOAuthUsage.ts index 270ed24ab..609dbd9a3 100644 --- a/apps/server/src/provider/claudeOAuthUsage.ts +++ b/apps/server/src/provider/claudeOAuthUsage.ts @@ -354,6 +354,9 @@ export const fetchOAuthUsageWithToken = Effect.fn("fetchOAuthUsageWithToken")(fu readonly freshForMs?: number | undefined; /** Share transient failures as well as successful readings and throttles. */ readonly shareFailures?: boolean | undefined; + /** Private random cache correlation for fenced materializations. */ + readonly configRevision?: string | undefined; + readonly commitGuard?: Effect.Effect | undefined; }): Effect.fn.Return< ClaudeOAuthUsageRead, never, @@ -362,7 +365,7 @@ export const fetchOAuthUsageWithToken = Effect.fn("fetchOAuthUsageWithToken")(fu const cacheDir = input.sharedCacheDir ?? (yield* resolveSharedUsageCacheDir); const nowMs = yield* Effect.clockWith((clock) => clock.currentTimeMillis); const shared = decideSharedUsageRead( - yield* readSharedUsageEntry(cacheDir, input.cacheKey), + yield* readSharedUsageEntry(cacheDir, input.cacheKey, input.configRevision), nowMs, input.freshForMs, ); @@ -384,7 +387,7 @@ export const fetchOAuthUsageWithToken = Effect.fn("fetchOAuthUsageWithToken")(fu if (!acquiredLock) { // Another server is reading this account right now; its answer lands in // the shared file shortly. Serve what is there and check back soon. - const existing = yield* readSharedUsageEntry(cacheDir, input.cacheKey); + const existing = yield* readSharedUsageEntry(cacheDir, input.cacheKey, input.configRevision); return { usageLimits: existing?.usageLimits, cacheForMs: SHARED_USAGE_BUSY_RETRY_MS, @@ -396,8 +399,15 @@ export const fetchOAuthUsageWithToken = Effect.fn("fetchOAuthUsageWithToken")(fu const failedAt = DateTime.formatIso(DateTime.makeUnsafe(nowMs)); const failedRead = input.shareFailures ? Effect.gen(function* () { - yield* markSharedUsageReadFailed(cacheDir, input.cacheKey, failedAt); - const retained = yield* readSharedUsageEntry(cacheDir, input.cacheKey); + yield* markSharedUsageReadFailed(cacheDir, input.cacheKey, failedAt, { + configRevision: input.configRevision, + commitGuard: input.commitGuard, + }); + const retained = yield* readSharedUsageEntry( + cacheDir, + input.cacheKey, + input.configRevision, + ); return { usageLimits: retained?.usageLimits, cacheForMs: SHARED_USAGE_FAILURE_BACKOFF_MS, @@ -432,11 +442,17 @@ export const fetchOAuthUsageWithToken = Effect.fn("fetchOAuthUsageWithToken")(fu if (Option.isNone(attempt) || attempt.value.kind === "failed") return yield* failedRead; if (attempt.value.kind === "throttled") { const throttledForMs = throttleDelayFromRetryAfter(attempt.value.retryAfter, nowMs); - yield* writeSharedUsageEntry(cacheDir, input.cacheKey, { - version: 1, - readAt: DateTime.formatIso(DateTime.makeUnsafe(nowMs)), - throttledUntil: DateTime.formatIso(DateTime.makeUnsafe(nowMs + throttledForMs)), - }); + yield* writeSharedUsageEntry( + cacheDir, + input.cacheKey, + { + version: 1, + ...(input.configRevision === undefined ? {} : { configRevision: input.configRevision }), + readAt: DateTime.formatIso(DateTime.makeUnsafe(nowMs)), + throttledUntil: DateTime.formatIso(DateTime.makeUnsafe(nowMs + throttledForMs)), + }, + input.commitGuard, + ); return { usageLimits: undefined, cacheForMs: throttledForMs, didRead: false }; } @@ -446,11 +462,17 @@ export const fetchOAuthUsageWithToken = Effect.fn("fetchOAuthUsageWithToken")(fu input.source, ); if (!usageLimits) return yield* failedRead; - yield* writeSharedUsageEntry(cacheDir, input.cacheKey, { - version: 1, - readAt: DateTime.formatIso(DateTime.makeUnsafe(nowMs)), - usageLimits, - }); + yield* writeSharedUsageEntry( + cacheDir, + input.cacheKey, + { + version: 1, + ...(input.configRevision === undefined ? {} : { configRevision: input.configRevision }), + readAt: DateTime.formatIso(DateTime.makeUnsafe(nowMs)), + usageLimits, + }, + input.commitGuard, + ); return { usageLimits, cacheForMs: SHARED_USAGE_READ_TTL_MS, didRead: true }; }).pipe(Effect.ensuring(releaseSharedUsageLock(cacheDir, input.cacheKey))); }); diff --git a/apps/server/src/provider/makeManagedServerProvider.test.ts b/apps/server/src/provider/makeManagedServerProvider.test.ts index 8b3c47a7c..83358aebb 100644 --- a/apps/server/src/provider/makeManagedServerProvider.test.ts +++ b/apps/server/src/provider/makeManagedServerProvider.test.ts @@ -450,6 +450,7 @@ describe("makeManagedServerProvider", () => { ready: Effect.void, getSettings: Ref.get(serverSettingsRef), updateSettings: () => Effect.die(new Error("unused in this test")), + mutateProviderInstances: () => Effect.die(new Error("unused in this test")), streamChanges: Stream.empty, subscribeChanges: PubSub.subscribe(serverSettingsChanges).pipe( Effect.map((subscription) => Stream.fromSubscription(subscription)), diff --git a/apps/server/src/provider/makeManagedServerProvider.ts b/apps/server/src/provider/makeManagedServerProvider.ts index b1ffc770a..a3dfc177e 100644 --- a/apps/server/src/provider/makeManagedServerProvider.ts +++ b/apps/server/src/provider/makeManagedServerProvider.ts @@ -59,6 +59,8 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")( readonly refreshInterval?: Duration.Input; readonly refreshOnInterval?: boolean; readonly checkProviderOnSettingsChange?: (previous: Settings, next: Settings) => boolean; + /** Process-local replacement fence. A stale async result is discarded at commit. */ + readonly commitGuard?: Effect.Effect; }): Effect.fn.Return< ManagedServerProviderShape, ServerSettingsError, @@ -81,11 +83,13 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")( const settingsRef = yield* Ref.make(initialSettings); const enrichmentFiberRef = yield* Ref.make | null>(null); const scope = yield* Effect.scope; + const isCommitCurrent = input.commitGuard ?? Effect.succeed(true); const publishEnrichedSnapshotBase = Effect.fn("publishEnrichedSnapshot")(function* ( generation: number, nextSnapshot: ServerProvider, ) { + if (!(yield* isCommitCurrent)) return; const snapshotToPublish = yield* Ref.modify(snapshotStateRef, (state) => { if (state.enrichmentGeneration !== generation) { return [null, state] as const; @@ -109,7 +113,9 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")( if (snapshotToPublish === null) { return; } - yield* PubSub.publish(changesPubSub, snapshotToPublish); + if (yield* isCommitCurrent) { + yield* PubSub.publish(changesPubSub, snapshotToPublish); + } }); const publishEnrichedSnapshot = (generation: number, nextSnapshot: ServerProvider) => refreshSemaphore.withPermits(1)(publishEnrichedSnapshotBase(generation, nextSnapshot)); @@ -171,6 +177,7 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")( input.checkProviderWithPublishedModels !== undefined ? input.checkProviderWithPublishedModels : input.checkProvider; + if (!(yield* isCommitCurrent)) return stateBeforeCheck.snapshot; const [nextSnapshot, nextGeneration] = yield* Ref.modify(snapshotStateRef, (state) => { const generation = input.enrichSnapshot ? state.enrichmentGeneration + 1 @@ -190,6 +197,7 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")( ] as const; }); yield* Ref.set(settingsRef, nextSettings); + if (!(yield* isCommitCurrent)) return nextSnapshot; yield* PubSub.publish(changesPubSub, nextSnapshot); yield* restartSnapshotEnrichment(nextSettings, nextSnapshot, nextGeneration); return nextSnapshot; @@ -200,6 +208,7 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")( const publishModelsBase = Effect.fn("publishModels")(function* ( models: ServerProvider["models"], ) { + if (!(yield* isCommitCurrent)) return; const snapshotToPublish = yield* Ref.modify(snapshotStateRef, (state) => { const snapshotWithModels = Equal.equals(state.snapshot.models, models) ? state.snapshot @@ -222,7 +231,9 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")( if (snapshotToPublish === null) { return; } - yield* PubSub.publish(changesPubSub, snapshotToPublish); + if (yield* isCommitCurrent) { + yield* PubSub.publish(changesPubSub, snapshotToPublish); + } }); const publishModels = (models: ServerProvider["models"]) => refreshSemaphore.withPermits(1)(publishModelsBase(models)); diff --git a/apps/server/src/provider/prime/PrimeAgentBackendSelection.test.ts b/apps/server/src/provider/prime/PrimeAgentBackendSelection.test.ts index 57fe75447..bf5408964 100644 --- a/apps/server/src/provider/prime/PrimeAgentBackendSelection.test.ts +++ b/apps/server/src/provider/prime/PrimeAgentBackendSelection.test.ts @@ -21,15 +21,31 @@ const testManager = (id: string) => ({ prepare: () => Effect.void, }); -const baseInput: PrimeAgentBackendNegotiationInput = { - enabled: true, - binaryPath: "prime-agent", - launchArgs: "", - settings: { agentHomePath: "" }, - environment: { PATH: "/configured/bin" }, +const makeInput = ( + settings: Partial = {}, + launchEnv: Readonly> = { PATH: "/configured/bin" }, +): PrimeAgentBackendNegotiationInput => ({ + identity: { + instanceId: ProviderInstanceId.make("primeAgent"), + generation: { _tag: "PrimeAgentRuntimeGeneration" }, + configRevision: "test-revision", + effectiveHome: "/prime/home", + launchEnv, + nativeMultipleInstancesRequired: false, + settings: { + enabled: true, + binaryPath: "prime-agent", + launchArgs: "", + agentHomePath: "/prime/home", + customModels: [], + ...settings, + }, + }, stateDir: "/pylon/state", - providerInstanceId: ProviderInstanceId.make("primeAgent"), -}; + platform: "linux", +}); + +const baseInput = makeInput(); it.layer(NodeServices.layer)("negotiatePrimeAgentBackend", (it) => { it.effect("selects daemon with a PATH-resolved Prime Agent executable", () => @@ -53,13 +69,13 @@ it.layer(NodeServices.layer)("negotiatePrimeAgentBackend", (it) => { const managerCalls: PrimeAgentDaemonManagerInput[] = []; const manager = testManager("path-resolved"); const selected = yield* negotiatePrimeAgentBackend( - { - ...baseInput, - environment: { + makeInput( + {}, + { PATH: tempDir, ...(platform === "win32" ? { PATHEXT: ".CMD" } : {}), }, - }, + ), { resolveExecutable: (command, environment) => resolveCommandPath(command, { env: environment }), @@ -108,7 +124,7 @@ it.layer(NodeServices.layer)("negotiatePrimeAgentBackend", (it) => { expect(selected).toEqual({ runtime: "daemon", manager }); expect(prepareAttempts).toBe(1); expect(resolutionCalls).toEqual([ - { command: "prime-agent", environment: baseInput.environment }, + { command: "prime-agent", environment: baseInput.identity.launchEnv }, ]); expect(managerCalls).toHaveLength(1); expect(NodePath.isAbsolute(managerCalls[0]?.executablePath ?? "")).toBe(true); @@ -121,7 +137,7 @@ it.layer(NodeServices.layer)("negotiatePrimeAgentBackend", (it) => { let resolutionAttempts = 0; let managerAttempts = 0; const selected = yield* negotiatePrimeAgentBackend( - { ...baseInput, launchArgs: " --verbose " }, + makeInput({ launchArgs: " --verbose " }), { resolveExecutable: () => Effect.sync(() => { @@ -143,10 +159,68 @@ it.layer(NodeServices.layer)("negotiatePrimeAgentBackend", (it) => { }), ); + it.effect("never selects ACP when a multiple-instance participant requires native", () => + Effect.gen(function* () { + let managerAttempts = 0; + const launchArgs = yield* negotiatePrimeAgentBackend( + { ...makeInput({ launchArgs: "--acp-only" }), requireNative: true }, + { + resolveExecutable: () => Effect.succeed("/unused/prime-agent"), + makeManager: () => + Effect.sync(() => { + managerAttempts += 1; + return testManager("unused"); + }), + }, + ); + const resolution = yield* negotiatePrimeAgentBackend( + { ...makeInput({ binaryPath: "/private/unavailable" }), requireNative: true }, + { + resolveExecutable: () => + Effect.fail({ _tag: "TestResolutionFailure" as const, detail: "private cause" }), + makeManager: () => Effect.succeed(testManager("unused")), + }, + ); + const setup = yield* negotiatePrimeAgentBackend( + { ...baseInput, requireNative: true }, + { + resolveExecutable: () => Effect.succeed("/private/resolved/prime-agent"), + makeManager: () => + Effect.succeed({ + prepare: () => + Effect.fail({ _tag: "TestPrepareFailure" as const, detail: "private cause" }), + }), + }, + ); + const windows = yield* negotiatePrimeAgentBackend( + { ...baseInput, platform: "win32", requireNative: true }, + { + resolveExecutable: () => Effect.succeed("/unused/prime-agent"), + makeManager: () => Effect.succeed(testManager("unused")), + }, + ); + + expect([launchArgs.runtime, resolution.runtime, setup.runtime, windows.runtime]).toEqual([ + "unavailable", + "unavailable", + "unavailable", + "unavailable", + ]); + expect(managerAttempts).toBe(0); + for (const selected of [launchArgs, resolution, setup, windows]) { + expect(selected.runtime).not.toBe("acp"); + const message = selected.runtime === "unavailable" ? selected.message : ""; + expect(message).toContain("native-only"); + expect(message).not.toContain("/private/"); + expect(message).not.toContain("private cause"); + } + }), + ); + it.effect("turns resolution failures into bounded ACP reasons without raw details", () => Effect.gen(function* () { const selected = yield* negotiatePrimeAgentBackend( - { ...baseInput, binaryPath: "/secret/configured/prime-agent" }, + makeInput({ binaryPath: "/secret/configured/prime-agent" }), { resolveExecutable: () => Effect.fail({ @@ -217,12 +291,12 @@ it.layer(NodeServices.layer)("negotiatePrimeAgentBackend", (it) => { }), ); - it.effect("quietly selects ACP when disabled and skips daemon setup", () => + it.effect("selects ACP on native Windows without resolving, importing, or starting Prime", () => Effect.gen(function* () { let resolutionAttempts = 0; let managerAttempts = 0; const selected = yield* negotiatePrimeAgentBackend( - { ...baseInput, enabled: false }, + { ...baseInput, platform: "win32" }, { resolveExecutable: () => Effect.sync(() => { @@ -237,6 +311,29 @@ it.layer(NodeServices.layer)("negotiatePrimeAgentBackend", (it) => { }, ); + expect(selected).toMatchObject({ runtime: "acp", fallbackCategory: "daemon-setup" }); + expect(resolutionAttempts).toBe(0); + expect(managerAttempts).toBe(0); + }), + ); + + it.effect("quietly selects ACP when disabled and skips daemon setup", () => + Effect.gen(function* () { + let resolutionAttempts = 0; + let managerAttempts = 0; + const selected = yield* negotiatePrimeAgentBackend(makeInput({ enabled: false }), { + resolveExecutable: () => + Effect.sync(() => { + resolutionAttempts += 1; + return "/unused/prime-agent"; + }), + makeManager: () => + Effect.sync(() => { + managerAttempts += 1; + return testManager("unused"); + }), + }); + expect(selected).toEqual({ runtime: "acp" }); expect(resolutionAttempts).toBe(0); expect(managerAttempts).toBe(0); diff --git a/apps/server/src/provider/prime/PrimeAgentBackendSelection.ts b/apps/server/src/provider/prime/PrimeAgentBackendSelection.ts index 60dc920c3..0fe549c4b 100644 --- a/apps/server/src/provider/prime/PrimeAgentBackendSelection.ts +++ b/apps/server/src/provider/prime/PrimeAgentBackendSelection.ts @@ -1,4 +1,5 @@ -import type { PrimeAgentSettings, ProviderInstanceId } from "@t3tools/contracts"; +import type { ProviderRuntimeFence } from "../ProviderDriver.ts"; +import type { PrimeAgentMaterializedIdentity } from "./PrimeAgentRuntimeContext.ts"; import * as Effect from "effect/Effect"; import * as Path from "effect/Path"; import * as Result from "effect/Result"; @@ -13,6 +14,15 @@ const FALLBACK_MESSAGES = { manager: "Prime Agent daemon integration is unavailable; using ACP compatibility mode.", } as const; +const NATIVE_ONLY_MESSAGES = { + launchArgs: + "Multiple Prime Agent instances are native-only. Remove custom launch arguments; ACP compatibility is disabled while more than one Prime instance is enabled.", + resolution: + "Multiple Prime Agent instances are native-only, but Pylon could not resolve this instance's Prime executable. Select an installed Pylon-managed Prime build or reduce the enabled set to one before using ACP compatibility.", + manager: + "Multiple Prime Agent instances are native-only, but this instance did not prove the required public SDK, private daemon, and current caller-owned session contract. Update or repair the Pylon-managed Prime build, or reduce the enabled set to one before using ACP compatibility.", +} as const; + export interface PrimeAgentBackendPreparable { readonly prepare: () => Effect.Effect; } @@ -23,18 +33,22 @@ export type PrimeAgentBackendSelection = readonly runtime: "acp"; readonly fallbackCategory?: "launch-args" | "binary-resolution" | "daemon-setup"; readonly fallbackMessage?: string; + } + | { + readonly runtime: "unavailable"; + readonly reason: "launch-args" | "binary-resolution" | "daemon-setup"; + readonly message: string; }; export interface PrimeAgentBackendNegotiationInput { - readonly enabled: boolean; - readonly binaryPath: string; - readonly launchArgs: string; - readonly settings: Pick; - readonly environment: NodeJS.ProcessEnv; + readonly identity: PrimeAgentMaterializedIdentity; + readonly runtimeFence?: ProviderRuntimeFence | undefined; readonly stateDir: string; - readonly providerInstanceId: ProviderInstanceId; + readonly platform: NodeJS.Platform; readonly recoveryEnabled?: boolean; readonly architecture?: string; + /** Fail closed instead of returning ACP for an enabled multi-instance participant. */ + readonly requireNative?: boolean; } export interface PrimeAgentBackendNegotiationDependencies< @@ -78,54 +92,95 @@ export function negotiatePrimeAgentBackend< Path.Path | ResolveServices | ManagerServices > { return Effect.gen(function* () { - if (!input.enabled) return { runtime: "acp" } as const; + const settings = input.identity.settings; + if (!settings.enabled) return { runtime: "acp" } as const; + if (input.platform === "win32") { + return input.requireNative === true + ? ({ + runtime: "unavailable", + reason: "daemon-setup", + message: NATIVE_ONLY_MESSAGES.manager, + } as const) + : ({ + runtime: "acp", + fallbackCategory: "daemon-setup", + fallbackMessage: FALLBACK_MESSAGES.manager, + } as const); + } - if (input.launchArgs.trim().length > 0) { - return { - runtime: "acp", - fallbackCategory: "launch-args", - fallbackMessage: FALLBACK_MESSAGES.launchArgs, - } as const; + if (settings.launchArgs.trim().length > 0) { + return input.requireNative === true + ? ({ + runtime: "unavailable", + reason: "launch-args", + message: NATIVE_ONLY_MESSAGES.launchArgs, + } as const) + : ({ + runtime: "acp", + fallbackCategory: "launch-args", + fallbackMessage: FALLBACK_MESSAGES.launchArgs, + } as const); } const path = yield* Path.Path; const resolution = yield* Effect.result( - dependencies.resolveExecutable(input.binaryPath || "prime-agent", input.environment), + dependencies.resolveExecutable( + settings.binaryPath || "prime-agent", + input.identity.launchEnv, + ), ); if (Result.isFailure(resolution)) { - return { - runtime: "acp", - fallbackCategory: "binary-resolution", - fallbackMessage: FALLBACK_MESSAGES.resolution, - } as const; + return input.requireNative === true + ? ({ + runtime: "unavailable", + reason: "binary-resolution", + message: NATIVE_ONLY_MESSAGES.resolution, + } as const) + : ({ + runtime: "acp", + fallbackCategory: "binary-resolution", + fallbackMessage: FALLBACK_MESSAGES.resolution, + } as const); } const manager = yield* Effect.result( dependencies.makeManager({ executablePath: path.resolve(resolution.success), - settings: input.settings, - environment: input.environment, + identity: input.identity, + ...(input.runtimeFence === undefined ? {} : { runtimeFence: input.runtimeFence }), stateDir: input.stateDir, - providerInstanceId: input.providerInstanceId, + platform: input.platform, ...(input.recoveryEnabled === undefined ? {} : { recoveryEnabled: input.recoveryEnabled }), ...(input.architecture === undefined ? {} : { architecture: input.architecture }), }), ); if (Result.isFailure(manager)) { - return { - runtime: "acp", - fallbackCategory: "daemon-setup", - fallbackMessage: FALLBACK_MESSAGES.manager, - } as const; + return input.requireNative === true + ? ({ + runtime: "unavailable", + reason: "daemon-setup", + message: NATIVE_ONLY_MESSAGES.manager, + } as const) + : ({ + runtime: "acp", + fallbackCategory: "daemon-setup", + fallbackMessage: FALLBACK_MESSAGES.manager, + } as const); } const prepared = yield* Effect.result(manager.success.prepare()); if (Result.isFailure(prepared)) { - return { - runtime: "acp", - fallbackCategory: "daemon-setup", - fallbackMessage: FALLBACK_MESSAGES.manager, - } as const; + return input.requireNative === true + ? ({ + runtime: "unavailable", + reason: "daemon-setup", + message: NATIVE_ONLY_MESSAGES.manager, + } as const) + : ({ + runtime: "acp", + fallbackCategory: "daemon-setup", + fallbackMessage: FALLBACK_MESSAGES.manager, + } as const); } return { runtime: "daemon", manager: manager.success } as const; diff --git a/apps/server/src/provider/prime/PrimeAgentDaemonAdapter.test.ts b/apps/server/src/provider/prime/PrimeAgentDaemonAdapter.test.ts index 6044249fa..3de3b1eb0 100644 --- a/apps/server/src/provider/prime/PrimeAgentDaemonAdapter.test.ts +++ b/apps/server/src/provider/prime/PrimeAgentDaemonAdapter.test.ts @@ -2298,6 +2298,49 @@ describe("PrimeAgentDaemonAdapter", () => { }), ).pipe(Effect.provide(testLayer)), ); + it.effect("rejects a thread MCP route owned by another provider instance", () => + Effect.scoped( + Effect.gen(function* () { + const captures = makeCaptures(); + const mismatchedThread = ThreadId.make("prime-daemon-mcp-mismatch"); + yield* Effect.acquireRelease( + Effect.sync(() => + McpProviderSession.setMcpProviderSession({ + providerSessionId: "provider-session-mismatch", + threadId: mismatchedThread, + environmentId: EnvironmentId.make("environment-mismatch"), + providerInstanceId: ProviderInstanceId.make("prime-daemon-other"), + endpoint: "http://127.0.0.1:4321/mcp/mismatch", + authorizationHeader: "Bearer must-not-route", + }), + ), + () => Effect.sync(() => McpProviderSession.clearMcpProviderSession(mismatchedThread)), + ); + const adapter = yield* makePrimeAgentDaemonAdapter(decodeSettings({}), manager, { + instanceId, + runtimeFactory: fakeRuntimeFactory(captures), + }); + + const result = yield* adapter + .startSession({ + threadId: mismatchedThread, + cwd: process.cwd(), + runtimeMode: "full-access", + }) + .pipe(Effect.result); + + expect(result).toMatchObject({ + _tag: "Failure", + failure: { + _tag: "ProviderAdapterValidationError", + issue: "The MCP route does not belong to this provider instance.", + }, + }); + expect(captures.runtimeInputs).toEqual([]); + }), + ).pipe(Effect.provide(testLayer)), + ); + it.effect("passes the thread-scoped Pylon MCP server into the daemon runtime", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/provider/prime/PrimeAgentDaemonAdapter.ts b/apps/server/src/provider/prime/PrimeAgentDaemonAdapter.ts index a98a9d990..525d4e6fd 100644 --- a/apps/server/src/provider/prime/PrimeAgentDaemonAdapter.ts +++ b/apps/server/src/provider/prime/PrimeAgentDaemonAdapter.ts @@ -61,6 +61,7 @@ import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import * as SynchronizedRef from "effect/SynchronizedRef"; +import { writeFileStringAtomically } from "../../atomicWrite.ts"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { ServerConfig } from "../../config.ts"; import * as McpProviderSession from "../../mcp/McpProviderSession.ts"; @@ -93,9 +94,11 @@ import { type PrimeDaemonUsage, } from "./PrimeAgentDaemonEvents.ts"; import type { PrimeAgentDaemonManager } from "./PrimeAgentDaemonManager.ts"; +import type { PrimeAgentRuntimeContext } from "./PrimeAgentRuntimeContext.ts"; import { PRIME_AGENT_RECOVERY_ADOPTION_MAX_ATTEMPTS, PrimeAgentRecoveryLedger, + primeAgentRecoveryOwnershipIdentities, type PrimeAgentRecoveryAdoptionProof, type PrimeAgentRecoveryAuthority, type PrimeAgentRecoveryLedgerShape, @@ -134,9 +137,12 @@ import { type PrimeAgentDaemonSessionRuntimeError, type PrimeAgentDaemonSessionRuntimeInput, } from "./PrimeAgentDaemonSessionRuntime.ts"; +import type { + PrimeAgentAcquiredOwnershipReceipt, + PrimeAgentOwnershipAdoptionClaim, +} from "./PrimeAgentOwnershipReceipt.ts"; import { PRIME_AGENT_SESSION_IDENTITY_FILENAME, - PRIME_AGENT_SESSION_IDENTITY_TEMP_FILENAME, decodePrimeAgentSessionIdentity, encodePrimeAgentSessionIdentity, primeAgentLegacySessionFileNames, @@ -156,7 +162,8 @@ const PRIME_AGENT_RECOVERY_TEST_CRASH_STAGE = "PRIME_AGENT_INTERNAL_PYLON_RECOVE type PrimeAgentRecoveryTestCrashStage = | "after-claim-persisted" | "after-native-response-before-commit" - | "after-commit-before-confirm"; + | "after-commit-before-receipt" + | "after-receipt-before-confirm"; function crashAtPrimeAgentRecoveryTestBarrier(stage: PrimeAgentRecoveryTestCrashStage): void { if (process.env[PRIME_AGENT_RECOVERY_TEST_CRASH_STAGE] === stage) { @@ -180,6 +187,7 @@ export interface PrimeAgentDaemonAdapterLiveOptions { readonly nativeEventLogPath?: string; readonly nativeEventLogger?: EventNdjsonLogger; readonly instanceId?: ProviderInstanceId; + readonly runtimeContext?: PrimeAgentRuntimeContext; /** Present only after the exact selected package passed Pylon managed-distribution proof. */ readonly recoveryManagedBuildId?: string; readonly recoveryLedger?: PrimeAgentRecoveryLedgerShape; @@ -794,9 +802,34 @@ export function makePrimeAgentDaemonAdapter( ) { return Effect.gen(function* () { const boundInstanceId = options?.instanceId ?? ProviderInstanceId.make("primeAgent"); + const primeRuntimeContext = options?.runtimeContext; + if ( + primeRuntimeContext !== undefined && + (primeRuntimeContext.backendKind !== "daemon" || + primeRuntimeContext.instanceId !== boundInstanceId || + primeRuntimeContext.instanceId !== manager.identity.instanceId || + primeRuntimeContext.configRevision !== manager.identity.configRevision || + primeRuntimeContext.effectiveHome !== manager.identity.effectiveHome || + primeRuntimeContext.launchEnv !== manager.identity.launchEnv) + ) { + return yield* Effect.die( + new Error("The Prime Agent runtime context does not own this daemon adapter."), + ); + } const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const serverConfig = yield* ServerConfig; + const commitGuard = primeRuntimeContext?.runtimeFence?.isCurrent ?? Effect.succeed(true); + const guardGeneration = (effect: Effect.Effect, stale: A) => + Effect.flatMap(commitGuard, (current) => + current + ? effect.pipe( + Effect.flatMap((result) => + Effect.map(commitGuard, (stillCurrent) => (stillCurrent ? result : stale)), + ), + ) + : Effect.succeed(stale), + ); const ledgerService = yield* Effect.serviceOption(PrimeAgentRecoveryLedger); const rawRecoveryLedger = options?.recoveryLedger ?? Option.getOrUndefined(ledgerService); const recoveryLedger = @@ -804,43 +837,99 @@ export function makePrimeAgentDaemonAdapter( ? undefined : { putPrepared: (input: Parameters[0]) => - rawRecoveryLedger.putPrepared(input).pipe(Effect.orDie), - get: (threadId: string) => rawRecoveryLedger.get(threadId).pipe(Effect.orDie), + guardGeneration( + rawRecoveryLedger.putPrepared(input, { commitGuard }).pipe(Effect.orDie), + undefined, + ), + get: (threadId: string) => + guardGeneration(rawRecoveryLedger.get(threadId).pipe(Effect.orDie), Option.none()), discardPrepared: ( input: Parameters[0], - ) => rawRecoveryLedger.discardPrepared(input).pipe(Effect.orDie), + ) => + guardGeneration( + rawRecoveryLedger.discardPrepared(input, { commitGuard }).pipe(Effect.orDie), + false, + ), markAdmitted: (input: Parameters[0]) => - rawRecoveryLedger.markAdmitted(input).pipe(Effect.orDie), + guardGeneration( + rawRecoveryLedger.markAdmitted(input, { commitGuard }).pipe(Effect.orDie), + false, + ), updateTranscriptProgress: ( input: Parameters[0], - ) => rawRecoveryLedger.updateTranscriptProgress(input).pipe(Effect.orDie), + ) => + guardGeneration( + rawRecoveryLedger + .updateTranscriptProgress(input, { commitGuard }) + .pipe(Effect.orDie), + false, + ), claim: (input: Parameters[0]) => - rawRecoveryLedger.claim(input).pipe(Effect.orDie), + guardGeneration( + rawRecoveryLedger.claim(input, { commitGuard }).pipe(Effect.orDie), + Option.none(), + ), beginAdoptionAttempt: ( input: Parameters[0], - ) => rawRecoveryLedger.beginAdoptionAttempt(input).pipe(Effect.orDie), + ) => + guardGeneration( + rawRecoveryLedger.beginAdoptionAttempt(input, { commitGuard }).pipe(Effect.orDie), + Option.none(), + ), releaseClaim: (input: Parameters[0]) => - rawRecoveryLedger.releaseClaim(input).pipe(Effect.orDie), + guardGeneration( + rawRecoveryLedger.releaseClaim(input, { commitGuard }).pipe(Effect.orDie), + false, + ), commitAdoption: ( input: Parameters[0], - ) => rawRecoveryLedger.commitAdoption(input).pipe(Effect.orDie), + ) => + guardGeneration( + rawRecoveryLedger.commitAdoption(input, { commitGuard }).pipe(Effect.orDie), + false, + ), beginAdoptionConfirmation: ( input: Parameters[0], - ) => rawRecoveryLedger.beginAdoptionConfirmation(input).pipe(Effect.orDie), + ) => + guardGeneration( + rawRecoveryLedger + .beginAdoptionConfirmation(input, { commitGuard }) + .pipe(Effect.orDie), + Option.none(), + ), finalizeAdoption: ( input: Parameters[0], - ) => rawRecoveryLedger.finalizeAdoption(input).pipe(Effect.orDie), + ) => + guardGeneration( + rawRecoveryLedger.finalizeAdoption(input, { commitGuard }).pipe(Effect.orDie), + false, + ), quarantineAdoption: ( input: Parameters[0], - ) => rawRecoveryLedger.quarantineAdoption(input).pipe(Effect.orDie), + ) => + guardGeneration( + rawRecoveryLedger.quarantineAdoption(input, { commitGuard }).pipe(Effect.orDie), + false, + ), markNativeCleanup: ( input: Parameters[0], - ) => rawRecoveryLedger.markNativeCleanup(input).pipe(Effect.orDie), + ) => + guardGeneration( + rawRecoveryLedger.markNativeCleanup(input, { commitGuard }).pipe(Effect.orDie), + false, + ), markTerminalProjected: ( input: Parameters[0], - ) => rawRecoveryLedger.markTerminalProjected(input).pipe(Effect.orDie), + ) => + guardGeneration( + rawRecoveryLedger.markTerminalProjected(input, { commitGuard }).pipe(Effect.orDie), + undefined, + ), deleteIfSettled: (threadId: string) => - rawRecoveryLedger.deleteIfSettled(threadId).pipe(Effect.orDie), + guardGeneration( + rawRecoveryLedger.deleteIfSettled(threadId, { commitGuard }).pipe(Effect.orDie), + false, + ), }; const crypto = yield* Crypto.Crypto; const runtimeContext = yield* Effect.context(); @@ -856,6 +945,12 @@ export function makePrimeAgentDaemonAdapter( // The already-created manager owns its launch environment. Retaining this option keeps the // daemon adapter's construction boundary compatible with the other Prime adapter. void options?.environment; + const mcpSessionMatchesGeneration = (threadId: ThreadId): boolean => + primeRuntimeContext?.runtimeFence === undefined || + McpProviderSession.isMcpProviderSessionOwnedByGeneration( + threadId, + primeRuntimeContext.runtimeFence, + ); const sessions = new Map(); type PendingRecoveryStart = @@ -876,6 +971,8 @@ export function makePrimeAgentDaemonAdapter( readonly requestId: string; readonly mcpOwnerId: string; readonly sessionFile: string; + readonly ownershipClaim: PrimeAgentOwnershipAdoptionClaim; + readonly previousOwnershipReceipt: PrimeAgentAcquiredOwnershipReceipt; }; const pendingRecoveryStarts = new Map(); const activeTeardowns = new Map< @@ -930,22 +1027,28 @@ export function makePrimeAgentDaemonAdapter( }); yield* Queue.take(orderedRuntimeEventQueue).pipe( Effect.flatMap(({ event, terminalDelivery }) => - PubSub.publish(runtimeEventPubSub, event).pipe( - Effect.flatMap((accepted) => - Effect.sync(() => { - pendingOrderedRuntimeEvents -= 1; - }).pipe( - Effect.andThen(completeTerminalDelivery(terminalDelivery, accepted)), - Effect.andThen(accepted ? Effect.void : logRejectedRuntimeEvent(event)), - ), - ), + Effect.flatMap(commitGuard, (current) => + current + ? PubSub.publish(runtimeEventPubSub, event).pipe( + Effect.flatMap((accepted) => + Effect.sync(() => { + pendingOrderedRuntimeEvents -= 1; + }).pipe( + Effect.andThen(completeTerminalDelivery(terminalDelivery, accepted)), + Effect.andThen(accepted ? Effect.void : logRejectedRuntimeEvent(event)), + ), + ), + ) + : Effect.sync(() => { + pendingOrderedRuntimeEvents -= 1; + }).pipe(Effect.andThen(completeTerminalDelivery(terminalDelivery, false))), ), ), Effect.forever, Effect.forkScoped, ); - const offerOrderedRuntimeEvent = (entry: OrderedRuntimeEvent) => + const offerOrderedRuntimeEventUnchecked = (entry: OrderedRuntimeEvent) => Effect.sync(() => { if ( pendingOrderedRuntimeEvents === 0 && @@ -972,6 +1075,12 @@ export function makePrimeAgentDaemonAdapter( ), ), ); + const offerOrderedRuntimeEvent = (entry: OrderedRuntimeEvent) => + Effect.flatMap(commitGuard, (current) => + current + ? offerOrderedRuntimeEventUnchecked(entry) + : completeTerminalDelivery(entry.terminalDelivery, false).pipe(Effect.as(false)), + ); const nowIso = Effect.map(DateTime.now, DateTime.formatIso); const randomUUIDv4 = crypto.randomUUIDv4.pipe( @@ -985,6 +1094,18 @@ export function makePrimeAgentDaemonAdapter( }), ), ); + const requireCurrentGeneration = (operation: string) => + Effect.flatMap(commitGuard, (current) => + current + ? Effect.void + : Effect.fail( + new ProviderAdapterRequestError({ + provider: PROVIDER, + method: operation, + detail: "The Prime Agent runtime was replaced while this operation was pending.", + }), + ), + ); const randomAdoptionRequestId = crypto.randomBytes(24).pipe( Effect.map((bytes) => Buffer.from(bytes).toString("hex")), Effect.mapError( @@ -1044,6 +1165,37 @@ export function makePrimeAgentDaemonAdapter( Effect.asVoid, ); }); + const runBoundedTeardownBarrier = Effect.fn( + "PrimeAgentDaemonAdapter.runBoundedTeardownBarrier", + )(function* (input: { + readonly threadId: ThreadId; + readonly phase: string; + readonly effect: Effect.Effect; + }) { + const barrierFiber = yield* input.effect.pipe(Effect.interruptible, Effect.forkDetach); + const settled = yield* Fiber.await(barrierFiber).pipe( + Effect.timeoutOption(Duration.millis(PRIME_AGENT_SESSION_TEARDOWN_TIMEOUT_MS)), + ); + if (Option.isNone(settled)) { + yield* Effect.logWarning("Prime Agent daemon teardown barrier timed out.", { + threadId: input.threadId, + phase: input.phase, + timeoutMs: PRIME_AGENT_SESSION_TEARDOWN_TIMEOUT_MS, + }); + yield* Fiber.interrupt(barrierFiber).pipe(Effect.forkDetach); + return false; + } + if (Exit.isFailure(settled.value)) { + yield* Effect.logError("Prime Agent daemon teardown barrier failed.", { + threadId: input.threadId, + phase: input.phase, + cause: Cause.pretty(settled.value.cause), + }); + return false; + } + return true; + }); + const withThreadLock = (threadId: ThreadId, effect: Effect.Effect) => Effect.flatMap(getThreadSemaphore(threadId), (semaphore) => semaphore.withPermit(effect).pipe(Effect.ensuring(drainActiveTeardown(threadId))), @@ -1136,6 +1288,7 @@ export function makePrimeAgentDaemonAdapter( }, }, threadId, + commitGuard, ); }).pipe( Effect.catchCause((cause) => @@ -3609,16 +3762,11 @@ export function makePrimeAgentDaemonAdapter( }); return Effect.gen(function* () { - yield* Scope.close(context.scope, Exit.void).pipe( - Effect.timeoutOption(Duration.millis(PRIME_AGENT_SESSION_TEARDOWN_TIMEOUT_MS)), - Effect.catchCause((cause) => - Effect.logError("Prime Agent daemon session scope cleanup failed.", { - threadId: context.threadId, - cause: Cause.pretty(cause), - }).pipe(Effect.as(Option.none())), - ), - Effect.forkDetach, - ); + yield* runBoundedTeardownBarrier({ + threadId: context.threadId, + phase: "session-scope-close", + effect: Scope.close(context.scope, Exit.void), + }); const workFiber = yield* teardownWork.pipe(Effect.forkDetach); const workResult = yield* Fiber.join(workFiber).pipe( Effect.exit, @@ -3636,7 +3784,10 @@ export function makePrimeAgentDaemonAdapter( cause: Cause.pretty(workResult.value.cause), }); } - yield* publishTerminalOnce; + yield* publishTerminalOnce.pipe( + Effect.timeoutOption(Duration.millis(PRIME_AGENT_TERMINAL_EVENT_TIMEOUT_MS)), + Effect.asVoid, + ); }).pipe( Effect.ensuring( Deferred.succeed(context.teardownCompletion, undefined).pipe( @@ -3814,6 +3965,7 @@ export function makePrimeAgentDaemonAdapter( ? input.modelSelection.model.trim() : undefined; const model = selectedModel || "default"; + yield* requireCurrentGeneration("startSession"); const sessionDir = primeAgentSessionDirectory({ stateDir: serverConfig.stateDir, instanceId: boundInstanceId, @@ -3832,6 +3984,7 @@ export function makePrimeAgentDaemonAdapter( }), ), ); + yield* requireCurrentGeneration("startSession"); const identityPath = path.join(sessionDir, PRIME_AGENT_SESSION_IDENTITY_FILENAME); let resumeSessionId: string | undefined; let expectedSessionFileName: string | undefined; @@ -3923,7 +4076,14 @@ export function makePrimeAgentDaemonAdapter( rootSessionDir: sessionDir, ...(permissionToken === undefined ? {} : { permissionToken }), }); - yield* fileSystem.writeFileString(managedExtensionPath, managedExtensionSource).pipe( + yield* writeFileStringAtomically({ + filePath: managedExtensionPath, + contents: managedExtensionSource, + commitGuard, + }).pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, path), + Effect.andThen(requireCurrentGeneration("startSession")), Effect.andThen(fileSystem.chmod(managedExtensionPath, 0o600)), Effect.mapError( (cause) => @@ -3941,10 +4101,26 @@ export function makePrimeAgentDaemonAdapter( yield* Effect.addFinalizer(() => scopeTransferred ? Effect.void : Scope.close(sessionScope, Exit.void), ); - const agentDir = primeAgentSettings.agentHomePath.trim(); + const agentDir = + primeRuntimeContext?.effectiveHome ?? primeAgentSettings.agentHomePath.trim(); const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); + if ( + mcpSession !== undefined && + (mcpSession.providerInstanceId !== boundInstanceId || + !mcpSessionMatchesGeneration(input.threadId)) + ) { + return yield* new ProviderAdapterValidationError({ + provider: PROVIDER, + operation: "startSession", + issue: "The MCP route does not belong to this provider instance.", + }); + } const runtime = yield* runtimeFactory({ manager, + ...(primeRuntimeContext === undefined ? {} : { runtimeContext: primeRuntimeContext }), + ...(recoveryStart?.kind === "adopt" + ? { ownershipReceipt: recoveryStart.ownershipClaim.handle } + : {}), cwd, sessionDir, ...(mcpSession === undefined @@ -3994,6 +4170,9 @@ export function makePrimeAgentDaemonAdapter( recovery: { kind: "create" as const, requestId: yield* randomUUIDv4, + threadId: input.threadId, + sessionIncarnationId: input.sessionIncarnationId!, + admissionRequestId: recoveryStart.admissionRequestId, correlationId: recoveryStart.correlationId, mcpOwnerId: recoveryStart.mcpOwnerId, onAuthorityReady: (authority) => @@ -4031,7 +4210,8 @@ export function makePrimeAgentDaemonAdapter( correlationId: recoveryStart.correlationId, mcpOwnerId: recoveryStart.mcpOwnerId, recoveryConfig: authority.recoveryConfig, - launchEnvironment: authority.launchEnvironment, + launchEnvironment: + primeRuntimeContext?.launchEnv ?? authority.launchEnvironment, transcriptMessageCount: recoveryStart.transcriptMessageCount, transcriptFingerprints: [...recoveryStart.transcriptFingerprints], ownerToken: recoveryStart.ownerToken, @@ -4067,7 +4247,8 @@ export function makePrimeAgentDaemonAdapter( previousMcpOwnerId: recoveryStart.authority.mcpOwnerId, mcpOwnerId: recoveryStart.mcpOwnerId, recoveryConfig: recoveryStart.authority.recoveryConfig, - launchEnvironment: recoveryStart.authority.launchEnvironment, + launchEnvironment: + primeRuntimeContext?.launchEnv ?? recoveryStart.authority.launchEnvironment, onAdoptionAttemptStarted: () => runPromise( Effect.gen(function* () { @@ -4136,7 +4317,44 @@ export function makePrimeAgentDaemonAdapter( detail: "Recoverable Prime Agent ownership was superseded.", }); } - crashAtPrimeAgentRecoveryTestBarrier("after-commit-before-confirm"); + crashAtPrimeAgentRecoveryTestBarrier("after-commit-before-receipt"); + const ownershipStore = primeRuntimeContext?.nativeOwnership?.store; + if (ownershipStore === undefined) { + return yield* new ProviderAdapterProcessError({ + provider: PROVIDER, + threadId: input.threadId, + detail: "Prime Agent native ownership storage is unavailable.", + }); + } + const rotated = yield* Effect.tryPromise({ + try: () => + ownershipStore.rotateAdoptionRecovery( + recoveryStart.ownershipClaim, + { + threadId: recoveryStart.authority.threadId, + sessionIncarnationId: + recoveryStart.authority.sessionIncarnationId, + admissionRequestId: recoveryStart.authority.admissionRequestId, + recoveryHandle, + ownershipGeneration: proof.ownershipGeneration, + }, + ), + catch: (cause) => + new ProviderAdapterProcessError({ + provider: PROVIDER, + threadId: input.threadId, + detail: "Could not durably rotate Prime Agent native ownership.", + cause, + }), + }); + if (rotated === undefined) { + return yield* new ProviderAdapterProcessError({ + provider: PROVIDER, + threadId: input.threadId, + detail: "Prime Agent native ownership rotation was superseded.", + }); + } + crashAtPrimeAgentRecoveryTestBarrier("after-receipt-before-confirm"); }), ), onAdoptionConfirming: ({ recoveryHandle, proof }) => @@ -4312,13 +4530,15 @@ export function makePrimeAgentDaemonAdapter( detail: "Prime Agent returned an invalid durable session identity.", }); } - const identityTempPath = path.join( - sessionDir, - PRIME_AGENT_SESSION_IDENTITY_TEMP_FILENAME, - ); - yield* fileSystem.writeFileString(identityTempPath, identitySource).pipe( - Effect.andThen(fileSystem.chmod(identityTempPath, 0o600)), - Effect.andThen(fileSystem.rename(identityTempPath, identityPath)), + yield* writeFileStringAtomically({ + filePath: identityPath, + contents: identitySource, + commitGuard, + }).pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, path), + Effect.andThen(requireCurrentGeneration("startSession")), + Effect.andThen(fileSystem.chmod(identityPath, 0o600)), Effect.mapError( (cause) => new ProviderAdapterProcessError({ @@ -4502,6 +4722,7 @@ export function makePrimeAgentDaemonAdapter( ...context.compaction, manualCompactionSettable: isManualCompactionSettable(context), }; + yield* requireCurrentGeneration("startSession"); sessions.set(input.threadId, context); scopeTransferred = true; if (recoveryStart === undefined) { @@ -4673,8 +4894,16 @@ export function makePrimeAgentDaemonAdapter( if (admissionRequestId === undefined || admissionRequestId.length === 0) { return undefined; } + const candidateMcpSession = McpProviderSession.readMcpProviderSession(input.threadId); + if ( + candidateMcpSession !== undefined && + (candidateMcpSession.providerInstanceId !== boundInstanceId || + !mcpSessionMatchesGeneration(input.threadId)) + ) { + return undefined; + } const ownerToken = yield* randomUUIDv4; - const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); + const mcpSession = candidateMcpSession; const recoveryStart: PendingRecoveryStart = { kind: "create", admissionRequestId, @@ -4817,6 +5046,86 @@ export function makePrimeAgentDaemonAdapter( return null; } + const candidateMcpSession = McpProviderSession.readMcpProviderSession(input.threadId); + if ( + candidateMcpSession !== undefined && + (candidateMcpSession.providerInstanceId !== boundInstanceId || + !mcpSessionMatchesGeneration(input.threadId)) + ) { + return null; + } + const ownershipAuthority = authority; + const nativeOwnership = primeRuntimeContext?.nativeOwnership; + const adoptionConfigRevision = primeRuntimeContext?.configRevision; + const adoptionEffectiveHome = primeRuntimeContext?.effectiveHome; + const adoptionReceipt = nativeOwnership?.adoptableReceipts.find( + (receipt) => + receipt.instanceId === ownershipAuthority.providerInstanceId && + receipt.effectiveHome === adoptionEffectiveHome && + receipt.activeSessionId === ownershipAuthority.activeSessionId && + receipt.nativeSessionId === ownershipAuthority.nativeSessionId && + receipt.recovery !== undefined && + primeAgentRecoveryOwnershipIdentities(ownershipAuthority).some( + (accepted) => + receipt.recovery?.threadId === accepted.threadId && + receipt.recovery.sessionIncarnationId === accepted.sessionIncarnationId && + receipt.recovery.admissionRequestId === accepted.admissionRequestId && + receipt.recovery.recoveryHandle === accepted.recoveryHandle && + receipt.recovery.ownershipGeneration === accepted.ownershipGeneration, + ), + ); + if ( + nativeOwnership === undefined || + adoptionConfigRevision === undefined || + adoptionReceipt === undefined || + adoptionReceipt.recovery === undefined + ) { + return null; + } + const claimOwnershipReceipt = () => + Effect.tryPromise({ + try: () => + nativeOwnership.store.claimForAdoption({ + receipt: adoptionReceipt, + nextConfigRevision: adoptionConfigRevision, + recovery: adoptionReceipt.recovery!, + }), + catch: (cause) => + new ProviderAdapterProcessError({ + provider: PROVIDER, + threadId: input.threadId, + detail: "Could not durably claim Prime Agent native ownership.", + cause, + }), + }); + const rotateOwnershipReceipt = ( + claim: PrimeAgentOwnershipAdoptionClaim, + recoveryHandle: string, + proof: PrimeAgentRecoveryAdoptionProof, + ) => + Effect.tryPromise({ + try: () => + nativeOwnership.store.rotateAdoptionRecovery(claim, { + threadId: ownershipAuthority.threadId, + sessionIncarnationId: ownershipAuthority.sessionIncarnationId, + admissionRequestId: ownershipAuthority.admissionRequestId, + recoveryHandle, + ownershipGeneration: proof.ownershipGeneration, + }), + catch: (cause) => + new ProviderAdapterProcessError({ + provider: PROVIDER, + threadId: input.threadId, + detail: "Could not durably rotate Prime Agent native ownership.", + cause, + }), + }); + let ownershipClaim: PrimeAgentOwnershipAdoptionClaim | undefined; + if (authority.state === "adopting") { + ownershipClaim = yield* claimOwnershipReceipt(); + if (ownershipClaim === undefined) return null; + } + if ( authority.state === "adopting" && (authority.adoptionPhase === "committed" || authority.adoptionPhase === "confirming") @@ -4828,6 +5137,7 @@ export function makePrimeAgentDaemonAdapter( manager.bridge.confirmRecoverableOwnedSessionAdoption; if ( route === undefined || + ownershipClaim === undefined || proof === null || recoveryHandle === null || confirmRecoverableOwnedSessionAdoption === undefined || @@ -4839,6 +5149,14 @@ export function makePrimeAgentDaemonAdapter( ) { return null; } + const rotatedOwnershipClaim = yield* rotateOwnershipReceipt( + ownershipClaim, + recoveryHandle, + proof, + ); + if (rotatedOwnershipClaim === undefined) return null; + ownershipClaim = rotatedOwnershipClaim; + crashAtPrimeAgentRecoveryTestBarrier("after-receipt-before-confirm"); const confirming = yield* recoveryLedger!.beginAdoptionConfirmation({ threadId: input.threadId, ownerToken: route.ownerToken, @@ -4912,11 +5230,10 @@ export function makePrimeAgentDaemonAdapter( if (authority.state === "active") { const ownerToken = yield* randomUUIDv4; const requestId = yield* randomAdoptionRequestId; - const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); const mcpOwnerId = - mcpSession === undefined + candidateMcpSession === undefined ? `pylon:none:${yield* randomUUIDv4}` - : `pylon:${mcpSession.providerSessionId}`; + : `pylon:${candidateMcpSession.providerSessionId}`; const claimed = yield* recoveryLedger!.claim({ threadId: input.threadId, expectedOwnerToken: authority.ownerToken, @@ -4929,11 +5246,25 @@ export function makePrimeAgentDaemonAdapter( authority = claimed.value; route = completePrimeAgentAdoptionRoute(authority); crashAtPrimeAgentRecoveryTestBarrier("after-claim-persisted"); + if (ownershipClaim === undefined) { + ownershipClaim = yield* claimOwnershipReceipt(); + if (ownershipClaim === undefined) { + yield* recoveryLedger!.releaseClaim({ + threadId: input.threadId, + ownerToken, + previousOwnerToken: authority.ownerToken, + requestId, + updatedAt: yield* nowIso, + }); + return null; + } + } } else { route = completePrimeAgentAdoptionRoute(authority); } if ( route === undefined || + ownershipClaim === undefined || authority.adoptionAttempt >= PRIME_AGENT_RECOVERY_ADOPTION_MAX_ATTEMPTS ) { if (route !== undefined) { @@ -4973,6 +5304,8 @@ export function makePrimeAgentDaemonAdapter( requestId: route.requestId, mcpOwnerId: route.mcpOwnerId, sessionFile: path.join(recoverySessionDir, `${authority.nativeSessionId}.jsonl`), + ownershipClaim, + previousOwnershipReceipt: adoptionReceipt, }); const started = yield* Effect.result( startSession({ @@ -4996,6 +5329,10 @@ export function makePrimeAgentDaemonAdapter( current.adoptionPhase === "claimed" && current.adoptionAttempt === 0 ) { + const releasedOwnership = yield* Effect.tryPromise(() => + nativeOwnership.store.releaseAdoptionClaim(ownershipClaim, adoptionReceipt), + ).pipe(Effect.orElseSucceed(() => false)); + if (!releasedOwnership) return null; yield* recoveryLedger!.releaseClaim({ threadId: input.threadId, ownerToken: route.ownerToken, @@ -7377,15 +7714,29 @@ export function makePrimeAgentDaemonAdapter( if (context.recoveryOwnerToken !== undefined) { context.stopped = true; sessions.delete(context.threadId); - if (context.eventFiber !== undefined) yield* Fiber.interrupt(context.eventFiber); + if (context.eventFiber !== undefined) { + yield* runBoundedTeardownBarrier({ + threadId: context.threadId, + phase: "recovery-event-fiber-interrupt", + effect: Fiber.interrupt(context.eventFiber).pipe(Effect.asVoid), + }); + } context.backgroundQuiescenceController?.abort(); context.backgroundQuiescenceController = undefined; - yield* (context.runtime.detach ?? context.runtime.dispose).pipe( - Effect.mapError((error) => - runtimeOperationError(context.threadId, "shutdown", error), + yield* runBoundedTeardownBarrier({ + threadId: context.threadId, + phase: "recovery-runtime-detach", + effect: (context.runtime.detach ?? context.runtime.dispose).pipe( + Effect.mapError((error) => + runtimeOperationError(context.threadId, "shutdown", error), + ), ), - ); - yield* Scope.close(context.scope, Exit.void).pipe(Effect.ignore); + }); + yield* runBoundedTeardownBarrier({ + threadId: context.threadId, + phase: "recovery-session-scope-close", + effect: Scope.close(context.scope, Exit.void), + }); return undefined; } return yield* stopSessionInternal(context); @@ -7393,17 +7744,27 @@ export function makePrimeAgentDaemonAdapter( ), { concurrency: "unbounded" }, ); - yield* Effect.forEach( - ordinaryCompletions.filter((completion) => completion !== undefined), - Deferred.await, - { concurrency: "unbounded", discard: true }, - ); + const ordinary = ordinaryCompletions.filter((completion) => completion !== undefined); + const drained = yield* Effect.forEach(ordinary, Deferred.await, { + concurrency: "unbounded", + discard: true, + }).pipe(Effect.timeoutOption(Duration.millis(PRIME_AGENT_SESSION_TEARDOWN_TIMEOUT_MS))); + if (Option.isNone(drained) && ordinary.length > 0) { + yield* Effect.logWarning("Prime Agent shutdown reached its session teardown bound.", { + sessionCount: ordinary.length, + timeoutMs: PRIME_AGENT_SESSION_TEARDOWN_TIMEOUT_MS, + }); + } }); yield* Effect.addFinalizer(() => shutdownPrimeAgentEventPubSub({ component: "daemon", pubSub: runtimeEventPubSub, + // Recovery shutdown has three ordered bounded phases: event-fiber stop, + // owner detach, then scope close. Keep the outer event drain long enough + // for all phases while retaining one fixed total bound. + timeoutMs: PRIME_AGENT_SESSION_TEARDOWN_TIMEOUT_MS * 4, drain: shutdown().pipe( Effect.andThen( Effect.suspend(() => @@ -7416,7 +7777,13 @@ export function makePrimeAgentDaemonAdapter( ), }).pipe( Effect.ensuring(Queue.shutdown(orderedRuntimeEventQueue)), - Effect.ensuring(managedNativeEventLogger?.close() ?? Effect.void), + Effect.ensuring( + (managedNativeEventLogger?.close() ?? Effect.void).pipe( + Effect.interruptible, + Effect.timeoutOption(Duration.millis(PRIME_AGENT_SESSION_TEARDOWN_TIMEOUT_MS)), + Effect.asVoid, + ), + ), ), ); diff --git a/apps/server/src/provider/prime/PrimeAgentDaemonBridge.test.ts b/apps/server/src/provider/prime/PrimeAgentDaemonBridge.test.ts index 35af06fbe..d9892c754 100644 --- a/apps/server/src/provider/prime/PrimeAgentDaemonBridge.test.ts +++ b/apps/server/src/provider/prime/PrimeAgentDaemonBridge.test.ts @@ -170,6 +170,17 @@ describe("PrimeAgentDaemonBridge", () => { }), ); + it.effect("loads the exact scoped Prime coding-agent package used for integration proof", () => + Effect.gen(function* () { + const pkg = makePackage({ name: "@earendil-works/pi-coding-agent" }); + + const bridge = yield* loadPrimeAgentDaemonBridge(pkg.cliPath); + + expect(bridge.packageRoot).toBe(NodeFS.realpathSync(pkg.root)); + expect(bridge.version).toBe("0.7.1"); + }), + ); + it.effect("loads the public API from a direct dist/cli.js executable", () => Effect.gen(function* () { const pkg = makePackage(); diff --git a/apps/server/src/provider/prime/PrimeAgentDaemonBridge.ts b/apps/server/src/provider/prime/PrimeAgentDaemonBridge.ts index 4f2f0c2b2..d0fcea275 100644 --- a/apps/server/src/provider/prime/PrimeAgentDaemonBridge.ts +++ b/apps/server/src/provider/prime/PrimeAgentDaemonBridge.ts @@ -14,6 +14,8 @@ export const PRIME_AGENT_DAEMON_PROTOCOL_NAME = "prime-agent.daemon" as const; export const PRIME_AGENT_MIN_DAEMON_PROTOCOL_VERSION = 7 as const; export const PRIME_AGENT_NEGOTIATED_DAEMON_SESSION_CAPABILITIES_FEATURE = "negotiated_daemon_session_capabilities_v1" as const; +export const PRIME_AGENT_CALLER_OWNED_SESSION_ENVIRONMENT_CLEANUP_FEATURE = + "caller_owned_session_environment_cleanup_v1" as const; const bridgeErrorReason = Schema.Literals([ "path-not-found", @@ -51,6 +53,51 @@ export interface PrimeAgentDaemonHello { readonly serverCapabilities: ReadonlyArray; } +export interface PrimeAgentOwnedSessionDaemonIdentity { + readonly protocolName: string; + readonly protocolVersion: number; + readonly schemaRevision: number; + readonly appVersion?: string; + readonly buildId?: string; + readonly supervisorGeneration: string; + readonly transportGeneration: number; +} + +export interface PrimeAgentOwnedSessionContractProof { + readonly feature: typeof PRIME_AGENT_CALLER_OWNED_SESSION_ENVIRONMENT_CLEANUP_FEATURE; + readonly status: "attached"; + readonly daemon: PrimeAgentOwnedSessionDaemonIdentity; +} + +interface PrimeAgentOwnedSessionDisposeBase { + readonly feature: typeof PRIME_AGENT_CALLER_OWNED_SESSION_ENVIRONMENT_CLEANUP_FEATURE; + readonly started?: PrimeAgentOwnedSessionContractProof; + readonly observed?: PrimeAgentOwnedSessionDaemonIdentity; +} + +/** Frozen observable cleanup outcomes from Prime Agent #37. */ +export type PrimeAgentOwnedSessionDisposeResult = + | (PrimeAgentOwnedSessionDisposeBase & { + readonly status: "completed" | "already_completed"; + readonly started: PrimeAgentOwnedSessionContractProof; + readonly observed: PrimeAgentOwnedSessionDaemonIdentity; + readonly daemonReplaced: boolean; + }) + | (PrimeAgentOwnedSessionDisposeBase & { + readonly status: "replacement_settled"; + readonly started: PrimeAgentOwnedSessionContractProof; + readonly observed: PrimeAgentOwnedSessionDaemonIdentity; + readonly daemonReplaced: true; + }) + | (PrimeAgentOwnedSessionDisposeBase & { readonly status: "owner_mismatch" }) + | (PrimeAgentOwnedSessionDisposeBase & { + readonly status: "uncertain"; + readonly reason: "active" | "stopping"; + }) + | (PrimeAgentOwnedSessionDisposeBase & { + readonly status: "transport_failure" | "unsupported"; + }); + export interface PrimeAgentDaemonEventCursor { readonly generation: string; readonly sequence: number; @@ -58,6 +105,8 @@ export interface PrimeAgentDaemonEventCursor { export interface PrimeAgentDaemonClient { readonly isConnected: boolean; + /** Public random protocol identity. Exact native multi-instance proofs compare it. */ + readonly clientId?: string; readonly hello?: PrimeAgentDaemonHello; readonly connect: (timeoutMs?: number) => Promise; readonly waitForHello: (timeoutMs?: number) => Promise; @@ -155,7 +204,7 @@ export interface PrimeAgentDaemonAgentConnection { ) => Promise; readonly cancelPromptLifecycle?: (correlationId: string) => Promise; readonly getPromptLifecycles?: () => Promise; - readonly supportsNegotiatedCapability?: (capability: "correlated_prompt_lifecycle_v1") => boolean; + readonly supportsNegotiatedCapability?: (capability: string) => boolean; readonly waitForHeadlessCompletion?: (options?: { readonly waitForRlmQuiescence?: boolean; }) => Promise; @@ -216,7 +265,7 @@ export interface PrimeAgentDaemonAgentConnection { readonly getSessionStats: () => Promise; readonly getRlmMaxDepthStatus?: () => Promise; readonly setRlmMaxDepth?: (maxDepth: number) => Promise; - readonly getOwnedSessionContractProof?: () => unknown; + readonly getOwnedSessionContractProof?: () => PrimeAgentOwnedSessionContractProof | undefined; readonly disposeOwnedSession?: (options?: { readonly timeoutMs?: number }) => Promise; readonly dispose: () => Promise; } @@ -228,6 +277,8 @@ export type PrimeAgentDaemonAgentConnectionOptions = Readonly Promise; /** Fresh owner-held configuration used only if a client-owned worker must be relaunched. */ readonly ownedSessionRecoveryConfig?: Readonly>; + /** Exact immutable caller environment used for create, attach fallback, and recovery. */ + readonly ownedSessionLaunchEnv?: Readonly>; }; export interface PrimeAgentDaemonAgentConnectionConstructor { @@ -337,7 +388,7 @@ const packageIdentitySchema = Schema.Struct({ }); const primeAgentPackageSchema = Schema.Struct({ - name: Schema.Literal("prime-agent"), + name: Schema.Literals(["prime-agent", "@earendil-works/pi-coding-agent"]), version: Schema.String, exports: Schema.Union([ Schema.String, @@ -409,7 +460,10 @@ async function locatePrimeAgentPackage(binaryPath: string): Promise void; } -function fakeProcess(pid: number): Effect.Effect { +function fakeProcess( + pid: number, + options?: { readonly killHangs?: boolean; readonly postKillExitHangs?: boolean }, +): Effect.Effect { return Effect.sync(() => { let exitCompleted = false; let exitResume: ((effect: Effect.Effect) => void) | undefined; const process: FakeProcess = { running: true, kills: 0, + handleKills: 0, + killSignals: [], complete: () => { process.running = false; exitCompleted = true; @@ -67,11 +77,12 @@ function fakeProcess(pid: number): Effect.Effect { pid: ChildProcessSpawner.ProcessId(pid), exitCode, isRunning: Effect.sync(() => process.running), - kill: () => + kill: (killOptions) => Effect.sync(() => { - process.kills += 1; - process.complete(); - }), + process.handleKills += 1; + process.killSignals.push(killOptions?.killSignal ?? "SIGTERM"); + if (options?.postKillExitHangs !== true) process.complete(); + }).pipe(options?.killHangs === true ? Effect.andThen(Effect.never) : (effect) => effect), unref: Effect.succeed(Effect.void), stdin: Sink.drain, stdout: Stream.empty, @@ -95,6 +106,9 @@ function fakeBridge(input: { readonly existingLive?: { value: boolean }; readonly readinessFailures?: { value: number }; readonly calls?: { connect: number; readiness: number; hello: number; prompt: number }; + readonly shutdownHangs?: boolean; + readonly shutdownDoesNotExit?: boolean; + readonly shutdownRejects?: boolean; }): PrimeAgentDaemonBridge { const hello = input.hello ?? @@ -141,8 +155,10 @@ function fakeBridge(input: { if (command.type === "shutdown") { input.shutdownRequests.push(this.socketPath); input.events?.push(input.existingLive?.value === true ? "existing-shutdown" : "shutdown"); + if (input.shutdownHangs === true) return new Promise(() => undefined); + if (input.shutdownRejects === true) return Promise.reject(new Error("shutdown rejected")); if (input.existingLive) input.existingLive.value = false; - input.processes.at(-1)?.complete(); + if (input.shutdownDoesNotExit !== true) input.processes.at(-1)?.complete(); } return Promise.resolve({ type: "response", success: true }); } @@ -159,6 +175,25 @@ function fakeBridge(input: { subscribe(): () => void { return () => undefined; } + supportsNegotiatedCapability(): boolean { + return true; + } + getOwnedSessionContractProof() { + return { + feature: "caller_owned_session_environment_cleanup_v1" as const, + status: "attached" as const, + daemon: { + protocolName: "prime-agent.daemon", + protocolVersion: 7, + schemaRevision: 30, + supervisorGeneration: "supervisor-1", + transportGeneration: 0, + }, + }; + } + disposeOwnedSession(): Promise { + return Promise.resolve(); + } getCommands(): Promise { return Promise.resolve([]); } @@ -192,8 +227,11 @@ function fakeBridge(input: { version: "0.7.1", protocolName: "prime-agent.daemon", protocolVersion: 7, - negotiatedDaemonSessionCapabilitiesAvailable: false, - sdkFeatures: [], + negotiatedDaemonSessionCapabilitiesAvailable: true, + sdkFeatures: [ + "negotiated_daemon_session_capabilities_v1", + "caller_owned_session_environment_cleanup_v1", + ], recoverableOwnedSessionAdoptionAvailable: false, DaemonClient: FakeClient, DaemonAgentConnection: FakeAgentConnection, @@ -211,6 +249,21 @@ function managerFixture(options?: { readonly platform?: NodeJS.Platform; readonly injectBridge?: boolean; readonly recoverable?: boolean; + readonly sdkFeatures?: ReadonlyArray; + readonly currentGeneration?: { readonly value: boolean }; + readonly shutdownTimeout?: Duration.Input; + readonly shutdownHangs?: boolean; + readonly shutdownDoesNotExit?: boolean; + readonly shutdownRejects?: boolean; + readonly killHangs?: boolean; + readonly postKillExitHangs?: boolean; + readonly scopeCloseBarrier?: Promise; + readonly socketCleanupHangs?: boolean; + readonly staleProcessIdentity?: boolean; + readonly processIdentity?: { value: string }; + readonly beforeAtomicSignal?: () => Promise; + readonly atomicSignalAvailable?: boolean; + readonly instanceId?: string; }) { const commands: CapturedCommand[] = []; const processes: FakeProcess[] = []; @@ -220,9 +273,12 @@ function managerFixture(options?: { const readinessFailures = { value: options?.readinessFailures ?? 0 }; const connectionAvailable = { value: true }; const calls = { connect: 0, readiness: 0, hello: 0, prompt: 0 }; + let identityInspections = 0; + let socketCleanupCalls = 0; + const instanceId = ProviderInstanceId.make(options?.instanceId ?? "prime-work"); const paths = derivePrimeAgentDaemonPaths({ stateDir: "/tmp/pylon-state", - providerInstanceId: ProviderInstanceId.make("prime-work"), + providerInstanceId: instanceId, platform: options?.platform ?? "linux", tempDir: options?.tempDir ?? "/tmp", }); @@ -250,14 +306,28 @@ function managerFixture(options?: { readinessFailures, connectionAvailable, calls, + ...(options?.shutdownHangs === undefined ? {} : { shutdownHangs: options.shutdownHangs }), + ...(options?.shutdownDoesNotExit === undefined + ? {} + : { shutdownDoesNotExit: options.shutdownDoesNotExit }), + ...(options?.shutdownRejects === undefined ? {} : { shutdownRejects: options.shutdownRejects }), ...(options?.hello === undefined && recoveryHello === undefined ? {} : { hello: options?.hello ?? recoveryHello }), ...(options?.failConnect === undefined ? {} : { failConnect: options.failConnect }), }); + if (options?.sdkFeatures !== undefined) { + Object.assign(bridge, { + sdkFeatures: [...options.sdkFeatures], + negotiatedDaemonSessionCapabilitiesAvailable: options.sdkFeatures.includes( + "negotiated_daemon_session_capabilities_v1", + ), + }); + } if (options?.recoverable) { Object.assign(bridge, { sdkFeatures: [ + "negotiated_daemon_session_capabilities_v1", "recoverable_owned_session_adoption_v1", "caller_owned_session_environment_cleanup_v1", ], @@ -271,41 +341,105 @@ function managerFixture(options?: { confirmRecoverableOwnedSessionAdoption: async () => undefined, }); } - const spawner = Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make((command) => - Effect.gen(function* () { - commands.push(command as unknown as CapturedCommand); - events.push("spawn"); - if (processes.length > 0 && options?.restoreConnectionOnSpawn) { - connectionAvailable.value = true; - } - const process = yield* fakeProcess(processes.length + 1); - processes.push(process); - return process.handle; - }), - ), - ); + const spawnProcess: NonNullable< + Parameters[0]["spawnProcess"] + > = ({ executablePath, args, environment, output }) => + Effect.gen(function* () { + commands.push({ + command: executablePath, + args, + options: { env: environment, extendEnv: false, output }, + }); + events.push("spawn"); + if (processes.length > 0 && options?.restoreConnectionOnSpawn) { + connectionAvailable.value = true; + } + if (options?.scopeCloseBarrier !== undefined) { + yield* Effect.addFinalizer(() => Effect.promise(() => options.scopeCloseBarrier!)); + } + const process = yield* fakeProcess(processes.length + 1, { + ...(options?.killHangs === undefined ? {} : { killHangs: options.killHangs }), + ...(options?.postKillExitHangs === undefined + ? {} + : { postKillExitHangs: options.postKillExitHangs }), + }); + processes.push(process); + return process.handle; + }); const make = makePrimeAgentDaemonManager({ executablePath: "/resolved/bin/prime-agent", - settings: { agentHomePath: "~/.prime/pylon" }, - environment: { - PATH: "/usr/bin", - PRIME_AGENT_INTERNAL_ROLE: "worker", - PRIME_AGENT_INTERNAL_TOKEN: "secret", - KEEP_ME: "yes", + identity: { + instanceId, + generation: { _tag: "PrimeAgentRuntimeGeneration" }, + configRevision: "test-revision", + effectiveHome: NodePath.resolve(process.env.HOME!, ".prime/pylon"), + nativeMultipleInstancesRequired: false, + launchEnv: { + PATH: "/usr/bin", + KEEP_ME: "yes", + PRIME_AGENT_HOME: NodePath.resolve(process.env.HOME!, ".prime/pylon"), + PRIME_AGENT_CODING_AGENT_DIR: NodePath.resolve(process.env.HOME!, ".prime/pylon"), + }, + settings: { + enabled: true, + binaryPath: "prime-agent", + agentHomePath: NodePath.resolve(process.env.HOME!, ".prime/pylon"), + launchArgs: "", + customModels: [], + }, }, + ...(options?.currentGeneration === undefined + ? {} + : { + runtimeFence: { + generation: {}, + configRevision: "test-revision", + isCurrent: Effect.sync(() => options.currentGeneration!.value), + }, + }), stateDir: "/tmp/pylon-state", - providerInstanceId: ProviderInstanceId.make("prime-work"), platform: options?.platform ?? "linux", tempDir: options?.tempDir ?? "/tmp", readinessRetryDelay: Duration.zero, readinessRetries: 4, - shutdownTimeout: Duration.zero, + shutdownTimeout: options?.shutdownTimeout ?? Duration.millis(100), recoveryEnabled: options?.recoverable === true, architecture: "arm64", + spawnProcess, + inspectProcessIdentity: async (pid) => { + identityInspections += 1; + return `${options?.processIdentity?.value ?? "test"}:${pid}`; + }, + ...(options?.atomicSignalAvailable === false + ? {} + : { + signalProcessIdentity: async ({ pid, startIdentity }) => { + identityInspections += 1; + await options?.beforeAtomicSignal?.(); + const currentIdentity = + options?.staleProcessIdentity === true && identityInspections > 1 + ? `stale:${pid}` + : `${options?.processIdentity?.value ?? "test"}:${pid}`; + if (currentIdentity !== startIdentity) return false; + const process = processes.find((candidate) => candidate.handle.pid === pid); + if (process === undefined) return false; + process.kills += 1; + process.killSignals.push("SIGKILL"); + if (options?.killHangs === true) await new Promise(() => undefined); + if (options?.postKillExitHangs !== true) process.complete(); + return true; + }, + }), + ...(options?.socketCleanupHangs === true + ? { + removePrivateSocket: async () => { + socketCleanupCalls += 1; + if (socketCleanupCalls > 1) await new Promise(() => undefined); + }, + } + : {}), ...(options?.injectBridge === false ? {} : { bridge }), - }).pipe(Effect.provide(Layer.merge(NodeServices.layer, spawner))); + }).pipe(Effect.provide(NodeServices.layer)); return { make, commands, @@ -318,6 +452,16 @@ function managerFixture(options?: { }; } +function closeFixtureLive(fixture: ReturnType) { + return Effect.scoped( + Effect.gen(function* () { + const manager = yield* fixture.make; + const client = yield* manager.openClient(); + client.close(); + }), + ); +} + describe("PrimeAgentDaemonManager paths and environment", () => { it("derives stable short Unix sockets and Windows private pipes", () => { const input = { @@ -367,6 +511,22 @@ describe("PrimeAgentDaemonManager lifecycle", () => { }); }); + it.effect("rejects missing frozen SDK proof before spawning or opening a daemon", () => { + const fixture = managerFixture({ + sdkFeatures: ["negotiated_daemon_session_capabilities_v1"], + }); + return Effect.gen(function* () { + const result = yield* fixture.make.pipe(Effect.result); + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") { + expect(result.failure.reason).toBe("incompatible-hello"); + expect(result.failure.detail).not.toContain("/resolved/bin/prime-agent"); + } + expect(fixture.commands).toEqual([]); + expect(fixture.calls.connect).toBe(0); + }); + }); + it.effect("does not touch a stable socket when the lazy manager was never opened", () => { const fixture = managerFixture({ existingLive: true }); return Effect.scoped(Effect.asVoid(fixture.make)).pipe( @@ -503,10 +663,7 @@ describe("PrimeAgentDaemonManager lifecycle", () => { expect(fixture.commands).toHaveLength(1); expect(fixture.processes).toHaveLength(1); expect(fixture.commands[0]!.options).toMatchObject({ - detached: true, - stdin: "ignore", - stdout: "ignore", - stderr: "ignore", + output: "ignore", extendEnv: false, }); expect(fixture.processes[0]!.handle.pid).toBe(1); @@ -631,6 +788,26 @@ describe("PrimeAgentDaemonManager lifecycle", () => { }).pipe(Effect.scoped); }); + it.effect("retired cleanup kills only its captured child and never the successor socket", () => { + const currentGeneration = { value: true }; + const fixture = managerFixture({ currentGeneration }); + return Effect.scoped( + Effect.gen(function* () { + const manager = yield* fixture.make; + const client = yield* manager.openClient(); + client.close(); + currentGeneration.value = false; + }), + ).pipe( + Effect.andThen( + Effect.sync(() => { + expect(fixture.shutdownRequests).toEqual([]); + expect(fixture.processes[0]?.kills).toBe(1); + }), + ), + ); + }); + it.effect( "requests public graceful shutdown and awaits the captured child without killing it", () => { @@ -674,6 +851,164 @@ describe("PrimeAgentDaemonManager lifecycle", () => { ); }); + it.effect("contains only the exact instance-private daemon during cleanup", () => + Effect.gen(function* () { + const first = managerFixture({ instanceId: "prime-first" }); + const second = managerFixture({ instanceId: "prime-second" }); + const firstScope = yield* Scope.make("sequential"); + const secondScope = yield* Scope.make("sequential"); + yield* Effect.gen(function* () { + const firstManager = yield* first.make.pipe(Effect.provideService(Scope.Scope, firstScope)); + const secondManager = yield* second.make.pipe( + Effect.provideService(Scope.Scope, secondScope), + ); + const firstClient = yield* firstManager.openClient(); + firstClient.close(); + const secondClient = yield* secondManager.openClient(); + secondClient.close(); + + expect(first.paths.socket).not.toBe(second.paths.socket); + yield* Scope.close(firstScope, Exit.void); + expect(first.processes[0]?.running).toBe(false); + expect(second.processes[0]?.running).toBe(true); + expect(second.shutdownRequests).toEqual([]); + }).pipe( + Effect.ensuring(Scope.close(firstScope, Exit.void).pipe(Effect.ignore)), + Effect.ensuring(Scope.close(secondScope, Exit.void).pipe(Effect.ignore)), + ); + }), + ); + + it.live("bounds a hung graceful shutdown before exact-identity force containment", () => { + const fixture = managerFixture({ shutdownHangs: true, shutdownTimeout: Duration.millis(5) }); + return closeFixtureLive(fixture).pipe( + Effect.andThen( + Effect.sync(() => { + expect(fixture.shutdownRequests).toEqual([fixture.paths.socket]); + expect(fixture.processes[0]?.kills).toBe(1); + expect(fixture.processes[0]?.handleKills).toBe(0); + expect(fixture.processes[0]?.killSignals).toEqual(["SIGKILL"]); + }), + ), + ); + }); + + it.live("bounds a hung force-kill request", () => { + const fixture = managerFixture({ + shutdownRejects: true, + shutdownDoesNotExit: true, + killHangs: true, + shutdownTimeout: Duration.millis(5), + }); + return closeFixtureLive(fixture).pipe( + Effect.andThen(Effect.sync(() => expect(fixture.processes[0]?.kills).toBe(1))), + ); + }); + + it.live("bounds the post-kill exit receipt wait", () => { + const fixture = managerFixture({ + shutdownRejects: true, + shutdownDoesNotExit: true, + postKillExitHangs: true, + shutdownTimeout: Duration.millis(5), + }); + return closeFixtureLive(fixture).pipe( + Effect.andThen( + Effect.sync(() => { + expect(fixture.processes[0]?.kills).toBe(1); + expect(fixture.processes[0]?.running).toBe(true); + }), + ), + ); + }); + + it.live("bounds private socket cleanup and process scope close barriers", () => { + let releaseScope!: () => void; + const scopeCloseBarrier = new Promise((resolve) => { + releaseScope = resolve; + }); + const fixture = managerFixture({ + socketCleanupHangs: true, + scopeCloseBarrier, + shutdownTimeout: Duration.millis(5), + }); + return closeFixtureLive(fixture).pipe( + Effect.ensuring(Effect.sync(() => releaseScope())), + Effect.andThen(Effect.sync(() => expect(fixture.processes[0]?.running).toBe(false))), + ); + }); + + it.live("refuses forced signaling when the host has no atomic identity-bound syscall", () => { + const fixture = managerFixture({ + shutdownRejects: true, + shutdownDoesNotExit: true, + atomicSignalAvailable: false, + shutdownTimeout: Duration.millis(5), + }); + return closeFixtureLive(fixture).pipe( + Effect.andThen( + Effect.sync(() => { + expect(fixture.processes[0]?.kills).toBe(0); + expect(fixture.processes[0]?.handleKills).toBe(0); + expect(fixture.processes[0]?.running).toBe(true); + }), + ), + ); + }); + + it.live( + "rechecks identity inside the atomic signal barrier and scope close sends no fallback signal", + () => { + const identity = { value: "captured" }; + let entered!: () => void; + let release!: () => void; + const signalEntered = new Promise((resolve) => { + entered = resolve; + }); + const signalRelease = new Promise((resolve) => { + release = resolve; + }); + const fixture = managerFixture({ + shutdownRejects: true, + shutdownDoesNotExit: true, + processIdentity: identity, + beforeAtomicSignal: async () => { + entered(); + await signalRelease; + }, + shutdownTimeout: Duration.millis(50), + }); + return Effect.gen(function* () { + const closing = yield* closeFixtureLive(fixture).pipe(Effect.forkChild); + yield* Effect.promise(() => signalEntered); + identity.value = "reused"; + release(); + yield* Fiber.join(closing); + expect(fixture.processes[0]?.kills).toBe(0); + expect(fixture.processes[0]?.handleKills).toBe(0); + expect(fixture.processes[0]?.running).toBe(true); + }).pipe(Effect.ensuring(Effect.sync(() => release()))); + }, + ); + + it.live("never signals a stale or unproved PID/start identity", () => { + const fixture = managerFixture({ + shutdownRejects: true, + shutdownDoesNotExit: true, + staleProcessIdentity: true, + shutdownTimeout: Duration.millis(5), + }); + return closeFixtureLive(fixture).pipe( + Effect.andThen( + Effect.sync(() => { + expect(fixture.processes[0]?.kills).toBe(0); + expect(fixture.processes[0]?.handleKills).toBe(0); + expect(fixture.processes[0]?.running).toBe(true); + }), + ), + ); + }); + it.effect("retires an adopted recovery supervisor on clean shutdown", () => { const fixture = managerFixture({ existingLive: true, recoverable: true }); return Effect.scoped( diff --git a/apps/server/src/provider/prime/PrimeAgentDaemonManager.ts b/apps/server/src/provider/prime/PrimeAgentDaemonManager.ts index d22fa01f5..01cac0652 100644 --- a/apps/server/src/provider/prime/PrimeAgentDaemonManager.ts +++ b/apps/server/src/provider/prime/PrimeAgentDaemonManager.ts @@ -1,29 +1,37 @@ // @effect-diagnostics nodeBuiltinImport:off import * as NodeCrypto from "node:crypto"; +import * as NodeChildProcess from "node:child_process"; import * as NodeFSP from "node:fs/promises"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; -import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import type { PrimeAgentSettings, ProviderInstanceId } from "@t3tools/contracts"; +import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as Option from "effect/Option"; +import * as PlatformError from "effect/PlatformError"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Schedule from "effect/Schedule"; import * as Semaphore from "effect/Semaphore"; +import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; import * as FileSystem from "effect/FileSystem"; -import { ChildProcess } from "effect/unstable/process"; import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; import { resolveProviderHomePath } from "../../pathExpansion.ts"; +import type { ProviderRuntimeFence } from "../ProviderDriver.ts"; +import type { PrimeAgentMaterializedIdentity } from "./PrimeAgentRuntimeContext.ts"; import { loadPrimeAgentDaemonBridge, + PRIME_AGENT_CALLER_OWNED_SESSION_ENVIRONMENT_CLEANUP_FEATURE, PRIME_AGENT_DAEMON_PROTOCOL_NAME, PRIME_AGENT_MIN_DAEMON_PROTOCOL_VERSION, + PRIME_AGENT_NEGOTIATED_DAEMON_SESSION_CAPABILITIES_FEATURE, type PrimeAgentDaemonBridge, type PrimeAgentDaemonBridgeError, type PrimeAgentDaemonClient, @@ -40,6 +48,8 @@ export const PRIME_AGENT_REQUIRED_DAEMON_CAPABILITIES = [ "extension_ui", "session_input_admission", "prompt_admission_cancellation", + "caller_owned_session_environment_cleanup_v1", + "authoritative_owned_session_cleanup_v1", ] as const; const managerErrorReason = Schema.Literals([ @@ -70,6 +80,7 @@ export type PrimeAgentDaemonManagerOpenError = PrimeAgentDaemonManagerError; export interface PrimeAgentDaemonManager { readonly bridge: PrimeAgentDaemonBridge; + readonly identity: PrimeAgentMaterializedIdentity; readonly socket: string; readonly sessionDir: string; /** Starts the daemon and validates its control-plane hello without opening an agent session. */ @@ -91,10 +102,9 @@ export interface PrimeAgentDaemonManager { export interface PrimeAgentDaemonManagerInput { readonly executablePath: string; - readonly settings: Pick; - readonly environment?: NodeJS.ProcessEnv; + readonly identity: PrimeAgentMaterializedIdentity; + readonly runtimeFence?: ProviderRuntimeFence | undefined; readonly stateDir: string; - readonly providerInstanceId: ProviderInstanceId | string; readonly connectTimeoutMs?: number; readonly readinessRetryDelay?: Duration.Input; readonly readinessRetries?: number; @@ -108,11 +118,37 @@ export interface PrimeAgentDaemonManagerInput { readonly architecture?: string; /** Tests may supply the already validated public bridge without importing a real installation. */ readonly bridge?: PrimeAgentDaemonBridge; + /** Test seam for capturing a process start identity. */ + readonly inspectProcessIdentity?: (pid: number) => Promise; + /** + * Optional host primitive that must bind identity validation and signal delivery + * atomically. Pylon supplies no default because Node's kill APIs cannot do so. + */ + readonly signalProcessIdentity?: (input: { + readonly pid: number; + readonly startIdentity: string; + readonly signal: "SIGKILL"; + }) => Promise; + /** Test seam; production uses a direct spawn with no process-signaling scope finalizer. */ + readonly spawnProcess?: (input: { + readonly executablePath: string; + readonly args: ReadonlyArray; + readonly environment: Readonly>; + readonly output: "ignore" | "pipe"; + }) => Effect.Effect< + ChildProcessSpawner.ChildProcessHandle, + PlatformError.PlatformError, + Scope.Scope + >; + /** Test seam for a private socket cleanup barrier. */ + readonly removePrivateSocket?: (socket: string) => Promise; } interface RunningDaemon { readonly handle: ChildProcessSpawner.ChildProcessHandle; + /** Output-drain scope only. It never owns a process-signaling finalizer. */ readonly scope: Scope.Scope; + readonly startIdentity: string | undefined; } const daemonHelloSchema = Schema.Struct({ @@ -241,6 +277,15 @@ function connectClient(input: { cause, ), }).pipe( + Effect.timeoutOrElse({ + duration: input.timeoutMs, + orElse: () => + managerError( + input.socket, + "readiness-failed", + "Timed out while opening the Pylon-owned daemon control connection.", + ), + }), Effect.onError(() => Effect.sync(() => { client.close(); @@ -258,6 +303,141 @@ function connectClient(input: { }); } +async function inspectNativeProcessIdentity( + pid: number, + platform: NodeJS.Platform, +): Promise { + if (!Number.isSafeInteger(pid) || pid <= 0) return undefined; + if (platform === "linux") { + try { + const stat = await NodeFSP.readFile(`/proc/${pid}/stat`, "utf8"); + const commandEnd = stat.lastIndexOf(")"); + if (commandEnd < 0) return undefined; + const fields = stat + .slice(commandEnd + 2) + .trim() + .split(/\s+/u); + const startTicks = fields[19]; + return startTicks === undefined ? undefined : `${pid}:${startTicks}`; + } catch { + return undefined; + } + } + if (platform !== "darwin") return undefined; + return await new Promise((resolve) => { + NodeChildProcess.execFile( + "/bin/ps", + ["-o", "lstart=", "-p", String(pid)], + { timeout: 1_000, maxBuffer: 1_024, encoding: "utf8" }, + (error, stdout) => { + const started = stdout.trim(); + resolve(error === null && started.length > 0 ? `${pid}:${started}` : undefined); + }, + ); + }); +} + +function childProcessPlatformError( + method: string, + pathOrDescriptor: string, + cause?: unknown, +): PlatformError.PlatformError { + return PlatformError.systemError({ + _tag: "Unknown", + module: "PrimeAgentDaemonManager", + method, + pathOrDescriptor, + ...(cause === undefined ? {} : { cause }), + }); +} + +function spawnPrimeAgentDaemon(input: { + readonly executablePath: string; + readonly args: ReadonlyArray; + readonly environment: Readonly>; + readonly output: "ignore" | "pipe"; +}): Effect.Effect { + return Effect.callback( + (resume) => { + let settled = false; + const child = NodeChildProcess.spawn(input.executablePath, [...input.args], { + env: input.environment, + detached: true, + stdio: ["ignore", input.output, input.output], + }); + let exit: readonly [number | null, NodeJS.Signals | null] | undefined; + const exitWaiters = new Set<(code: ChildProcessSpawner.ExitCode) => void>(); + child.once("exit", (code, signal) => { + exit = [code, signal]; + const normalized = ChildProcessSpawner.ExitCode(code ?? 1); + for (const waiter of exitWaiters) waiter(normalized); + exitWaiters.clear(); + }); + const onError = (cause: Error) => { + if (settled) return; + settled = true; + resume(Effect.fail(childProcessPlatformError("spawn", input.executablePath, cause))); + }; + child.once("error", onError); + child.once("spawn", () => { + if (settled) return; + settled = true; + const output = (readable: typeof child.stdout) => + readable === null + ? Stream.empty + : Stream.fromAsyncIterable(readable, (cause) => + childProcessPlatformError("read-output", input.executablePath, cause), + ); + const stdout = output(child.stdout); + const stderr = output(child.stderr); + const exitCode = Effect.callback((complete) => { + if (exit !== undefined) { + complete(Effect.succeed(ChildProcessSpawner.ExitCode(exit[0] ?? 1))); + return; + } + const waiter = (code: ChildProcessSpawner.ExitCode) => complete(Effect.succeed(code)); + exitWaiters.add(waiter); + return Effect.sync(() => exitWaiters.delete(waiter)); + }); + const unsupportedSignal = () => + Effect.fail( + childProcessPlatformError( + "kill", + String(child.pid), + new Error("Unbound Node process signaling is disabled"), + ), + ); + resume( + Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(child.pid!), + exitCode, + isRunning: Effect.sync(() => exit === undefined), + kill: unsupportedSignal, + unref: Effect.sync(() => { + child.unref(); + return Effect.sync(() => child.ref()); + }), + stdin: Sink.drain, + stdout, + stderr, + all: Stream.merge(stdout, stderr), + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }), + ), + ); + }); + return Effect.sync(() => { + if (!settled) { + settled = true; + child.unref(); + } + }); + }, + ); +} + function drainProcessOutput( stream: Stream.Stream, streamName: "stdout" | "stderr", @@ -280,14 +460,17 @@ export const makePrimeAgentDaemonManager = Effect.fn("makePrimeAgentDaemonManage ): Effect.fn.Return< PrimeAgentDaemonManager, PrimeAgentDaemonBridgeError | PrimeAgentDaemonManagerError, - ChildProcessSpawner.ChildProcessSpawner | FileSystem.FileSystem | Scope.Scope + FileSystem.FileSystem | Scope.Scope > { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const fileSystem = yield* FileSystem.FileSystem; const hostPlatform = yield* HostProcessPlatform; - const hostEnvironment = yield* HostProcessEnvironment; const platform = input.platform ?? hostPlatform; - const paths = derivePrimeAgentDaemonPaths({ ...input, platform }); + const paths = derivePrimeAgentDaemonPaths({ + stateDir: input.stateDir, + providerInstanceId: input.identity.instanceId, + platform, + ...(input.tempDir === undefined ? {} : { tempDir: input.tempDir }), + }); if (platform === "win32") { return yield* managerError( paths.socket, @@ -296,22 +479,25 @@ export const makePrimeAgentDaemonManager = Effect.fn("makePrimeAgentDaemonManage ); } const bridge = input.bridge ?? (yield* loadPrimeAgentDaemonBridge(input.executablePath)); + if ( + !bridge.sdkFeatures?.includes(PRIME_AGENT_NEGOTIATED_DAEMON_SESSION_CAPABILITIES_FEATURE) || + !bridge.sdkFeatures.includes(PRIME_AGENT_CALLER_OWNED_SESSION_ENVIRONMENT_CLEANUP_FEATURE) + ) { + return yield* managerError( + paths.socket, + "incompatible-hello", + "The installed Prime Agent SDK does not provide the required caller-owned session contract.", + ); + } const recoveryEnabled = input.recoveryEnabled === true && bridge.recoverableOwnedSessionAdoptionAvailable === true; - const launchEnvironment = Object.fromEntries( - Object.entries( - makePrimeAgentDaemonEnvironment({ - settings: input.settings, - environment: input.environment ?? hostEnvironment, - }), - ).filter((entry): entry is [string, string] => typeof entry[1] === "string"), - ); + const launchEnvironment = input.identity.launchEnv; let recoveryRetainers = 0; let retainedExistingDaemon = false; let adoptedExistingDaemon = false; const retainForRecovery = () => { recoveryRetainers += 1; - if (retainedExistingDaemon) adoptedExistingDaemon = true; + adoptedExistingDaemon = true; let released = false; return () => { if (released) return; @@ -331,23 +517,59 @@ export const makePrimeAgentDaemonManager = Effect.fn("makePrimeAgentDaemonManage Schedule.recurs(input.readinessRetries ?? 20), ]); const shutdownTimeout = input.shutdownTimeout ?? Duration.seconds(5); + const shutdownTimeoutMs = Math.max( + 1, + Duration.toMillis(Duration.fromInputUnsafe(shutdownTimeout)), + ); + const inspectProcessIdentity = + input.inspectProcessIdentity ?? ((pid: number) => inspectNativeProcessIdentity(pid, platform)); const semaphore = yield* Semaphore.make(1); let running: RunningDaemon | undefined; + let starting: RunningDaemon | undefined; let closing = false; + const isCurrentGeneration = input.runtimeFence?.isCurrent ?? Effect.succeed(true); + const isStartAllowed = Effect.map(isCurrentGeneration, (current) => current && !closing); const removeSocket = () => - fileSystem - .remove(socket, { force: true }) - .pipe( - Effect.mapError((cause) => - managerError( - socket, - "state-directory-failed", - "Could not clean the daemon socket.", - cause, - ), - ), + Effect.gen(function* () { + // Recheck at unlink, not when cleanup was scheduled: a replacement may now own this path. + if (!(yield* isCurrentGeneration)) return; + const cleanup = + input.removePrivateSocket === undefined + ? fileSystem + .remove(socket, { force: true }) + .pipe( + Effect.mapError((cause) => + managerError( + socket, + "state-directory-failed", + "Could not clean the daemon socket.", + cause, + ), + ), + ) + : Effect.tryPromise({ + try: () => input.removePrivateSocket!(socket), + catch: (cause) => + managerError( + socket, + "state-directory-failed", + "Could not clean the private daemon socket.", + cause, + ), + }); + yield* cleanup.pipe( + Effect.timeoutOrElse({ + duration: shutdownTimeout, + orElse: () => + managerError( + socket, + "state-directory-failed", + "Timed out while cleaning the private daemon socket.", + ), + }), ); + }); const ensurePrivateSocketDirectory = Effect.fn( "PrimeAgentDaemonManager.ensurePrivateSocketDirectory", @@ -491,89 +713,195 @@ export const makePrimeAgentDaemonManager = Effect.fn("makePrimeAgentDaemonManage }); const closeProcessScope = (state: RunningDaemon) => - Scope.close(state.scope, Exit.void).pipe(Effect.ignore); + Effect.gen(function* () { + // Scope finalizers are uninterruptible. Await them through a detached fiber so a + // wedged subprocess/stream finalizer cannot defeat the explicit shutdown bound. + const closer = yield* Effect.forkDetach(Scope.close(state.scope, Exit.void)); + const result = yield* Fiber.await(closer).pipe(Effect.timeoutOption(shutdownTimeout)); + if (Option.isNone(result)) { + yield* Effect.logWarning("Prime Agent daemon process scope close timed out.", { + provider: "primeAgent", + }); + return false; + } + if (Exit.isFailure(result.value)) { + yield* Effect.logWarning("Prime Agent daemon process scope close failed.", { + provider: "primeAgent", + }); + return false; + } + return true; + }); - const stopCapturedDaemon = Effect.fn("PrimeAgentDaemonManager.stopCapturedDaemon")(function* ( + const awaitCapturedExit = (state: RunningDaemon, phase: "graceful" | "post-kill") => + state.handle.exitCode.pipe( + Effect.timeoutOption(shutdownTimeout), + Effect.catch((cause) => + Effect.logWarning("Could not await Prime Agent daemon exit.").pipe( + Effect.annotateLogs({ provider: "primeAgent", phase, cause }), + Effect.as(Option.none()), + ), + ), + ); + + const closeProcessScopeSafely = Effect.fn("PrimeAgentDaemonManager.closeProcessScopeSafely")( + function* (state: RunningDaemon, processExited: boolean) { + if (!processExited) { + // The direct production spawn has no process finalizer. Unref the exact + // captured handle and close only output drains; neither action signals a PID or PGID. + yield* state.handle.unref.pipe(Effect.timeoutOption(shutdownTimeout), Effect.ignore); + } + return yield* closeProcessScope(state); + }, + ); + + const forceKillCaptured = Effect.fn("PrimeAgentDaemonManager.forceKillCaptured")(function* ( state: RunningDaemon, - controlPlaneReady = true, ) { - let controlClient: PrimeAgentDaemonClient | undefined; - const isRunning = yield* state.handle.isRunning.pipe( - Effect.catch((cause) => { - return Effect.logWarning("Could not inspect Prime Agent daemon during shutdown.").pipe( - Effect.annotateLogs({ provider: "primeAgent", cause }), - Effect.as(true), - ); + if (state.startIdentity === undefined || input.signalProcessIdentity === undefined) { + yield* Effect.logWarning( + "Refusing forced Prime Agent daemon signaling because no atomic PID/start-identity signal primitive is available.", + { provider: "primeAgent" }, + ); + return false; + } + // This is the only forced-signal path. The callback contract binds the + // identity check and syscall into one primitive; there is no check-then-kill gap. + return yield* Effect.tryPromise(() => + input.signalProcessIdentity!({ + pid: state.handle.pid, + startIdentity: state.startIdentity!, + signal: "SIGKILL", }), + ).pipe( + Effect.timeoutOption(shutdownTimeout), + Effect.flatMap( + Option.match({ + onNone: () => + Effect.logWarning("Prime Agent daemon atomic force-signal request timed out.", { + provider: "primeAgent", + }).pipe(Effect.as(false)), + onSome: Effect.succeed, + }), + ), + Effect.catch((cause) => + Effect.logWarning("Could not atomically signal the captured Prime Agent daemon.").pipe( + Effect.annotateLogs({ provider: "primeAgent", cause }), + Effect.as(false), + ), + ), ); + }); - if (isRunning) { - if (controlPlaneReady) { - controlClient = yield* connectClient({ bridge, socket, timeoutMs }).pipe( - Effect.catch((error) => - Effect.logWarning(error.message).pipe( - Effect.annotateLogs({ provider: "primeAgent" }), - Effect.as(undefined), - ), - ), - ); - if (controlClient) { - yield* Effect.tryPromise({ - try: async () => { - const response = await controlClient!.request({ type: "shutdown" }, timeoutMs); - if (!isDaemonSuccessResponse(response)) { - throw new Error("shutdown response was not a successful public daemon response"); - } - }, - catch: (cause) => - managerError( - socket, - "shutdown-failed", - "The daemon rejected or did not answer its public shutdown command.", - cause, - ), + const stopCapturedDaemonBody = Effect.fn("PrimeAgentDaemonManager.stopCapturedDaemonBody")( + function* (state: RunningDaemon, controlPlaneReady = true) { + let controlClient: PrimeAgentDaemonClient | undefined; + const runningReceipt = yield* state.handle.isRunning.pipe( + Effect.timeoutOption(shutdownTimeout), + Effect.catch((cause) => { + return Effect.logWarning("Could not inspect Prime Agent daemon during shutdown.").pipe( + Effect.annotateLogs({ provider: "primeAgent", cause }), + Effect.as(Option.none()), + ); + }), + ); + let exited = Option.isSome(runningReceipt) && runningReceipt.value === false; + const isRunning = !exited; + + if (isRunning && !(yield* isCurrentGeneration)) { + const killed = yield* forceKillCaptured(state); + if (killed) exited = Option.isSome(yield* awaitCapturedExit(state, "post-kill")); + yield* closeProcessScopeSafely(state, exited); + return; + } + + if (isRunning) { + if (controlPlaneReady) { + controlClient = yield* connectClient({ + bridge, + socket, + timeoutMs: shutdownTimeoutMs, }).pipe( Effect.catch((error) => Effect.logWarning(error.message).pipe( Effect.annotateLogs({ provider: "primeAgent" }), + Effect.as(undefined), ), ), - Effect.ensuring( - Effect.sync(() => { - controlClient?.close(); - }), - ), ); - } - } - - const gracefulExit = controlPlaneReady - ? yield* state.handle.exitCode.pipe( - Effect.timeoutOption(shutdownTimeout), - Effect.catch((cause) => - Effect.logWarning("Could not await Prime Agent daemon exit.").pipe( - Effect.annotateLogs({ provider: "primeAgent", cause }), - Effect.as(Option.none()), + if (controlClient && (yield* isCurrentGeneration)) { + yield* Effect.tryPromise({ + try: async () => { + const response = await controlClient!.request( + { type: "shutdown" }, + shutdownTimeoutMs, + ); + if (!isDaemonSuccessResponse(response)) { + throw new Error("shutdown response was not a successful public daemon response"); + } + }, + catch: (cause) => + managerError( + socket, + "shutdown-failed", + "The daemon rejected or did not answer its public shutdown command.", + cause, + ), + }).pipe( + Effect.timeoutOption(shutdownTimeout), + Effect.catch((error) => + Effect.logWarning(error.message).pipe( + Effect.annotateLogs({ provider: "primeAgent" }), + Effect.as(Option.none()), + ), ), - ), - ) - : Option.none(); - if (Option.isNone(gracefulExit)) { - yield* state.handle - .kill() - .pipe( - Effect.catch((cause) => - Effect.logWarning("Could not interrupt captured Prime Agent daemon process.").pipe( - Effect.annotateLogs({ provider: "primeAgent", pid: state.handle.pid, cause }), + Effect.ensuring( + Effect.sync(() => { + controlClient?.close(); + }), ), - ), - ); - yield* state.handle.exitCode.pipe(Effect.ignore); + ); + } else { + yield* Effect.sync(() => controlClient?.close()); + } + } + + const gracefulExit = controlPlaneReady + ? yield* awaitCapturedExit(state, "graceful") + : Option.none(); + exited = Option.isSome(gracefulExit); + if (!exited) { + const killed = yield* forceKillCaptured(state); + if (killed) exited = Option.isSome(yield* awaitCapturedExit(state, "post-kill")); + } } - } - yield* removeSocket().pipe(Effect.ignore); - yield* closeProcessScope(state); + if (exited) yield* removeSocket().pipe(Effect.ignore); + yield* closeProcessScopeSafely(state, exited); + }, + ); + + const daemonStopFlights = new WeakMap>(); + const stopCapturedDaemon = Effect.fn("PrimeAgentDaemonManager.stopCapturedDaemon")(function* ( + state: RunningDaemon, + controlPlaneReady = true, + ) { + const existing = daemonStopFlights.get(state); + if (existing !== undefined) return yield* Deferred.await(existing); + const completion = yield* Deferred.make(); + daemonStopFlights.set(state, completion); + return yield* stopCapturedDaemonBody(state, controlPlaneReady).pipe( + Effect.onExit((exit) => + Deferred.done(completion, exit).pipe( + Effect.andThen( + Effect.sync(() => { + if (Exit.isFailure(exit)) daemonStopFlights.delete(state); + }), + ), + Effect.ignore, + ), + ), + ); }); const probeExistingDaemon = Effect.fn("PrimeAgentDaemonManager.probeExistingDaemon")( @@ -582,11 +910,21 @@ export const makePrimeAgentDaemonManager = Effect.fn("makePrimeAgentDaemonManage PrimeAgentDaemonManagerError > { const client = new bridge.DaemonClient(socket); - const connected = yield* Effect.tryPromise({ - try: () => client.connect(timeoutMs), - catch: () => undefined, - }).pipe(Effect.option); + const connected = yield* Effect.promise(() => + client.connect(timeoutMs).then( + () => true, + () => false, + ), + ).pipe(Effect.timeoutOption(timeoutMs)); if (Option.isNone(connected)) { + client.close(); + return yield* managerError( + socket, + "readiness-failed", + "Timed out while probing the private daemon socket.", + ); + } + if (!connected.value) { client.close(); return Option.none(); } @@ -599,7 +937,18 @@ export const makePrimeAgentDaemonManager = Effect.fn("makePrimeAgentDaemonManage "A process accepted the private daemon socket but did not send daemon_hello; refusing to unlink its live socket.", cause, ), - }).pipe(Effect.onError(() => Effect.sync(() => client.close()))); + }).pipe( + Effect.timeoutOrElse({ + duration: timeoutMs, + orElse: () => + managerError( + socket, + "readiness-failed", + "Timed out while validating the private daemon socket listener.", + ), + }), + Effect.onError(() => Effect.sync(() => client.close())), + ); yield* validatedHello(socket, hello).pipe( Effect.onError(() => Effect.sync(() => client.close())), ); @@ -610,12 +959,21 @@ export const makePrimeAgentDaemonManager = Effect.fn("makePrimeAgentDaemonManage const waitForSocketClosure = Effect.fn("PrimeAgentDaemonManager.waitForSocketClosure")( function* () { const client = new bridge.DaemonClient(socket); - const connected = yield* Effect.tryPromise({ - try: () => client.connect(timeoutMs), - catch: () => undefined, - }).pipe(Effect.option); + const connected = yield* Effect.promise(() => + client.connect(timeoutMs).then( + () => true, + () => false, + ), + ).pipe(Effect.timeoutOption(timeoutMs)); client.close(); - if (Option.isSome(connected)) { + if (Option.isNone(connected)) { + return yield* managerError( + socket, + "shutdown-failed", + "Timed out while confirming private daemon socket closure.", + ); + } + if (connected.value) { return yield* managerError( socket, "shutdown-failed", @@ -628,6 +986,14 @@ export const makePrimeAgentDaemonManager = Effect.fn("makePrimeAgentDaemonManage const retireExistingDaemon = Effect.fn("PrimeAgentDaemonManager.retireExistingDaemon")(function* ( client: PrimeAgentDaemonClient, ) { + if (!(yield* isCurrentGeneration)) { + client.close(); + return yield* managerError( + socket, + "readiness-failed", + "This Prime Agent runtime generation was replaced.", + ); + } const response = yield* Effect.tryPromise({ try: () => client.request({ type: "shutdown" }, timeoutMs), catch: (cause) => @@ -637,7 +1003,18 @@ export const makePrimeAgentDaemonManager = Effect.fn("makePrimeAgentDaemonManage "Could not stop the prior Pylon-owned daemon on the stable private socket.", cause, ), - }).pipe(Effect.ensuring(Effect.sync(() => client.close()))); + }).pipe( + Effect.timeoutOrElse({ + duration: shutdownTimeout, + orElse: () => + managerError( + socket, + "shutdown-failed", + "Timed out while requesting private daemon shutdown.", + ), + }), + Effect.ensuring(Effect.sync(() => client.close())), + ); if (!isDaemonSuccessResponse(response)) { return yield* managerError( socket, @@ -645,10 +1022,17 @@ export const makePrimeAgentDaemonManager = Effect.fn("makePrimeAgentDaemonManage "The prior daemon did not acknowledge its public shutdown command.", ); } - yield* waitForSocketClosure().pipe(Effect.retry(readinessSchedule)); + yield* waitForSocketClosure(); }); const startLocked = Effect.fn("PrimeAgentDaemonManager.startLocked")(function* () { + if (!(yield* isStartAllowed)) { + return yield* managerError( + socket, + "readiness-failed", + "This Prime Agent runtime generation was replaced.", + ); + } if (closing) { return yield* managerError( socket, @@ -659,6 +1043,15 @@ export const makePrimeAgentDaemonManager = Effect.fn("makePrimeAgentDaemonManage if (running) { const current = running; const isRunning = yield* current.handle.isRunning.pipe( + Effect.timeoutOrElse({ + duration: shutdownTimeout, + orElse: () => + managerError( + socket, + "process-status-failed", + "Timed out while inspecting the daemon process.", + ), + }), Effect.mapError((cause) => managerError( socket, @@ -678,18 +1071,31 @@ export const makePrimeAgentDaemonManager = Effect.fn("makePrimeAgentDaemonManage ), ); if (Option.isSome(healthClient)) { - return healthClient.value; + if (yield* isStartAllowed) return healthClient.value; + healthClient.value.close(); + return yield* managerError( + socket, + "readiness-failed", + "This Prime Agent runtime generation was replaced.", + ); } running = undefined; yield* stopCapturedDaemon(current, false); } else { running = undefined; yield* removeSocket(); - yield* closeProcessScope(current); + yield* closeProcessScopeSafely(current, true); } } yield* ensurePrivateSocketDirectory(); + if (!(yield* isStartAllowed)) { + return yield* managerError( + socket, + "readiness-failed", + "This Prime Agent runtime generation was replaced.", + ); + } yield* fileSystem.makeDirectory(sessionDir, { recursive: true, mode: 0o700 }).pipe( Effect.andThen(fileSystem.chmod(sessionDir, 0o700)), Effect.mapError((cause) => @@ -716,6 +1122,14 @@ export const makePrimeAgentDaemonManager = Effect.fn("makePrimeAgentDaemonManage "authoritative_owned_session_cleanup_v1", ].every((capability) => hello.value.serverCapabilities.includes(capability)); if (recoverable) { + if (!(yield* isStartAllowed)) { + existing.value.close(); + return yield* managerError( + socket, + "readiness-failed", + "This Prime Agent runtime generation was replaced.", + ); + } retainedExistingDaemon = true; return existing.value; } @@ -725,28 +1139,74 @@ export const makePrimeAgentDaemonManager = Effect.fn("makePrimeAgentDaemonManage yield* removeSocket(); const processScope = yield* Scope.make("sequential"); - const command = ChildProcess.make( - input.executablePath, - ["--mode", "daemon", "--daemon-socket", socket, "--offline", "--session-dir", sessionDir], - recoveryEnabled - ? { - env: launchEnvironment, - extendEnv: false, - detached: true, - stdin: "ignore", - stdout: "ignore", - stderr: "ignore", - } - : { env: launchEnvironment, extendEnv: false }, - ); - const handle = yield* spawner.spawn(command).pipe( + const daemonArgs = [ + "--mode", + "daemon", + "--daemon-socket", + socket, + "--offline", + "--session-dir", + sessionDir, + ] as const; + if (!(yield* isStartAllowed)) { + const closer = yield* Effect.forkDetach(Scope.close(processScope, Exit.void)); + yield* Fiber.await(closer).pipe(Effect.timeoutOption(shutdownTimeout), Effect.ignore); + return yield* managerError( + socket, + "readiness-failed", + "This Prime Agent runtime generation was replaced.", + ); + } + const spawnEffect = + input.spawnProcess?.({ + executablePath: input.executablePath, + args: daemonArgs, + environment: launchEnvironment, + output: recoveryEnabled ? "ignore" : "pipe", + }) ?? + spawnPrimeAgentDaemon({ + executablePath: input.executablePath, + args: daemonArgs, + environment: launchEnvironment, + output: recoveryEnabled ? "ignore" : "pipe", + }); + const handle = yield* spawnEffect.pipe( Effect.provideService(Scope.Scope, processScope), Effect.mapError((cause) => managerError(socket, "spawn-failed", "Could not spawn the Prime Agent daemon.", cause), ), - Effect.onError(() => Scope.close(processScope, Exit.void).pipe(Effect.ignore)), + Effect.onError(() => + Effect.logWarning( + "Prime Agent daemon spawn failed before a process identity could be captured; no process-signaling scope was installed.", + { provider: "primeAgent" }, + ), + ), + ); + const capturedStartIdentity = yield* Effect.tryPromise(() => + inspectProcessIdentity(handle.pid), + ).pipe( + Effect.timeoutOption(timeoutMs), + Effect.orElseSucceed(() => Option.none()), ); - const state = { handle, scope: processScope } satisfies RunningDaemon; + const state = { + handle, + scope: processScope, + startIdentity: Option.getOrUndefined(capturedStartIdentity), + } satisfies RunningDaemon; + starting = state; + if (state.startIdentity === undefined) { + starting = undefined; + yield* handle.unref.pipe(Effect.timeoutOption(shutdownTimeout), Effect.ignore); + yield* Effect.logError( + "Prime Agent daemon start identity was unavailable; refusing any unproved process signal.", + { provider: "primeAgent", pid: handle.pid }, + ); + return yield* managerError( + socket, + "process-status-failed", + "Could not capture the Prime Agent daemon process start identity.", + ); + } if (!recoveryEnabled) { yield* Effect.forkIn(drainProcessOutput(handle.stdout, "stdout"), processScope); yield* Effect.forkIn(drainProcessOutput(handle.stderr, "stderr"), processScope); @@ -757,8 +1217,26 @@ export const makePrimeAgentDaemonManager = Effect.fn("makePrimeAgentDaemonManage while: (error) => error.reason === "readiness-failed", schedule: readinessSchedule, }), - Effect.onError(() => stopCapturedDaemon(state, false)), + Effect.onError(() => + stopCapturedDaemon(state, false).pipe( + Effect.ensuring( + Effect.sync(() => { + if (starting === state) starting = undefined; + }), + ), + ), + ), ); + if (!(yield* isStartAllowed)) { + readinessClient.close(); + yield* stopCapturedDaemon(state, false); + return yield* managerError( + socket, + "readiness-failed", + "This Prime Agent runtime generation was replaced.", + ); + } + starting = undefined; running = state; retainedExistingDaemon = false; return readinessClient; @@ -776,14 +1254,14 @@ export const makePrimeAgentDaemonManager = Effect.fn("makePrimeAgentDaemonManage const openClient = () => semaphore.withPermit(startLocked()); - const shutdown = semaphore.withPermit( + const shutdownLocked = semaphore.withPermit( Effect.gen(function* () { - closing = true; - const captured = running; + const captured = running ?? starting; running = undefined; - if (recoveryRetainers > 0) { - // The standalone process scope is deliberately left open. The daemon is detached - // from this Pylon process and retains the exact same supervisor generation. + starting = undefined; + if (recoveryRetainers > 0 && (yield* isCurrentGeneration)) { + // Server shutdown retains exact #84 recovery authority. A material replacement + // has already retired this generation and must contain only its private daemon. return; } if (captured) { @@ -800,10 +1278,31 @@ export const makePrimeAgentDaemonManager = Effect.fn("makePrimeAgentDaemonManage } }), ); - yield* Effect.addFinalizer(() => shutdown.pipe(Effect.ignore)); + const shutdown = Effect.gen(function* () { + closing = true; + const overallTimeoutMs = shutdownTimeoutMs * 8; + const settled = yield* shutdownLocked.pipe(Effect.timeoutOption(overallTimeoutMs)); + if (Option.isSome(settled)) return; + yield* Effect.logWarning("Prime Agent daemon shutdown reached its total bound.", { + provider: "primeAgent", + timeoutMs: overallTimeoutMs, + }); + if (recoveryRetainers > 0 && (yield* isCurrentGeneration)) return; + const captured = running ?? starting; + running = undefined; + starting = undefined; + if (captured !== undefined) yield* stopCapturedDaemon(captured).pipe(Effect.ignore); + }); + + yield* Effect.addFinalizer(() => + // Scope finalizers are uninterruptible. A fresh runtime fiber preserves the + // interruptibility required by each teardown phase timeout. + Effect.promise(() => runPromise(shutdown).catch(() => undefined)), + ); return { bridge, + identity: input.identity, socket, sessionDir, prepare, diff --git a/apps/server/src/provider/prime/PrimeAgentDaemonSessionRuntime.test.ts b/apps/server/src/provider/prime/PrimeAgentDaemonSessionRuntime.test.ts index 6d7eb6b49..5602ee605 100644 --- a/apps/server/src/provider/prime/PrimeAgentDaemonSessionRuntime.test.ts +++ b/apps/server/src/provider/prime/PrimeAgentDaemonSessionRuntime.test.ts @@ -1,7 +1,13 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + import { describe, expect, it } from "@effect/vitest"; import { PROVIDER_AGENT_CONTROL_ID_MAX_CHARS, PROVIDER_SESSION_AGENT_MESSAGE_MAX_CHARS, + ProviderInstanceId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; @@ -28,12 +34,14 @@ import { type PrimeAgentDaemonExtensionUiResponse, type PrimeAgentDaemonImage, type PrimeAgentDaemonPromptOptions, + type PrimeAgentOwnedSessionContractProof, type PrimeAgentDaemonQueuedMessageMutation, type PrimeAgentDaemonSessionWatcher, type PrimeAgentDaemonServiceTier, type PrimeAgentDaemonThinkingLevel, } from "./PrimeAgentDaemonBridge.ts"; import type { PrimeAgentDaemonManager } from "./PrimeAgentDaemonManager.ts"; +import { PrimeAgentOwnershipReceiptStore } from "./PrimeAgentOwnershipReceipt.ts"; import { PRIME_AGENT_PLAN_TOOL_DEFINITION } from "./PrimeAgentManagedExtension.ts"; import { PRIME_AGENT_EVENT_BUFFER_CAPACITY } from "./PrimeAgentEventBuffer.ts"; import { @@ -279,6 +287,7 @@ function fixture(options?: { readonly correlatedPromptLifecycleProof?: boolean; readonly omitNegotiatedCapabilityAccessor?: boolean; readonly negotiatedCapabilityProofImpl?: () => boolean; + readonly ownedSessionContractProofImpl?: () => PrimeAgentOwnedSessionContractProof | undefined; readonly submitCorrelatedPromptImpl?: ( message: string, options: { @@ -302,6 +311,7 @@ function fixture(options?: { readonly unsubscribeImpl?: () => void; readonly disposeImpl?: () => Promise; readonly recoveryMode?: "create" | "adopt"; + readonly ownershipStore?: PrimeAgentOwnershipReceiptStore; }) { const captures: Captures = { order: [], @@ -576,11 +586,28 @@ function fixture(options?: { captures.connectionCalls.push({ method: "getPromptLifecycles", args: [] }); return options?.getPromptLifecyclesImpl?.() ?? Promise.resolve({ records: [], expired: [] }); } - supportsNegotiatedCapability(capability: "correlated_prompt_lifecycle_v1"): boolean { - return ( - capability === "correlated_prompt_lifecycle_v1" && - (options?.negotiatedCapabilityProofImpl?.() ?? correlatedPromptLifecycleProof) - ); + supportsNegotiatedCapability(capability: string): boolean { + return capability === "caller_owned_session_environment_cleanup_v1" + ? true + : capability === "correlated_prompt_lifecycle_v1" && + (options?.negotiatedCapabilityProofImpl?.() ?? correlatedPromptLifecycleProof); + } + getOwnedSessionContractProof(): PrimeAgentOwnedSessionContractProof | undefined { + if (options?.ownedSessionContractProofImpl !== undefined) { + return options.ownedSessionContractProofImpl(); + } + return { + feature: "caller_owned_session_environment_cleanup_v1" as const, + status: "attached" as const, + daemon: { + protocolName: "prime-agent.daemon", + protocolVersion: 7, + schemaRevision: 30, + appVersion: "0.7.1", + supervisorGeneration: "supervisor-1", + transportGeneration: 0, + }, + }; } waitForHeadlessCompletion( waitOptions: { readonly waitForRlmQuiescence?: boolean } = {}, @@ -862,9 +889,18 @@ function fixture(options?: { Promise.resolve({ maxDepth, source: "chat", globalSaved: false }) ); } - disposeOwnedSession(): Promise { + async disposeOwnedSession(): Promise { captures.order.push("dispose-owned"); - return Promise.resolve({ status: "completed" }); + captures.disposeCount += 1; + await options?.disposeImpl?.(); + const started = this.getOwnedSessionContractProof(); + return { + feature: "caller_owned_session_environment_cleanup_v1", + status: "completed", + started, + observed: started?.daemon, + daemonReplaced: false, + }; } dispose(): Promise { captures.order.push("dispose"); @@ -932,8 +968,24 @@ function fixture(options?: { DaemonAgentConnection: FakeConnection, defaultDaemonSocketPath: () => "/tmp/prime-agent.sock", }; + const identity = { + instanceId: ProviderInstanceId.make("primeAgent"), + generation: { _tag: "PrimeAgentRuntimeGeneration" as const }, + configRevision: "runtime-test-revision", + effectiveHome: "/state/prime-agent-home", + launchEnv: { HOME: "/private/home" }, + nativeMultipleInstancesRequired: false, + settings: { + enabled: true, + binaryPath: "prime-agent", + agentHomePath: "/state/prime-agent-home", + launchArgs: "", + customModels: [], + }, + }; const manager: PrimeAgentDaemonManager = { bridge, + identity, socket: "/tmp/pylon-prime.sock", sessionDir: "/state/shared-daemon-sessions", launchEnvironment: { HOME: "/private/home" }, @@ -965,6 +1017,31 @@ function fixture(options?: { ) => makePrimeAgentDaemonSessionRuntime({ manager, + runtimeContext: { + ...identity, + backendKind: "daemon", + backendIdentity: { + kind: "daemon", + proof: { + sdkFeatures: [ + "negotiated_daemon_session_capabilities_v1", + "caller_owned_session_environment_cleanup_v1", + ], + requiredServerCapabilities: [ + "caller_owned_session_environment_cleanup_v1", + "authoritative_owned_session_cleanup_v1", + ], + }, + }, + ...(options?.ownershipStore === undefined + ? {} + : { + nativeOwnership: { + store: options.ownershipStore, + adoptableReceipts: [], + }, + }), + }, cwd: "/work/project", sessionDir: "/state/provider-sessions/thread-safe", agentDir: "/state/prime-agent-home", @@ -1042,6 +1119,51 @@ function captureNextSetConstruction(run: () => A): readonly [A, Set] } describe("PrimeAgentDaemonSessionRuntime", () => { + it.effect("rejects a missing current owned-attachment proof before prompt admission", () => + Effect.gen(function* () { + const test = fixture({ ownedSessionContractProofImpl: () => undefined }); + + const error = yield* Effect.flip(Effect.scoped(test.make())); + + expect(error).toMatchObject({ + operation: "attach-session", + reason: "invalid-response", + }); + expect(error.detail).not.toContain("active-secret-1"); + expect(test.captures.connectionCalls.some((call) => call.method === "prompt")).toBe(false); + expect(test.captures.disposeCount).toBe(1); + expect(test.captures.closeCount).toBe(1); + }), + ); + + it.effect("rejects a current proof that contradicts the successful daemon hello", () => + Effect.gen(function* () { + const test = fixture({ + ownedSessionContractProofImpl: () => ({ + feature: "caller_owned_session_environment_cleanup_v1", + status: "attached", + daemon: { + protocolName: "prime-agent.daemon", + protocolVersion: 7, + schemaRevision: 30, + appVersion: "0.7.1", + supervisorGeneration: "stale-supervisor", + transportGeneration: 0, + }, + }), + }); + + const error = yield* Effect.flip(Effect.scoped(test.make())); + + expect(error).toMatchObject({ + operation: "attach-session", + reason: "invalid-response", + }); + expect(error.detail).not.toContain("stale-supervisor"); + expect(test.captures.disposeCount).toBe(1); + }), + ); + it.effect("keeps a server offer on the ordinary prompt path without the frozen SDK feature", () => Effect.scoped( Effect.gen(function* () { @@ -1060,6 +1182,9 @@ describe("PrimeAgentDaemonSessionRuntime", () => { expect( test.captures.connectionCalls.filter((call) => call.method === "submitCorrelatedPrompt"), ).toHaveLength(0); + expect( + test.captures.commands.some((command) => command.type === "complete_owned_session"), + ).toBe(false); }), ), ); @@ -3249,6 +3374,8 @@ describe("PrimeAgentDaemonSessionRuntime", () => { { type: "create", lifecycle: "client_owned", + launchEnv: { HOME: "/private/home" }, + launchEnvMode: "replace", continueRecent: false, config: { cwd: "/work/project", @@ -3296,6 +3423,50 @@ describe("PrimeAgentDaemonSessionRuntime", () => { }), ); + it.effect("persists native ownership before create and clears it only after proved cleanup", () => + Effect.scoped( + Effect.gen(function* () { + const root = yield* Effect.acquireRelease( + Effect.promise(() => + NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "prime-owner-order-")), + ), + (directory) => + Effect.promise(() => NodeFSP.rm(directory, { recursive: true, force: true })), + ); + class ObservedOwnershipStore extends PrimeAgentOwnershipReceiptStore { + receiptWritten = false; + override async begin(input: Parameters[0]) { + const handle = await super.begin(input); + this.receiptWritten = true; + return handle; + } + } + const ownershipStore = new ObservedOwnershipStore(root, { + inspectProcessIdentity: async (pid) => `test:${pid}`, + }); + let receiptPresentBeforeCreate = false; + const side = fixture({ + ownershipStore, + createRequestObserved: () => { + receiptPresentBeforeCreate = ownershipStore.receiptWritten; + }, + }); + const runtime = yield* side.make(); + + expect(receiptPresentBeforeCreate).toBe(true); + expect(yield* Effect.promise(() => ownershipStore.scan())).toMatchObject({ + corrupt: false, + receipts: [{ state: "acquired" }], + }); + yield* runtime.dispose; + expect(yield* Effect.promise(() => ownershipStore.scan())).toMatchObject({ + corrupt: false, + receipts: [], + }); + }), + ), + ); + it.effect("attaches Pylon's scoped MCP server before the initial snapshot and releases it", () => Effect.gen(function* () { const { captures, make } = fixture(); @@ -3334,7 +3505,9 @@ describe("PrimeAgentDaemonSessionRuntime", () => { method: "releaseAcpMcpServers", args: [mcpServer.ownerId, [mcpServer.server.name]], }); - expect(captures.order.indexOf("release-mcp")).toBeLessThan(captures.order.indexOf("dispose")); + expect(captures.order.indexOf("release-mcp")).toBeLessThan( + captures.order.indexOf("dispose-owned"), + ); expect(captures.disposeCount).toBe(1); expect(captures.closeCount).toBe(1); }), @@ -12630,7 +12803,7 @@ describe("Prime Agent live activity privacy boundary", () => { schedulerChecks += 1; // The first checks enter dispose and elect this fiber. Yield once // immediately afterward, before the returned owner onExit is installed. - return schedulerChecks === 3; + return schedulerChecks === 4; }, makeDispatcher: () => ({ scheduleTask: (task) => { @@ -12681,7 +12854,7 @@ describe("Prime Agent live activity privacy boundary", () => { ), Effect.forkChild({ startImmediately: true }), ); - expect(schedulerChecks).toBe(3); + expect(schedulerChecks).toBe(4); expect(scheduledTasks).toHaveLength(1); const cancelledWaiter = yield* runtime.dispose.pipe( @@ -12733,10 +12906,10 @@ describe("Prime Agent live activity privacy boundary", () => { expect(Exit.isFailure(ownerExit)).toBe(true); if (Exit.isFailure(ownerExit)) expect(Cause.hasInterruptsOnly(ownerExit.cause)).toBe(true); - expect(sharedWaiterExit).toEqual(ownerExit); - expect(lateExit).toEqual(ownerExit); + expect(sharedWaiterExit).toEqual(Exit.void); + expect(lateExit).toEqual(Exit.void); const scopeCloseExit = yield* Scope.close(sessionScope, Exit.void).pipe(Effect.exit); - expect(Exit.isFailure(scopeCloseExit)).toBe(true); + expect(scopeCloseExit).toEqual(Exit.void); if (Exit.isFailure(scopeCloseExit)) { expect(Cause.hasInterruptsOnly(scopeCloseExit.cause)).toBe(true); } @@ -12751,7 +12924,7 @@ describe("Prime Agent live activity privacy boundary", () => { expect(side.captures.order).toEqual([ "retire", "unsubscribe", - "dispose", + "dispose-owned", "close", "queue-shutdown", "queue-shutdown", diff --git a/apps/server/src/provider/prime/PrimeAgentDaemonSessionRuntime.ts b/apps/server/src/provider/prime/PrimeAgentDaemonSessionRuntime.ts index af862da3c..9b5bbca95 100644 --- a/apps/server/src/provider/prime/PrimeAgentDaemonSessionRuntime.ts +++ b/apps/server/src/provider/prime/PrimeAgentDaemonSessionRuntime.ts @@ -25,18 +25,23 @@ import { } from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Option from "effect/Option"; import * as Predicate from "effect/Predicate"; import * as Queue from "effect/Queue"; +import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import { + PRIME_AGENT_CALLER_OWNED_SESSION_ENVIRONMENT_CLEANUP_FEATURE, + PRIME_AGENT_DAEMON_PROTOCOL_NAME, type PrimeAgentDaemonAcpMcpServer, type PrimeAgentDaemonAgentConnection, + type PrimeAgentDaemonClient, type PrimeAgentDaemonEventCursor, type PrimeAgentRecoverableOwnedSessionAdoptionProof, type PrimeAgentDaemonExtensionUiResponse, @@ -45,6 +50,8 @@ import { type PrimeAgentDaemonQueuedMessageLane, type PrimeAgentDaemonServiceTier, type PrimeAgentDaemonThinkingLevel, + type PrimeAgentOwnedSessionContractProof, + type PrimeAgentOwnedSessionDisposeResult, } from "./PrimeAgentDaemonBridge.ts"; import { decodePrimeAgentDaemonChildren, @@ -66,6 +73,8 @@ import { type PrimeDaemonUsage, } from "./PrimeAgentDaemonEvents.ts"; import type { PrimeAgentDaemonManager } from "./PrimeAgentDaemonManager.ts"; +import type { PrimeAgentOwnershipReceiptHandle } from "./PrimeAgentOwnershipReceipt.ts"; +import type { PrimeAgentRuntimeContext } from "./PrimeAgentRuntimeContext.ts"; import { PRIME_AGENT_PLAN_TOOL_DEFINITION, PRIME_AGENT_PLAN_TOOL_NAME, @@ -854,6 +863,9 @@ export class PrimeAgentDaemonSessionRuntimeError extends Schema.TaggedErrorClass export interface PrimeAgentDaemonSessionRuntimeInput { readonly manager: PrimeAgentDaemonManager; + readonly runtimeContext?: PrimeAgentRuntimeContext; + /** Exact dirty receipt claimed by identity-safe restart adoption. */ + readonly ownershipReceipt?: PrimeAgentOwnershipReceiptHandle; readonly cwd: string; /** Isolated, deterministic, server-owned directory for this Pylon thread. */ readonly sessionDir: string; @@ -889,6 +901,9 @@ export interface PrimeAgentDaemonSessionRuntimeInput { | { readonly kind: "create"; readonly requestId: string; + readonly threadId?: string; + readonly sessionIncarnationId?: string; + readonly admissionRequestId?: string; readonly correlationId: string; readonly mcpOwnerId: string; readonly onAuthorityReady: (authority: { @@ -1196,7 +1211,12 @@ export interface PrimeAgentDaemonSessionRuntime { ) => PrimeAgentDaemonEventCursor | undefined; /** Close only this Pylon owner. The recoverable worker and MCP authority stay with the daemon. */ readonly detach?: Effect.Effect; - /** Explicit cleanup requires Prime's authoritative owned-session result. */ + /** Preserves Prime's fixed authoritative cleanup outcome for private lifecycle decisions. */ + readonly disposeOwnedSession?: Effect.Effect< + PrimeAgentOwnedSessionDisposeResult, + PrimeAgentDaemonSessionRuntimeError + >; + /** Compatibility view: only proved settlement succeeds. */ readonly dispose: Effect.Effect; } @@ -1369,6 +1389,55 @@ function recoveryCursorFromSnapshot(value: unknown): PrimeAgentDaemonEventCursor : undefined; } +function currentOwnedSessionContractProof( + connection: PrimeAgentDaemonAgentConnection, + client: PrimeAgentDaemonClient, +): PrimeAgentOwnedSessionContractProof | undefined { + try { + if ( + !Predicate.isFunction(connection.getOwnedSessionContractProof) || + !Predicate.isFunction(connection.supportsNegotiatedCapability) || + connection.supportsNegotiatedCapability( + PRIME_AGENT_CALLER_OWNED_SESSION_ENVIRONMENT_CLEANUP_FEATURE, + ) !== true + ) { + return undefined; + } + const proof = connection.getOwnedSessionContractProof(); + const hello = client.hello; + if (proof === undefined || hello === undefined) return undefined; + return proof.feature === PRIME_AGENT_CALLER_OWNED_SESSION_ENVIRONMENT_CLEANUP_FEATURE && + proof.status === "attached" && + proof.daemon.protocolName === PRIME_AGENT_DAEMON_PROTOCOL_NAME && + proof.daemon.protocolName === hello.protocol.name && + proof.daemon.protocolVersion === hello.protocol.version && + Number.isSafeInteger(proof.daemon.protocolVersion) && + proof.daemon.schemaRevision === hello.schemaRevision && + Number.isSafeInteger(proof.daemon.schemaRevision) && + proof.daemon.supervisorGeneration.length > 0 && + proof.daemon.supervisorGeneration === hello.supervisorGeneration && + Number.isSafeInteger(proof.daemon.transportGeneration) && + proof.daemon.transportGeneration >= 0 && + hello.serverCapabilities.includes( + PRIME_AGENT_CALLER_OWNED_SESSION_ENVIRONMENT_CLEANUP_FEATURE, + ) && + hello.serverCapabilities.includes("authoritative_owned_session_cleanup_v1") && + (hello.appVersion === undefined || proof.daemon.appVersion === hello.appVersion) && + (hello.buildId === undefined || proof.daemon.buildId === hello.buildId) + ? proof + : undefined; + } catch { + return undefined; + } +} + +function hasCurrentOwnedSessionContractProof( + connection: PrimeAgentDaemonAgentConnection, + client: PrimeAgentDaemonClient, +): boolean { + return currentOwnedSessionContractProof(connection, client) !== undefined; +} + export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemonSessionRuntime")( function* ( input: PrimeAgentDaemonSessionRuntimeInput, @@ -1377,6 +1446,21 @@ export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemo PrimeAgentDaemonSessionRuntimeError, Scope.Scope > { + const primeRuntimeContext = input.runtimeContext; + if ( + primeRuntimeContext === undefined || + primeRuntimeContext.backendKind !== "daemon" || + primeRuntimeContext.instanceId !== input.manager.identity.instanceId || + primeRuntimeContext.configRevision !== input.manager.identity.configRevision || + primeRuntimeContext.effectiveHome !== input.manager.identity.effectiveHome || + primeRuntimeContext.launchEnv !== input.manager.identity.launchEnv + ) { + return yield* runtimeError( + "create-session", + "invalid-input", + "The Prime Agent runtime context does not own this daemon manager.", + ); + } const cwd = yield* validateNonEmpty("create-session", "cwd", input.cwd); const sessionDir = yield* validateNonEmpty("create-session", "sessionDir", input.sessionDir); const shouldContinue = input.resumeCursor !== undefined; @@ -1424,7 +1508,53 @@ export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemo let disposed = false; let detached = false; let disposeStarted = false; - const disposeCompletion = yield* Deferred.make(); + const ownershipStore = primeRuntimeContext.nativeOwnership?.store; + let ownershipReceipt = input.ownershipReceipt; + if ( + ownershipReceipt !== undefined && + (input.recovery?.kind !== "adopt" || + ownershipReceipt.instanceId !== primeRuntimeContext.instanceId || + ownershipReceipt.currentConfigRevision !== primeRuntimeContext.configRevision || + ownershipReceipt.effectiveHome !== primeRuntimeContext.effectiveHome) + ) { + client.close(); + return yield* runtimeError( + "create-session", + "invalid-input", + "The native ownership receipt does not belong to this adoption generation.", + ); + } + if ( + ownershipStore !== undefined && + input.recovery?.kind === "adopt" && + ownershipReceipt === undefined + ) { + client.close(); + return yield* runtimeError( + "create-session", + "invalid-input", + "Recoverable adoption is missing its exact native ownership receipt.", + ); + } + const beginOwnershipReceipt = () => + Effect.tryPromise({ + try: () => + ownershipStore!.begin({ + instanceId: primeRuntimeContext.instanceId, + configRevision: primeRuntimeContext.configRevision, + effectiveHome: primeRuntimeContext.effectiveHome, + }), + catch: () => + runtimeError( + "create-session", + "request-failed", + "Could not durably record native ownership before session creation.", + ), + }).pipe(Effect.tap((receipt) => Effect.sync(() => (ownershipReceipt = receipt)))); + const disposeCompletion = yield* Deferred.make< + PrimeAgentOwnedSessionDisposeResult | undefined, + PrimeAgentDaemonSessionRuntimeError + >(); let needsResumeAfterAbort = false; let connectionGeneration = 0; type RouteRetirementSignal = { @@ -1881,6 +2011,7 @@ export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemo "Recoverable Prime Agent execution is unavailable.", ); } + if (ownershipStore !== undefined) yield* beginOwnershipReceipt(); const created = yield* Effect.tryPromise({ try: () => createRecoverableOwnedSession(client, { @@ -1996,9 +2127,12 @@ export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemo sessionFile = recovery.sessionFile; adoptedRecovery = { recoveryHandle: adopted.recoveryHandle, proof: adopted.proof }; } else { + if (ownershipStore !== undefined) yield* beginOwnershipReceipt(); const createCommand = { type: "create", lifecycle: "client_owned", + launchEnv: input.manager.launchEnvironment ?? {}, + launchEnvMode: "replace", ...(resumeSessionId === undefined ? { continueRecent: shouldContinue } : { sessionPath: resumeSessionId, continueRecent: false }), @@ -2027,6 +2161,16 @@ export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemo if (Option.isNone(created)) { yield* closeClient; const alreadyActive = decodeCreateSessionAlreadyActiveFailure(createResponse); + const failed = decodeCreateFailure(createResponse); + if ( + ownershipStore !== undefined && + ownershipReceipt !== undefined && + (Option.isSome(alreadyActive) || Option.isSome(failed)) + ) { + yield* Effect.tryPromise(() => ownershipStore.proveNeverAcquired(ownershipReceipt!)).pipe( + Effect.ignore, + ); + } if (Option.isSome(alreadyActive)) { return yield* runtimeError( "create-session", @@ -2034,7 +2178,6 @@ export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemo "SessionAlreadyActiveError: Prime Agent session is already active in another client.", ); } - const failed = decodeCreateFailure(createResponse); return yield* runtimeError( "create-session", Option.isSome(failed) ? "request-failed" : "invalid-response", @@ -2078,6 +2221,7 @@ export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemo supportsExtensionUi: true, ownedSession: true, ownedSessionRecoveryConfig: sessionRuntimeConfig, + ownedSessionLaunchEnv: input.manager.launchEnvironment ?? {}, ...(input.disableAutoReconnect === true ? {} : { recoverDaemon: input.manager.recover }), @@ -2111,6 +2255,83 @@ export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemo await connection?.dispose().catch(() => undefined); client.close(); }); + const initialOwnedSessionContractProof = currentOwnedSessionContractProof(connection, client); + if (initialOwnedSessionContractProof === undefined) { + yield* closeUnusableAttachedConnection; + if (ownershipReceipt !== undefined) ownershipStore?.markUnsafe(ownershipReceipt); + return yield* runtimeError( + "attach-session", + "invalid-response", + "Prime Agent did not prove the required caller-owned session contract for this attachment.", + ); + } + if (ownershipStore !== undefined && ownershipReceipt !== undefined) { + const recoveryIdentity = + input.recovery?.kind === "create" && + createdRecovery !== undefined && + input.recovery.threadId !== undefined && + input.recovery.sessionIncarnationId !== undefined && + input.recovery.admissionRequestId !== undefined + ? { + threadId: input.recovery.threadId, + sessionIncarnationId: input.recovery.sessionIncarnationId, + admissionRequestId: input.recovery.admissionRequestId, + recoveryHandle: createdRecovery.recoveryHandle, + ownershipGeneration: createdRecovery.ownershipGeneration, + } + : undefined; + const ownershipUpdate = + input.recovery?.kind === "adopt" + ? ownershipStore.refreshAttachProof(ownershipReceipt, initialOwnedSessionContractProof) + : ownershipStore + .markAcquired(ownershipReceipt, { + activeSessionId, + nativeSessionId: sessionId, + attachProof: initialOwnedSessionContractProof, + ...(recoveryIdentity === undefined ? {} : { recovery: recoveryIdentity }), + }) + .then((handle) => { + ownershipReceipt = handle; + }); + const recorded = yield* Effect.tryPromise(() => ownershipUpdate).pipe(Effect.result); + if (Result.isFailure(recorded)) { + ownershipStore.markUnsafe(ownershipReceipt); + yield* closeUnusableAttachedConnection; + return yield* runtimeError( + "attach-session", + "request-failed", + "Prime Agent native ownership could not be durably acquired.", + ); + } + } + let ownedSessionContractProofCurrent = true; + const requireCurrentOwnedSessionContract = (operation: "prompt" | "steer" | "follow-up") => + Effect.suspend(() => { + const proof = currentOwnedSessionContractProof(connection!, client); + ownedSessionContractProofCurrent = ownedSessionContractProofCurrent && proof !== undefined; + if (!ownedSessionContractProofCurrent || proof === undefined) { + if (ownershipReceipt !== undefined) ownershipStore?.markUnsafe(ownershipReceipt); + return runtimeError( + operation, + "incompatible-api", + "Prime Agent no longer proves the caller-owned session contract for this attachment.", + ); + } + return ownershipStore === undefined || ownershipReceipt === undefined + ? Effect.void + : Effect.tryPromise(() => + ownershipStore.refreshAttachProof(ownershipReceipt!, proof), + ).pipe( + Effect.mapError(() => + runtimeError( + operation, + "request-failed", + "Prime Agent native ownership proof could not be durably refreshed.", + ), + ), + ); + }); + if (input.manager.bridge.negotiatedDaemonSessionCapabilitiesAvailable) { if (!Predicate.isFunction(connection.supportsNegotiatedCapability)) { yield* closeUnusableAttachedConnection; @@ -4449,6 +4670,18 @@ export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemo Predicate.isObject(raw) && "type" in raw && Predicate.isString(raw.type) ? raw.type : undefined; + const rawStatus = + rawType === "connection_status" && + Predicate.isObject(raw) && + "status" in raw && + Predicate.isString(raw.status) + ? raw.status + : undefined; + if (rawStatus === "reconnecting") { + ownedSessionContractProofCurrent = false; + } else if (rawStatus === "connected" || rawType === "session_resynced") { + ownedSessionContractProofCurrent = hasCurrentOwnedSessionContractProof(connection!, client); + } if (correlatedPromptLifecycleAvailable) { if (rawType !== "closed") return routeWorkerAwareRawEvent(raw); if (correlatedWorkerCloseRoute !== undefined) return correlatedWorkerCloseRoute; @@ -6914,6 +7147,7 @@ export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemo } yield* awaitProviderRecovery; yield* refreshMcpAfterAdoption(); + yield* requireCurrentOwnedSessionContract("prompt"); const resumedAfterAbort = yield* resumeAfterAbort(); const images = yield* validateImages("prompt", promptInput.images); yield* validatePromptContent("prompt", promptInput.text, images); @@ -7031,6 +7265,7 @@ export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemo yield* validatePromptContent("steer", promptInput.text, images); const recovery = yield* inputAdmissionAfterRecovery("steer"); if (recovery === "recovering") return recovery; + yield* requireCurrentOwnedSessionContract("steer"); const method = yield* requireMethod("steer", connection!.steer); yield* callVoid("steer", () => method.call(connection, promptInput.text, images)); return "accepted" as const; @@ -7044,6 +7279,7 @@ export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemo yield* validatePromptContent("follow-up", promptInput.text, images); const recovery = yield* inputAdmissionAfterRecovery("follow-up"); if (recovery === "recovering") return recovery; + yield* requireCurrentOwnedSessionContract("follow-up"); const method = yield* requireMethod("follow-up", connection!.followUp); yield* callVoid("follow-up", () => method.call(connection, promptInput.text, images)); return "accepted" as const; @@ -7907,60 +8143,67 @@ export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemo }), ); - const dispose = Effect.uninterruptibleMask((restore) => { + const disposeOwnedSession = Effect.uninterruptibleMask((restore) => { // Scope cleanup and explicit teardown can race. Every caller joins the - // first bounded native disposal instead of treating "started" as done. - if (disposeStarted) return restore(Deferred.await(disposeCompletion)); - disposeStarted = true; - - const recoveryOwned = input.recovery !== undefined; - const nativeDispose = recoveryOwned - ? Effect.tryPromise({ - try: async () => { - const cleanup = connection?.disposeOwnedSession; - if (!Predicate.isFunction(cleanup)) { - throw new Error("authoritative cleanup is unavailable"); - } - const result = await cleanup.call(connection, { timeoutMs: COMMAND_TIMEOUT_MS }); - if ( - !Predicate.isObject(result) || - (result.status !== "completed" && result.status !== "already_completed") - ) { - throw new Error("authoritative cleanup was not proven"); - } - }, - catch: () => - runtimeError( + // first exact generation/attachment/session disposal. + const awaitDisposeCompletion = restore(Deferred.await(disposeCompletion)).pipe( + Effect.flatMap((outcome) => + outcome === undefined + ? runtimeError( "dispose", "request-failed", - "Prime Agent could not prove authoritative native cleanup.", - ), - }).pipe( - Effect.tap(() => - Effect.sync(() => { - releaseManagerRecoveryRetention?.(); - releaseManagerRecoveryRetention = undefined; - }), - ), - ) - : Effect.tryPromise({ - try: () => connection!.dispose(), - catch: () => - runtimeError("dispose", "request-failed", "Could not dispose the daemon session."), - }).pipe( - Effect.flatMap((output) => - output === undefined - ? Effect.void - : Effect.fail( - runtimeError( - "dispose", - "invalid-response", - "The daemon dispose operation returned an invalid response.", - ), - ), - ), - ); - const boundedNativeDispose = nativeDispose.pipe( + "The owned session was detached without authoritative cleanup.", + ) + : Effect.succeed(outcome), + ), + ); + if (disposeStarted) return awaitDisposeCompletion; + disposeStarted = true; + if (ownershipStore !== undefined && ownershipReceipt !== undefined) { + ownershipStore.markUnsafe(ownershipReceipt); + } + + const nativeDispose = Effect.tryPromise({ + try: async () => { + const cleanup = connection?.disposeOwnedSession; + if (!Predicate.isFunction(cleanup)) { + return { + feature: PRIME_AGENT_CALLER_OWNED_SESSION_ENVIRONMENT_CLEANUP_FEATURE, + status: "unsupported", + } satisfies PrimeAgentOwnedSessionDisposeResult; + } + const proof = currentOwnedSessionContractProof(connection!, client); + if (proof !== undefined) { + if (ownershipStore !== undefined && ownershipReceipt !== undefined) { + await ownershipStore.refreshAttachProof(ownershipReceipt, proof); + } + } + const result = await cleanup.call(connection, { timeoutMs: COMMAND_TIMEOUT_MS }); + if ( + !Predicate.isObject(result) || + result.feature !== PRIME_AGENT_CALLER_OWNED_SESSION_ENVIRONMENT_CLEANUP_FEATURE || + !Predicate.isString(result.status) || + ![ + "completed", + "already_completed", + "replacement_settled", + "owner_mismatch", + "uncertain", + "transport_failure", + "unsupported", + ].includes(result.status) + ) { + throw new Error("authoritative cleanup returned an invalid outcome"); + } + return result as unknown as PrimeAgentOwnedSessionDisposeResult; + }, + catch: () => + runtimeError( + "dispose", + "request-failed", + "Prime Agent could not return an authoritative native cleanup outcome.", + ), + }).pipe( Effect.timeoutOrElse({ duration: COMMAND_TIMEOUT_MS, orElse: () => @@ -7970,10 +8213,39 @@ export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemo "Timed out while disposing the daemon session.", ), }), + Effect.tap((result) => + Effect.gen(function* () { + const settled = + result.status === "completed" || + result.status === "already_completed" || + result.status === "replacement_settled"; + if (!settled) { + if (ownershipReceipt !== undefined) ownershipStore?.markUnsafe(ownershipReceipt); + return; + } + if (ownershipStore !== undefined && ownershipReceipt !== undefined) { + const cleared = yield* Effect.tryPromise(() => + ownershipStore.clearAfterCleanup(ownershipReceipt!, { + activeSessionId, + nativeSessionId: sessionId, + result, + }), + ).pipe(Effect.orElseSucceed(() => false)); + if (!cleared) { + ownershipStore.markUnsafe(ownershipReceipt); + return yield* runtimeError( + "dispose", + "invalid-response", + "Authoritative cleanup did not match the exact native ownership receipt.", + ); + } + } + releaseManagerRecoveryRetention?.(); + releaseManagerRecoveryRetention = undefined; + }), + ), ); - const beginDisposeOwner = retireLocalOwner.pipe( - Effect.map(() => [...activePrivateSideQuestions.entries()] as const), - ); + const nativeSideQuestions = [...activePrivateSideQuestions.entries()] as const; const disposeOwnerBody = ( nativeSideQuestions: ReadonlyArray, ) => @@ -7981,11 +8253,19 @@ export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemo bestEffortAbortSideQuestion(nativeId, active), ).pipe( Effect.andThen(failActivePrivateSideQuestions()), - Effect.andThen(recoveryOwned ? Effect.void : releaseMcpServer), + Effect.andThen(input.recovery === undefined ? releaseMcpServer : Effect.void), + Effect.timeoutOption(COMMAND_TIMEOUT_MS), + Effect.asVoid, ); - return beginDisposeOwner.pipe( - Effect.flatMap((nativeSideQuestions) => restore(disposeOwnerBody(nativeSideQuestions))), - Effect.onExit(() => boundedNativeDispose), + const work = Effect.gen(function* () { + const retirement = yield* retireLocalOwner.pipe(Effect.exit); + const localCleanup = yield* disposeOwnerBody(nativeSideQuestions).pipe(Effect.exit); + const nativeCleanup = yield* nativeDispose.pipe(Effect.exit); + if (Exit.isFailure(retirement)) return yield* Effect.failCause(retirement.cause); + if (Exit.isFailure(localCleanup)) return yield* Effect.failCause(localCleanup.cause); + return yield* nativeCleanup; + }).pipe( + Effect.interruptible, Effect.ensuring( Effect.gen(function* () { disposed = true; @@ -7999,9 +8279,32 @@ export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemo ), Effect.onExit((exit) => Deferred.done(disposeCompletion, exit).pipe(Effect.ignore)), ); + return Effect.gen(function* () { + // Cleanup authority belongs to the exact session, not to the first caller's + // interruptibility. Install one detached cleanup fiber before any caller waits. + yield* Effect.sync(() => { + void runPromise(work).catch(() => undefined); + }); + return yield* awaitDisposeCompletion; + }); }); + const dispose = disposeOwnedSession.pipe( + Effect.flatMap((result) => + result.status === "completed" || + result.status === "already_completed" || + result.status === "replacement_settled" + ? Effect.void + : runtimeError( + "dispose", + result.status === "uncertain" ? "request-timed-out" : "request-failed", + `Prime Agent native cleanup did not settle (${result.status}).`, + ), + ), + ); - yield* Effect.addFinalizer(() => (detached ? Effect.void : dispose.pipe(Effect.ignore))); + yield* Effect.addFinalizer(() => + detached ? Effect.void : disposeOwnedSession.pipe(Effect.ignore), + ); return { resumeCursor: PRIME_AGENT_DAEMON_RESUME_CURSOR, @@ -8115,6 +8418,7 @@ export const makePrimeAgentDaemonSessionRuntime = Effect.fn("makePrimeAgentDaemo }), recoveryCursorForEvent: (event) => consumedRecoveryCursors.get(event), detach, + disposeOwnedSession, dispose, } satisfies PrimeAgentDaemonSessionRuntime; }, diff --git a/apps/server/src/provider/prime/PrimeAgentEnvironment.ts b/apps/server/src/provider/prime/PrimeAgentEnvironment.ts index 35345e02b..8cbb11c2c 100644 --- a/apps/server/src/provider/prime/PrimeAgentEnvironment.ts +++ b/apps/server/src/provider/prime/PrimeAgentEnvironment.ts @@ -2,6 +2,12 @@ * A Pylon-owned Prime Agent process is always a top-level runtime, even when * Pylon itself was launched from an existing Prime Agent session. */ +export function sanitizePrimeAgentTopLevelEnvironment( + environment: Readonly>, +): Record; +export function sanitizePrimeAgentTopLevelEnvironment( + environment: NodeJS.ProcessEnv, +): NodeJS.ProcessEnv; export function sanitizePrimeAgentTopLevelEnvironment( environment: NodeJS.ProcessEnv, ): NodeJS.ProcessEnv { diff --git a/apps/server/src/provider/prime/PrimeAgentEventBuffer.ts b/apps/server/src/provider/prime/PrimeAgentEventBuffer.ts index 236d75c39..74994f21b 100644 --- a/apps/server/src/provider/prime/PrimeAgentEventBuffer.ts +++ b/apps/server/src/provider/prime/PrimeAgentEventBuffer.ts @@ -16,14 +16,17 @@ export function shutdownPrimeAgentEventPubSub(input: { readonly component: "acp" | "daemon"; readonly pubSub: PubSub.PubSub; readonly drain: Effect.Effect; + readonly timeoutMs?: number; }): Effect.Effect { + const timeoutMs = input.timeoutMs ?? PRIME_AGENT_EVENT_TEARDOWN_TIMEOUT_MS; return Effect.raceFirst( - input.drain, - Effect.sleep(PRIME_AGENT_EVENT_TEARDOWN_TIMEOUT_MS).pipe( + Effect.interruptible(input.drain), + Effect.sleep(timeoutMs).pipe( + Effect.interruptible, Effect.tap(() => Effect.logError("Prime Agent event drain timed out during adapter teardown.", { component: input.component, - timeoutMs: PRIME_AGENT_EVENT_TEARDOWN_TIMEOUT_MS, + timeoutMs, outcome: "forced-pubsub-shutdown", }), ), diff --git a/apps/server/src/provider/prime/PrimeAgentGenerationFence.test.ts b/apps/server/src/provider/prime/PrimeAgentGenerationFence.test.ts new file mode 100644 index 000000000..324b2a87c --- /dev/null +++ b/apps/server/src/provider/prime/PrimeAgentGenerationFence.test.ts @@ -0,0 +1,65 @@ +import { expect, it } from "@effect/vitest"; +import { ProviderDriverKind } from "@t3tools/contracts"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Ref from "effect/Ref"; +import * as Stream from "effect/Stream"; + +import type { PrimeAgentAdapterShape } from "../Services/PrimeAgentAdapter.ts"; +import type { ProviderRuntimeFence } from "../ProviderDriver.ts"; +import { fencePrimeAgentAdapter } from "./PrimeAgentGenerationFence.ts"; + +it.effect("rejects a session result and queued events released after replacement", () => + Effect.gen(function* () { + const current = yield* Ref.make(true); + const sessionStarted = yield* Deferred.make(); + const releaseSession = yield* Deferred.make(); + const listStarted = yield* Deferred.make(); + const releaseList = yield* Deferred.make(); + const releaseEvent = yield* Deferred.make(); + const fence: ProviderRuntimeFence = { + generation: {}, + configRevision: "private-test-revision", + isCurrent: Ref.get(current), + }; + const raw = { + provider: ProviderDriverKind.make("primeAgent"), + startSession: () => + Deferred.succeed(sessionStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseSession)), + Effect.as({ threadId: "thread-a" }), + ), + listSessions: () => + Deferred.succeed(listStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseList)), + Effect.as([{ threadId: "thread-a" }]), + ), + hasSession: () => Effect.succeed(true), + streamEvents: Deferred.await(releaseEvent).pipe( + Effect.as({ type: "session.started" }), + Stream.fromEffect, + ), + } as unknown as PrimeAgentAdapterShape; + const adapter = fencePrimeAgentAdapter(raw, fence); + + const startFiber = yield* adapter + .startSession({} as never) + .pipe(Effect.result, Effect.forkChild); + const eventFiber = yield* adapter.streamEvents.pipe(Stream.runCollect, Effect.forkChild); + const listFiber = yield* adapter.listSessions().pipe(Effect.forkChild); + yield* Deferred.await(sessionStarted); + yield* Deferred.await(listStarted); + yield* Ref.set(current, false); + yield* Deferred.succeed(releaseSession, undefined); + yield* Deferred.succeed(releaseList, undefined); + yield* Deferred.succeed(releaseEvent, undefined); + + const startResult = yield* Fiber.join(startFiber); + expect(startResult._tag).toBe("Failure"); + expect(Array.from(yield* Fiber.join(eventFiber))).toEqual([]); + expect(yield* Fiber.join(listFiber)).toEqual([]); + expect(yield* adapter.listSessions()).toEqual([]); + expect(yield* adapter.hasSession("thread-a" as never)).toBe(false); + }), +); diff --git a/apps/server/src/provider/prime/PrimeAgentGenerationFence.ts b/apps/server/src/provider/prime/PrimeAgentGenerationFence.ts new file mode 100644 index 000000000..33c42d763 --- /dev/null +++ b/apps/server/src/provider/prime/PrimeAgentGenerationFence.ts @@ -0,0 +1,234 @@ +import { ProviderDriverKind } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Stream from "effect/Stream"; + +import { ProviderAdapterRequestError, type ProviderAdapterError } from "../Errors.ts"; +import type { ProviderRuntimeFence } from "../ProviderDriver.ts"; +import type { PrimeAgentAdapterShape } from "../Services/PrimeAgentAdapter.ts"; + +const PROVIDER = ProviderDriverKind.make("primeAgent"); + +export const stalePrimeAgentGenerationError = (method: string) => + new ProviderAdapterRequestError({ + provider: PROVIDER, + method, + detail: "This Prime Agent runtime generation was replaced.", + reason: "stale", + }); + +export const requirePrimeAgentGeneration = ( + fence: ProviderRuntimeFence | undefined, + method: string, +): Effect.Effect => + fence === undefined + ? Effect.void + : fence.isCurrent.pipe( + Effect.flatMap((current) => + current ? Effect.void : Effect.fail(stalePrimeAgentGenerationError(method)), + ), + ); + +/** Fence every public Prime operation while leaving owner-scoped teardown able to finish. */ +export function fencePrimeAgentAdapter( + adapter: PrimeAgentAdapterShape, + runtimeFence: ProviderRuntimeFence, +): PrimeAgentAdapterShape { + const fenced = (method: string, effect: Effect.Effect) => + requirePrimeAgentGeneration(runtimeFence, method).pipe( + Effect.andThen(effect), + Effect.tap(() => requirePrimeAgentGeneration(runtimeFence, method)), + ); + + return { + ...adapter, + runtimeFence, + startSession: (input) => fenced("startSession", adapter.startSession(input)), + sendTurn: (input) => fenced("sendTurn", adapter.sendTurn(input)), + ...(adapter.prepareTurnRecovery === undefined + ? {} + : { + prepareTurnRecovery: (input) => + fenced("prepareTurnRecovery", adapter.prepareTurnRecovery!(input)), + }), + ...(adapter.recoverSession === undefined + ? {} + : { recoverSession: (input) => fenced("recoverSession", adapter.recoverSession!(input)) }), + ...(adapter.activateRecoveredSession === undefined + ? {} + : { + activateRecoveredSession: (threadId) => + fenced("activateRecoveredSession", adapter.activateRecoveredSession!(threadId)), + }), + interruptTurn: (threadId, turnId) => + fenced("interruptTurn", adapter.interruptTurn(threadId, turnId)), + respondToRequest: (threadId, requestId, decision) => + fenced("respondToRequest", adapter.respondToRequest(threadId, requestId, decision)), + respondToUserInput: (threadId, requestId, answers) => + fenced("respondToUserInput", adapter.respondToUserInput(threadId, requestId, answers)), + ...(adapter.respondToInteraction === undefined + ? {} + : { + respondToInteraction: (threadId, requestId, response) => + fenced( + "respondToInteraction", + adapter.respondToInteraction!(threadId, requestId, response), + ), + }), + ...(adapter.reloadSessionResources === undefined + ? {} + : { + reloadSessionResources: (threadId) => + fenced("reloadSessionResources", adapter.reloadSessionResources!(threadId)), + }), + ...(adapter.askSessionSideQuestion === undefined + ? {} + : { + askSessionSideQuestion: (threadId, requestId, question) => + fenced( + "askSessionSideQuestion", + adapter.askSessionSideQuestion!(threadId, requestId, question), + ), + }), + ...(adapter.cancelSessionSideQuestion === undefined + ? {} + : { + cancelSessionSideQuestion: (threadId, requestId) => + fenced( + "cancelSessionSideQuestion", + adapter.cancelSessionSideQuestion!(threadId, requestId), + ), + }), + ...(adapter.cancelSessionAgent === undefined + ? {} + : { + cancelSessionAgent: (threadId, agentId) => + fenced("cancelSessionAgent", adapter.cancelSessionAgent!(threadId, agentId)), + }), + ...(adapter.messageSessionAgent === undefined + ? {} + : { + messageSessionAgent: (threadId, agentId, message) => + fenced("messageSessionAgent", adapter.messageSessionAgent!(threadId, agentId, message)), + }), + ...(adapter.watchSessionAgentActivity === undefined + ? {} + : { + watchSessionAgentActivity: (threadId, agentId) => + adapter.watchSessionAgentActivity!(threadId, agentId).pipe( + Stream.filterEffect(() => runtimeFence.isCurrent), + ), + }), + ...(adapter.getSessionAgentDepth === undefined + ? {} + : { + getSessionAgentDepth: (threadId) => + fenced("getSessionAgentDepth", adapter.getSessionAgentDepth!(threadId)), + }), + ...(adapter.setSessionAgentDepth === undefined + ? {} + : { + setSessionAgentDepth: (threadId, maxDepth) => + fenced("setSessionAgentDepth", adapter.setSessionAgentDepth!(threadId, maxDepth)), + }), + ...(adapter.followUp === undefined + ? {} + : { followUp: (input) => fenced("followUp", adapter.followUp!(input)) }), + ...(adapter.getSessionInputQueue === undefined + ? {} + : { + getSessionInputQueue: (threadId) => + fenced("getSessionInputQueue", adapter.getSessionInputQueue!(threadId)), + }), + ...(adapter.clearSessionInputQueue === undefined + ? {} + : { + clearSessionInputQueue: (threadId) => + fenced("clearSessionInputQueue", adapter.clearSessionInputQueue!(threadId)), + }), + ...(adapter.removeOnlySessionInputQueueItem === undefined + ? {} + : { + removeOnlySessionInputQueueItem: (input) => + fenced( + "removeOnlySessionInputQueueItem", + adapter.removeOnlySessionInputQueueItem!(input), + ), + }), + ...(adapter.setSessionInputQueueMode === undefined + ? {} + : { + setSessionInputQueueMode: (input) => + fenced("setSessionInputQueueMode", adapter.setSessionInputQueueMode!(input)), + }), + ...(adapter.getSessionCompaction === undefined + ? {} + : { + getSessionCompaction: (threadId) => + fenced("getSessionCompaction", adapter.getSessionCompaction!(threadId)), + }), + ...(adapter.compactSession === undefined + ? {} + : { + compactSession: (threadId) => fenced("compactSession", adapter.compactSession!(threadId)), + }), + ...(adapter.abortSessionCompaction === undefined + ? {} + : { + abortSessionCompaction: (threadId) => + fenced("abortSessionCompaction", adapter.abortSessionCompaction!(threadId)), + }), + ...(adapter.setSessionAutoCompaction === undefined + ? {} + : { + setSessionAutoCompaction: (input) => + fenced("setSessionAutoCompaction", adapter.setSessionAutoCompaction!(input)), + }), + ...(adapter.refineSessionHarness === undefined + ? {} + : { + refineSessionHarness: (threadId) => + fenced("refineSessionHarness", adapter.refineSessionHarness!(threadId)), + }), + readThread: (threadId) => fenced("readThread", adapter.readThread(threadId)), + rollbackThread: (threadId, numTurns) => + fenced("rollbackThread", adapter.rollbackThread(threadId, numTurns)), + ...(adapter.uploadFeedback === undefined + ? {} + : { + uploadFeedback: (input) => fenced("uploadFeedback", adapter.uploadFeedback!(input)), + }), + listSessions: () => + runtimeFence.isCurrent.pipe( + Effect.flatMap((current) => + current + ? adapter + .listSessions() + .pipe( + Effect.flatMap((sessions) => + Effect.map(runtimeFence.isCurrent, (stillCurrent) => + stillCurrent ? sessions : [], + ), + ), + ) + : Effect.succeed([]), + ), + ), + hasSession: (threadId) => + runtimeFence.isCurrent.pipe( + Effect.flatMap((current) => + current + ? adapter + .hasSession(threadId) + .pipe( + Effect.flatMap((hasSession) => + Effect.map(runtimeFence.isCurrent, (stillCurrent) => + stillCurrent ? hasSession : false, + ), + ), + ) + : Effect.succeed(false), + ), + ), + streamEvents: adapter.streamEvents.pipe(Stream.filterEffect(() => runtimeFence.isCurrent)), + }; +} diff --git a/apps/server/src/provider/prime/PrimeAgentMultipleInstances.integration.test.ts b/apps/server/src/provider/prime/PrimeAgentMultipleInstances.integration.test.ts new file mode 100644 index 000000000..c612cd819 --- /dev/null +++ b/apps/server/src/provider/prime/PrimeAgentMultipleInstances.integration.test.ts @@ -0,0 +1,776 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import * as NodeUtil from "node:util"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import { + EnvironmentId, + PrimeAgentSettings, + ProviderDriverKind, + ProviderInstanceId, + ThreadId, + type ProviderRuntimeEvent, + type TurnId, +} from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Result from "effect/Result"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; + +import { checkpointRefForThreadTurn } from "../../checkpointing/Utils.ts"; +import { ServerConfig } from "../../config.ts"; +import { clearMcpProviderSession, setMcpProviderSession } from "../../mcp/McpProviderSession.ts"; +import { makePrimeAgentDaemonAdapter } from "./PrimeAgentDaemonAdapter.ts"; +import type { PrimeAgentAdapterShape } from "../Services/PrimeAgentAdapter.ts"; +import type { PrimeAgentDaemonClient } from "./PrimeAgentDaemonBridge.ts"; +import { sanitizePrimeAgentDaemonEnvironment } from "./PrimeAgentDaemonBridge.ts"; +import { + makePrimeAgentDaemonManager, + type PrimeAgentDaemonManager, +} from "./PrimeAgentDaemonManager.ts"; +import { + makePrimeAgentDaemonSessionRuntime, + type PrimeAgentDaemonSessionRuntimeInput, +} from "./PrimeAgentDaemonSessionRuntime.ts"; +import type { PrimeAgentRuntimeContext } from "./PrimeAgentRuntimeContext.ts"; + +const configuredExecutable = process.env.PYLON_REAL_PRIME_AGENT?.trim(); +const configuredAuthHome = process.env.PYLON_REAL_PRIME_AGENT_AUTH_HOME?.trim(); +const runMultipleInstanceProof = process.env.PYLON_REAL_PRIME_AGENT_MULTI_PROOF === "1"; +const configuredCount = Number(process.env.PYLON_REAL_PRIME_AGENT_MULTI_COUNT ?? "2"); +const RESOURCE_CEILINGS = new Map< + number, + { + readonly readinessMs: number; + readonly processCount: number; + readonly rssMiB: number; + readonly fdCount: number; + } +>([ + [1, { readinessMs: 120_000, processCount: 32, rssMiB: 8 * 1024, fdCount: 62_500 }], + [2, { readinessMs: 180_000, processCount: 64, rssMiB: 16 * 1024, fdCount: 125_000 }], + [4, { readinessMs: 240_000, processCount: 128, rssMiB: 32 * 1024, fdCount: 250_000 }], +] as const); +const execFile = NodeUtil.promisify(NodeChildProcess.execFile); +const decodeSettings = Schema.decodeSync(PrimeAgentSettings); +const testLayer = ServerConfig.layerTest(process.cwd(), { + prefix: "pylon-real-prime-multiple-instances-", +}).pipe(Layer.provideMerge(NodeServices.layer)); + +interface ProofInstance { + readonly name: string; + readonly home: string; + readonly scope: Scope.Scope; + readonly manager: PrimeAgentDaemonManager; + readonly adapter: PrimeAgentAdapterShape; + readonly threadId: ThreadId; + readonly checkpointRef: string; + readonly modelSentinel: string; + readonly credentialSentinel: string; + readonly completed: Set; + readonly failed: Set; + readonly completionWaiters: Map>; + readonly drain: Fiber.Fiber; + readonly reconnectObserved: Deferred.Deferred; + readonly reconnectToolObserved: Deferred.Deferred; + readonly clientIds: string[]; + readonly supervisorGenerations: string[]; + readonly activeSessionIds: string[]; + readonly sessionDirectories: string[]; + readonly openCount: { value: number }; + readonly close: Effect.Effect; + disconnectTransport: (() => void) | undefined; +} + +interface ProcessRow { + readonly pid: number; + readonly ppid: number; + readonly rssKiB: number; +} + +interface ResourceSnapshot { + readonly processCount: number; + readonly rssMiB: number; + readonly fdCount: number; + readonly socketCount: number; +} + +function copyAuthFixture(home: string): void { + NodeFS.mkdirSync(home, { recursive: true, mode: 0o700 }); + if (!configuredAuthHome) return; + for (const fileName of ["auth.json", "settings.json"]) { + const source = NodePath.join(configuredAuthHome, fileName); + if (!NodeFS.existsSync(source)) continue; + const destination = NodePath.join(home, fileName); + NodeFS.copyFileSync(source, destination); + NodeFS.chmodSync(destination, 0o600); + } +} + +function proofEnvironment( + home: string, + modelSentinel: string, + credentialSentinel: string, +): Readonly> { + return sanitizePrimeAgentDaemonEnvironment({ + ...Object.fromEntries( + Object.entries(process.env).filter( + (entry): entry is [string, string] => typeof entry[1] === "string", + ), + ), + PRIME_AGENT_HOME: home, + PRIME_AGENT_CODING_AGENT_DIR: home, + PYLON_PRIME_MODEL_SENTINEL: modelSentinel, + PYLON_PRIME_CREDENTIAL_SENTINEL: credentialSentinel, + }); +} + +function safeResponseField(value: unknown, field: string): string | undefined { + if (typeof value !== "object" || value === null) return undefined; + const fieldValue = (value as Record)[field]; + return typeof fieldValue === "string" && fieldValue.length > 0 ? fieldValue : undefined; +} + +function activeSessionFromList(value: unknown): + | { + readonly activeSessionId: string; + readonly sessionDirectory: string; + } + | undefined { + if (typeof value !== "object" || value === null) return undefined; + const data = (value as Record).data; + if (typeof data !== "object" || data === null) return undefined; + const sessions = (data as Record).sessions; + if (!Array.isArray(sessions)) return undefined; + const active = sessions.flatMap((session) => { + const activeSessionId = safeResponseField(session, "activeSessionId"); + const sessionFile = safeResponseField(session, "sessionFile"); + return activeSessionId === undefined || sessionFile === undefined + ? [] + : [{ activeSessionId, sessionDirectory: NodePath.dirname(sessionFile) }]; + }); + return active.length === 1 ? active[0] : undefined; +} + +function hasTruthyStopRequest(value: unknown): boolean { + if (Array.isArray(value)) return value.some(hasTruthyStopRequest); + if (typeof value !== "object" || value === null) return false; + for (const [key, child] of Object.entries(value)) { + if (key === "stopRequestedAt" && child !== null && child !== undefined && child !== false) { + return true; + } + if (hasTruthyStopRequest(child)) return true; + } + return false; +} + +function directoryHasTruthyStopRequest(root: string): boolean { + if (!NodeFS.existsSync(root)) return false; + const pending = [root]; + while (pending.length > 0) { + const current = pending.pop(); + if (current === undefined) break; + for (const entry of NodeFS.readdirSync(current, { withFileTypes: true })) { + const path = NodePath.join(current, entry.name); + if (entry.isDirectory()) { + pending.push(path); + } else if (entry.isFile() && entry.name.endsWith(".json")) { + try { + if (hasTruthyStopRequest(JSON.parse(NodeFS.readFileSync(path, "utf8")))) return true; + } catch { + // A concurrently-written or non-JSON file is not lifecycle evidence. + } + } + } + } + return false; +} + +async function socketOwnerPids(socket: string): Promise> { + try { + const { stdout } = await execFile("/usr/sbin/lsof", ["-t", "-a", "-U", socket], { + timeout: 2_000, + maxBuffer: 256 * 1024, + }); + return stdout + .split("\n") + .map((line) => Number(line.trim())) + .filter((pid) => Number.isSafeInteger(pid) && pid > 0); + } catch { + return []; + } +} + +async function capturedResourceSnapshot(sockets: ReadonlyArray): Promise { + const roots = new Set(); + for (const socket of sockets) for (const pid of await socketOwnerPids(socket)) roots.add(pid); + const { stdout } = await execFile("/bin/ps", ["-axo", "pid=,ppid=,rss="], { + timeout: 2_000, + maxBuffer: 4 * 1024 * 1024, + }); + const rows = stdout.split("\n").flatMap((line): ReadonlyArray => { + const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s*$/u.exec(line); + return match === null + ? [] + : [{ pid: Number(match[1]), ppid: Number(match[2]), rssKiB: Number(match[3]) }]; + }); + const captured = new Set(roots); + for (;;) { + const before = captured.size; + for (const row of rows) if (captured.has(row.ppid)) captured.add(row.pid); + if (captured.size === before) break; + } + const capturedRows = rows.filter((row) => captured.has(row.pid)); + let fdCount = 0; + for (const row of capturedRows) { + try { + const result = await execFile("/usr/sbin/lsof", ["-nP", "-p", String(row.pid)], { + timeout: 2_000, + maxBuffer: 4 * 1024 * 1024, + }); + fdCount += Math.max(0, result.stdout.split("\n").length - 2); + } catch { + // A short-lived child may exit between the captured ps and lsof calls. + } + } + return { + processCount: capturedRows.length, + rssMiB: Math.round((capturedRows.reduce((sum, row) => sum + row.rssKiB, 0) / 1024) * 10) / 10, + fdCount, + socketCount: sockets.filter((socket) => NodeFS.existsSync(socket)).length, + }; +} + +function safeCauseCategory(cause: Cause.Cause): string { + const failure = Cause.findFail(cause); + if (Result.isSuccess(failure)) { + const error = failure.success.error; + if (typeof error === "object" && error !== null) { + const record = error as Record; + const parts = [record._tag, record.operation, record.reason].filter( + (value): value is string => + typeof value === "string" && /^[A-Za-z][A-Za-z0-9-]{0,63}$/u.test(value), + ); + if (parts.length > 0) return parts.join("/"); + } + return "typed-failure"; + } + return Cause.hasInterruptsOnly(cause) ? "interrupted" : "defect"; +} + +it.live.skipIf(!configuredExecutable || !runMultipleInstanceProof)( + "proves exact native Prime N=1/N=2/N=4 isolation, removal, and reconnect without ACP", + () => { + const lifecycle = { + phase: "validation", + ready: 0, + closed: 0, + reconnects: 0, + }; + const proof = Effect.scoped( + Effect.gen(function* () { + if (!configuredExecutable || !NodePath.isAbsolute(configuredExecutable)) { + return yield* Effect.die(new Error("The configured real Prime executable is invalid.")); + } + const resourceCeiling = RESOURCE_CEILINGS.get(configuredCount); + if (resourceCeiling === undefined) { + return yield* Effect.die( + new Error("The native instance proof count must be 1, 2, or 4."), + ); + } + if (configuredAuthHome !== undefined && !NodePath.isAbsolute(configuredAuthHome)) { + return yield* Effect.die(new Error("The configured real Prime auth fixture is invalid.")); + } + + const platform = yield* HostProcessPlatform; + const executablePath = configuredExecutable; + const root = NodeFS.mkdtempSync( + NodePath.join(process.env.TMPDIR ?? "/tmp", "pylon-prime-native-proof-"), + ); + const reportSafePhase = (phase: string) => { + lifecycle.phase = phase; + }; + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), + ); + + const makeInstance = Effect.fn("makeRealPrimeNativeProofInstance")(function* ( + index: number, + ) { + const name = `prime_${index}`; + const home = NodePath.join(root, "homes", name); + const stateDir = NodePath.join(root, "state", name); + const modelSentinel = `model-${index}`; + const credentialSentinel = `credential-${index}`; + copyAuthFixture(home); + NodeFS.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); + NodeFS.writeFileSync(NodePath.join(home, "model-sentinel"), modelSentinel, { + mode: 0o600, + }); + NodeFS.writeFileSync(NodePath.join(home, "credential-sentinel"), credentialSentinel, { + mode: 0o600, + }); + + const instanceId = ProviderInstanceId.make(name); + const threadId = ThreadId.make(`native-proof-${index}`); + const settings = decodeSettings({ + binaryPath: executablePath, + agentHomePath: home, + }); + const launchEnv = proofEnvironment(home, modelSentinel, credentialSentinel); + const generation = { _tag: "PrimeAgentRuntimeGeneration" as const }; + const identity = { + instanceId, + generation, + configRevision: `native-proof-${index}`, + effectiveHome: home, + launchEnv, + nativeMultipleInstancesRequired: true, + settings, + }; + const runtimeContext: PrimeAgentRuntimeContext = { + ...identity, + backendKind: "daemon", + backendIdentity: { + kind: "daemon", + proof: { + sdkFeatures: [ + "negotiated_daemon_session_capabilities_v1", + "caller_owned_session_environment_cleanup_v1", + ], + requiredServerCapabilities: [ + "caller_owned_session_environment_cleanup_v1", + "authoritative_owned_session_cleanup_v1", + ], + }, + }, + }; + const scope = yield* Scope.make("sequential"); + let closed = false; + const close = Effect.suspend(() => { + if (closed) return Effect.void; + closed = true; + lifecycle.closed += 1; + return Scope.close(scope, Exit.void); + }); + yield* Effect.addFinalizer(() => close); + + const completed = new Set(); + const failed = new Set(); + const completionWaiters = new Map>(); + const reconnectObserved = yield* Deferred.make(); + const reconnectToolObserved = yield* Deferred.make(); + const openCount = { value: 0 }; + const clientIds: string[] = []; + const supervisorGenerations: string[] = []; + const activeSessionIds: string[] = []; + const sessionDirectories: string[] = []; + let disconnectTransport: (() => void) | undefined; + let latestClient: PrimeAgentDaemonClient | undefined; + + const built = yield* Effect.gen(function* () { + lifecycle.phase = `instance-${index}-manager`; + const baseManager = yield* makePrimeAgentDaemonManager({ + executablePath, + identity, + stateDir, + tempDir: process.env.TMPDIR ?? "/tmp", + }); + const manager: PrimeAgentDaemonManager = { + ...baseManager, + openClient: () => + baseManager.openClient().pipe( + Effect.tap((client) => + Effect.sync(() => { + openCount.value += 1; + latestClient = client; + if (typeof client.clientId === "string" && client.clientId.length > 0) { + clientIds.push(client.clientId); + } + const supervisorGeneration = client.hello?.supervisorGeneration; + if ( + typeof supervisorGeneration === "string" && + supervisorGeneration.length > 0 + ) { + supervisorGenerations.push(supervisorGeneration); + } + const transport = ( + client as PrimeAgentDaemonClient & { + socket?: { destroy: (error?: Error) => void }; + } + ).socket; + if (transport !== undefined) { + disconnectTransport = () => + transport.destroy(new Error("Native proof transport fault")); + } + }), + ), + ), + }; + const checkpointRef = checkpointRefForThreadTurn(threadId, 1); + setMcpProviderSession({ + environmentId: EnvironmentId.make(`native-proof-${index}`), + threadId, + providerSessionId: `provider-session-${index}`, + providerInstanceId: instanceId, + endpoint: `http://127.0.0.1:9/mcp/native-proof-${index}`, + authorizationHeader: `Bearer mcp-${credentialSentinel}`, + }); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + clearMcpProviderSession(threadId); + }), + ); + yield* manager.prepare(); + lifecycle.phase = `instance-${index}-adapter`; + const runtimeFactory = (input: PrimeAgentDaemonSessionRuntimeInput) => + makePrimeAgentDaemonSessionRuntime(input).pipe( + Effect.map((runtime) => ({ + ...runtime, + events: runtime.events.pipe( + Stream.tap((event) => + event._tag === "SessionResynced" && event.initialSnapshot !== true + ? Effect.sync(() => { + lifecycle.reconnects += 1; + }).pipe( + Effect.andThen(Deferred.succeed(reconnectObserved, undefined)), + Effect.ignore, + ) + : Effect.void, + ), + ), + })), + ); + const adapter = yield* makePrimeAgentDaemonAdapter(settings, manager, { + instanceId, + runtimeContext, + runtimeFactory, + }); + lifecycle.phase = `instance-${index}-start-session`; + yield* adapter.startSession({ + threadId, + provider: ProviderDriverKind.make("primeAgent"), + providerInstanceId: instanceId, + cwd: root, + runtimeMode: "full-access", + modelSelection: { instanceId, model: "default" }, + }); + return { manager, adapter, checkpointRef }; + }).pipe(Effect.provideService(Scope.Scope, scope)); + + lifecycle.phase = `instance-${index}-session-inspection`; + // Ask the runtime's exact owner client for its one native session + // without serializing the private response. + lifecycle.phase = `instance-${index}-owner-client`; + const inspectionClient = latestClient; + if (inspectionClient === undefined) { + return yield* Effect.die(new Error("Native proof owner client was unavailable.")); + } + lifecycle.phase = `instance-${index}-list-request`; + const listed = yield* Effect.promise(() => + inspectionClient.request({ type: "list", includeClientOwned: true }), + ); + lifecycle.phase = `instance-${index}-list-parse`; + const session = activeSessionFromList(listed); + if (session === undefined) { + return yield* Effect.die(new Error("Native proof session inspection was unavailable.")); + } + activeSessionIds.push(session.activeSessionId); + sessionDirectories.push(session.sessionDirectory); + + lifecycle.phase = `instance-${index}-event-drain`; + const drainReady = yield* Deferred.make(); + const drain = yield* Effect.gen(function* () { + yield* Deferred.succeed(drainReady, undefined); + yield* built.adapter.streamEvents.pipe( + Stream.runForEach((event: ProviderRuntimeEvent) => + Effect.gen(function* () { + if ( + event.type === "item.started" && + event.threadId === threadId && + event.payload.itemType === "command_execution" + ) { + yield* Deferred.succeed(reconnectToolObserved, undefined).pipe(Effect.ignore); + } + if ( + event.type !== "turn.completed" || + event.threadId !== threadId || + event.turnId === undefined + ) + return; + if (event.payload.state === "completed") completed.add(event.turnId); + else failed.add(event.turnId); + const waiter = completionWaiters.get(event.turnId); + if (waiter !== undefined) + yield* Deferred.succeed(waiter, undefined).pipe(Effect.ignore); + }), + ), + ); + }).pipe(Effect.forkScoped); + yield* Deferred.await(drainReady); + lifecycle.ready += 1; + + return { + name, + home, + scope, + manager: built.manager, + adapter: built.adapter, + threadId, + checkpointRef: built.checkpointRef, + modelSentinel, + credentialSentinel, + completed, + failed, + completionWaiters, + drain, + reconnectObserved, + reconnectToolObserved, + clientIds, + supervisorGenerations, + activeSessionIds, + sessionDirectories, + openCount, + close, + get disconnectTransport() { + return disconnectTransport; + }, + set disconnectTransport(value: (() => void) | undefined) { + disconnectTransport = value; + }, + } satisfies ProofInstance; + }); + + const waitForTurn = Effect.fn("waitForRealPrimeNativeTurn")(function* ( + instance: ProofInstance, + turnId: TurnId, + ) { + if (!instance.completed.has(turnId) && !instance.failed.has(turnId)) { + const completion = yield* Deferred.make(); + instance.completionWaiters.set(turnId, completion); + if (!instance.completed.has(turnId) && !instance.failed.has(turnId)) { + yield* Deferred.await(completion).pipe(Effect.timeout(Duration.seconds(120))); + } + instance.completionWaiters.delete(turnId); + } + expect(instance.failed.has(turnId)).toBe(false); + expect(instance.completed.has(turnId)).toBe(true); + const thread = yield* instance.adapter.readThread(instance.threadId); + expect(thread.turns.some((turn) => turn.id === turnId)).toBe(true); + }); + + const promptAndWait = Effect.fn("promptAndWaitForRealPrimeNativeProof")(function* ( + instance: ProofInstance, + token: string, + ) { + const started = yield* instance.adapter.sendTurn({ + threadId: instance.threadId, + input: `Reply with exactly ${token} and nothing else. Do not use tools.`, + attachments: [], + }); + yield* waitForTurn(instance, started.turnId); + }); + + reportSafePhase("native-readiness"); + const readinessStartedAt = performance.now(); + const instances = yield* Effect.forEach( + Array.from({ length: configuredCount }), + (_, index) => makeInstance(index), + { concurrency: 1 }, + ); + const readinessMs = performance.now() - readinessStartedAt; + expect(lifecycle.ready).toBe(configuredCount); + expect(readinessMs).toBeLessThan(resourceCeiling.readinessMs); + + const distinct = (values: ReadonlyArray) => new Set(values).size; + expect(distinct(instances.map((instance) => instance.manager.socket))).toBe( + configuredCount, + ); + expect(distinct(instances.map((instance) => instance.manager.sessionDir))).toBe( + configuredCount, + ); + expect(distinct(instances.map((instance) => instance.home))).toBe(configuredCount); + expect(distinct(instances.map((instance) => instance.threadId))).toBe(configuredCount); + expect(distinct(instances.map((instance) => instance.checkpointRef))).toBe(configuredCount); + expect(distinct(instances.flatMap((instance) => instance.clientIds.slice(0, 1)))).toBe( + configuredCount, + ); + expect( + distinct(instances.flatMap((instance) => instance.supervisorGenerations.slice(0, 1))), + ).toBe(configuredCount); + expect(distinct(instances.flatMap((instance) => instance.activeSessionIds))).toBe( + configuredCount, + ); + expect(distinct(instances.flatMap((instance) => instance.sessionDirectories))).toBe( + configuredCount, + ); + expect(distinct(instances.map((instance) => instance.modelSentinel))).toBe(configuredCount); + expect(distinct(instances.map((instance) => instance.credentialSentinel))).toBe( + configuredCount, + ); + for (const instance of instances) { + expect(NodeFS.readFileSync(NodePath.join(instance.home, "model-sentinel"), "utf8")).toBe( + instance.modelSentinel, + ); + expect( + NodeFS.readFileSync(NodePath.join(instance.home, "credential-sentinel"), "utf8"), + ).toBe(instance.credentialSentinel); + } + + const beforeTurns = + platform === "darwin" + ? yield* Effect.promise(() => + capturedResourceSnapshot(instances.map((instance) => instance.manager.socket)), + ) + : undefined; + + reportSafePhase("cold-turns"); + if (configuredCount === 2) { + yield* promptAndWait(instances[0]!, "PYLON_NATIVE_A_COLD_OK"); + yield* promptAndWait(instances[1]!, "PYLON_NATIVE_B_COLD_OK"); + } else { + yield* Effect.all( + instances.map((instance, index) => + promptAndWait(instance, `PYLON_NATIVE_${index}_COLD_OK`), + ), + { concurrency: "unbounded" }, + ); + } + + reportSafePhase("overlapping-turns"); + yield* Effect.all( + instances.map((instance, index) => + promptAndWait(instance, `PYLON_NATIVE_${index}_OVERLAP_OK`), + ), + { concurrency: "unbounded" }, + ); + + const afterTurns = + platform === "darwin" + ? yield* Effect.promise(() => + capturedResourceSnapshot(instances.map((instance) => instance.manager.socket)), + ) + : undefined; + if (afterTurns !== undefined) { + expect(afterTurns.processCount).toBeGreaterThanOrEqual(configuredCount); + expect(afterTurns.processCount).toBeLessThan(resourceCeiling.processCount); + expect(afterTurns.rssMiB).toBeGreaterThan(0); + expect(afterTurns.rssMiB).toBeLessThan(resourceCeiling.rssMiB); + expect(afterTurns.fdCount).toBeGreaterThan(0); + expect(afterTurns.fdCount).toBeLessThan(resourceCeiling.fdCount); + expect(afterTurns.socketCount).toBe(configuredCount); + } + + reportSafePhase("remove-instances"); + const removed = + configuredCount === 1 + ? [] + : configuredCount === 2 + ? [instances[0]!] + : [instances[0]!, instances[2]!]; + const survivors = + configuredCount === 1 + ? [instances[0]!] + : configuredCount === 2 + ? [instances[1]!] + : [instances[1]!, instances[3]!]; + const removedOpenCounts = removed.map((instance) => instance.openCount.value); + // Scope close is the only remove action. Do not hand-close the session first. + yield* Effect.forEach(removed, (instance) => instance.close, { discard: true }); + for (const instance of removed) + expect(NodeFS.existsSync(instance.manager.socket)).toBe(false); + for (const instance of survivors) { + expect(NodeFS.existsSync(instance.manager.socket)).toBe(true); + expect(directoryHasTruthyStopRequest(instance.manager.sessionDir)).toBe(false); + } + + reportSafePhase("survivors-after-removal"); + yield* Effect.all( + survivors.map((instance, index) => + promptAndWait(instance, `PYLON_NATIVE_SURVIVOR_${index}_OK`), + ), + { concurrency: "unbounded" }, + ); + expect(removed.map((instance) => instance.openCount.value)).toEqual(removedOpenCounts); + + reportSafePhase("survivor-reconnect"); + const reconnecting = survivors[0]!; + const disconnect = reconnecting.disconnectTransport; + if (disconnect === undefined) { + return yield* Effect.die(new Error("Native proof reconnect control was unavailable.")); + } + const reconnectTurn = yield* reconnecting.adapter.sendTurn({ + threadId: reconnecting.threadId, + input: + "Use the IPython tool exactly once to print PYLON_NATIVE_RECONNECT_TOOL_OK, then reply with exactly PYLON_NATIVE_AFTER_RECONNECT_OK and nothing else.", + attachments: [], + }); + yield* Deferred.await(reconnecting.reconnectToolObserved).pipe( + Effect.timeout(Duration.seconds(60)), + ); + disconnect(); + yield* Deferred.await(reconnecting.reconnectObserved).pipe( + Effect.timeout(Duration.seconds(30)), + ); + expect(NodeFS.existsSync(reconnecting.manager.socket)).toBe(true); + yield* waitForTurn(reconnecting, reconnectTurn.turnId); + expect(removed.map((instance) => instance.openCount.value)).toEqual(removedOpenCounts); + + reportSafePhase("final-teardown"); + yield* Effect.forEach(survivors, (instance) => instance.close, { discard: true }); + reportSafePhase("survivor-scopes-closed"); + // Event drains belong to this outer proof scope. They remain live across + // participant removal and are interrupted only when the proof scope exits. + expect(lifecycle.closed).toBe(configuredCount); + + if (afterTurns !== undefined) { + const resourceProof = { + instances: configuredCount, + readinessMs: Math.round(readinessMs), + processCount: afterTurns.processCount, + aggregateRssMiB: afterTurns.rssMiB, + deltaRssMiB: + beforeTurns === undefined + ? undefined + : Math.round((afterTurns.rssMiB - beforeTurns.rssMiB) * 10) / 10, + fdCount: afterTurns.fdCount, + socketCount: afterTurns.socketCount, + }; + // Aggregate measurements only. Never emit paths, PIDs, homes, or credentials. + // @effect-diagnostics-next-line globalConsoleInEffect:off preferSchemaOverJson:off + console.log(`PYLON_PRIME_NATIVE_MULTI_RESOURCE_PROOF=${JSON.stringify(resourceProof)}`); + } + reportSafePhase("effect-complete"); + }), + ); + + return proof.pipe( + Effect.catchCause((cause) => + Effect.die( + new Error( + `Native Prime multiple-instance proof failed safely (${[ + `phase=${lifecycle.phase}`, + `ready=${lifecycle.ready}`, + `closed=${lifecycle.closed}`, + `reconnects=${lifecycle.reconnects}`, + `cause=${safeCauseCategory(cause)}`, + ].join(", ")}).`, + ), + ), + ), + Effect.provide(testLayer), + ); + }, + 600_000, +); diff --git a/apps/server/src/provider/prime/PrimeAgentOwnershipReceipt.test.ts b/apps/server/src/provider/prime/PrimeAgentOwnershipReceipt.test.ts new file mode 100644 index 000000000..477baf4bc --- /dev/null +++ b/apps/server/src/provider/prime/PrimeAgentOwnershipReceipt.test.ts @@ -0,0 +1,613 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeChildProcess from "node:child_process"; +import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeTimers from "node:timers"; + +import { afterEach, describe, expect, it } from "vite-plus/test"; + +import type { + PrimeAgentOwnedSessionContractProof, + PrimeAgentOwnedSessionDisposeResult, +} from "./PrimeAgentDaemonBridge.ts"; +import { + PrimeAgentOwnershipReceiptStore, + primeAgentOwnershipHomeLockDigest, + primeAgentOwnershipHomesOverlap, +} from "./PrimeAgentOwnershipReceipt.ts"; + +const roots: string[] = []; +const inspectTestProcessIdentity = async (pid: number) => `test:${pid}`; + +afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => NodeFSP.rm(root, { recursive: true, force: true })), + ); +}); + +async function fixture() { + const createdRoot = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "pylon-prime-owner-")); + const root = await NodeFSP.realpath(createdRoot); + roots.push(root); + const home = NodePath.join(root, "prime-home"); + await NodeFSP.mkdir(home, { recursive: true }); + return { + root, + home, + store: new PrimeAgentOwnershipReceiptStore(root, { + inspectProcessIdentity: inspectTestProcessIdentity, + }), + }; +} + +async function lockPathForAttempt( + store: PrimeAgentOwnershipReceiptStore, + attemptId: string, +): Promise { + const entry = (await NodeFSP.readdir(store.directory)).find( + (name) => name.startsWith(`${attemptId}.`) && name.endsWith(".json.lock"), + ); + if (entry === undefined) throw new Error(`missing ownership lock for ${attemptId}`); + return NodePath.join(store.directory, entry); +} + +function controlledBarrier() { + let enter!: (value: { readonly attemptId: string; readonly effectiveHome: string }) => void; + let release!: () => void; + const entered = new Promise<{ readonly attemptId: string; readonly effectiveHome: string }>( + (resolve) => { + enter = resolve; + }, + ); + const blocked = new Promise((resolve) => { + release = resolve; + }); + return { + entered, + release, + wait: async (identity: Awaited) => { + enter(identity); + await blocked; + }, + }; +} + +const proof = (supervisorGeneration = "supervisor-a", transportGeneration = 1) => + ({ + feature: "caller_owned_session_environment_cleanup_v1", + status: "attached", + daemon: { + protocolName: "prime-agent.daemon", + protocolVersion: 7, + schemaRevision: 30, + appVersion: "1.2.3", + buildId: "managed-build", + supervisorGeneration, + transportGeneration, + }, + }) satisfies PrimeAgentOwnedSessionContractProof; + +const settled = ( + status: "completed" | "already_completed" | "replacement_settled", + started = proof(), +): PrimeAgentOwnedSessionDisposeResult => + status === "replacement_settled" + ? { + feature: "caller_owned_session_environment_cleanup_v1", + status, + started, + observed: proof("supervisor-b", 2).daemon, + daemonReplaced: true, + } + : { + feature: "caller_owned_session_environment_cleanup_v1", + status, + started, + observed: started.daemon, + daemonReplaced: false, + }; + +async function acquired(input: Awaited>, suffix = "a") { + const handle = await input.store.begin({ + instanceId: `prime-${suffix}`, + configRevision: `generation-${suffix}`, + effectiveHome: input.home, + }); + const acquiredHandle = await input.store.markAcquired(handle, { + activeSessionId: `active-${suffix}`, + nativeSessionId: `session-${suffix}`, + attachProof: proof(), + recovery: { + threadId: `thread-${suffix}`, + sessionIncarnationId: `incarnation-${suffix}`, + admissionRequestId: `admission-${suffix}`, + recoveryHandle: `opaque-recovery-${suffix}`, + ownershipGeneration: 1, + }, + }); + return acquiredHandle; +} + +describe("PrimeAgentOwnershipReceiptStore", () => { + it("persists a private dirty receipt before acquisition and keeps a crash dirty", async () => { + const test = await fixture(); + const handle = await test.store.begin({ + instanceId: "prime-a", + configRevision: "generation-a", + effectiveHome: test.home, + }); + + const scan = await new PrimeAgentOwnershipReceiptStore(test.root, { + inspectProcessIdentity: inspectTestProcessIdentity, + }).scan(); + expect(scan).toMatchObject({ corrupt: false, receipts: [{ state: "pending" }] }); + expect(scan.receipts[0]).toMatchObject({ attemptId: handle.attemptId }); + const fileInfo = await NodeFSP.stat( + NodePath.join(test.store.directory, `${handle.attemptId}.json`), + ); + if (process.getuid !== undefined) expect(fileInfo.mode & 0o777).toBe(0o600); + }); + + it.each(["completed", "already_completed", "replacement_settled"] as const)( + "clears only the same exact acquired receipt after %s proof", + async (status) => { + const test = await fixture(); + const handle = await acquired(test); + await expect( + test.store.clearAfterCleanup(handle, { + activeSessionId: "active-a", + nativeSessionId: "session-a", + result: settled(status), + }), + ).resolves.toBe(true); + await expect(test.store.scan()).resolves.toMatchObject({ corrupt: false, receipts: [] }); + }, + ); + + it.each([ + { feature: "caller_owned_session_environment_cleanup_v1", status: "owner_mismatch" }, + { + feature: "caller_owned_session_environment_cleanup_v1", + status: "uncertain", + reason: "active", + }, + { feature: "caller_owned_session_environment_cleanup_v1", status: "transport_failure" }, + { feature: "caller_owned_session_environment_cleanup_v1", status: "unsupported" }, + ] satisfies ReadonlyArray)( + "keeps non-settled cleanup outcome $status dirty", + async (result) => { + const test = await fixture(); + const handle = await acquired(test); + await expect( + test.store.clearAfterCleanup(handle, { + activeSessionId: "active-a", + nativeSessionId: "session-a", + result, + }), + ).resolves.toBe(false); + expect((await test.store.scan()).receipts).toHaveLength(1); + }, + ); + + it("rejects wrong generation, session, and attach proof without cross-clearing", async () => { + const test = await fixture(); + const first = await acquired(test, "a"); + const second = await acquired(test, "b"); + + await expect( + test.store.clearAfterCleanup( + { ...first, currentConfigRevision: second.currentConfigRevision }, + { + activeSessionId: "active-a", + nativeSessionId: "session-a", + result: settled("completed"), + }, + ), + ).resolves.toBe(false); + await expect( + test.store.clearAfterCleanup(first, { + activeSessionId: "active-b", + nativeSessionId: "session-a", + result: settled("completed"), + }), + ).resolves.toBe(false); + await expect( + test.store.clearAfterCleanup(first, { + activeSessionId: "active-a", + nativeSessionId: "session-a", + result: settled("completed", proof("wrong-supervisor", 9)), + }), + ).resolves.toBe(false); + expect((await test.store.scan()).receipts).toHaveLength(2); + }); + + it("claims only the matching recoverable receipt and clears after adopted proof", async () => { + const test = await fixture(); + const handle = await acquired(test); + const receipt = (await test.store.scan()).receipts[0]; + if (receipt?.state !== "acquired") throw new Error("expected acquired receipt"); + + await expect( + test.store.claimForAdoption({ + receipt, + nextConfigRevision: "replacement-generation", + recovery: { + threadId: "thread-a", + sessionIncarnationId: "incarnation-a", + admissionRequestId: "admission-a", + recoveryHandle: "wrong-handle", + ownershipGeneration: 1, + }, + }), + ).resolves.toBeUndefined(); + const claim = await test.store.claimForAdoption({ + receipt, + nextConfigRevision: "replacement-generation", + recovery: receipt.recovery!, + }); + expect(claim?.handle).toMatchObject({ + attemptId: handle.attemptId, + creationConfigRevision: "generation-a", + currentConfigRevision: "replacement-generation", + }); + await test.store.refreshAttachProof(claim!.handle, proof("supervisor-a", 2)); + await expect( + test.store.clearAfterCleanup(claim!.handle, { + activeSessionId: "active-a", + nativeSessionId: "session-a", + result: settled("already_completed", proof("supervisor-a", 2)), + }), + ).resolves.toBe(true); + }); + + it("rotates adoption recovery with an exact idempotent CAS", async () => { + const test = await fixture(); + await acquired(test); + const original = (await test.store.scan()).receipts[0]; + if (original?.state !== "acquired" || original.recovery === undefined) { + throw new Error("expected acquired recovery receipt"); + } + const claim = await test.store.claimForAdoption({ + receipt: original, + nextConfigRevision: "replacement-generation", + recovery: original.recovery, + }); + if (claim === undefined) throw new Error("expected adoption claim"); + const rotatedRecovery = { + ...original.recovery, + recoveryHandle: "rotated-recovery", + ownershipGeneration: original.recovery.ownershipGeneration + 1, + }; + + const rotated = await test.store.rotateAdoptionRecovery(claim, rotatedRecovery); + expect(rotated?.receipt.recovery).toEqual(rotatedRecovery); + await expect(test.store.rotateAdoptionRecovery(claim, rotatedRecovery)).resolves.toMatchObject({ + receipt: { recovery: rotatedRecovery }, + }); + await expect( + test.store.rotateAdoptionRecovery(claim, { + ...rotatedRecovery, + recoveryHandle: "uncommitted-recovery", + }), + ).resolves.toBeUndefined(); + + const restartedClaim = await test.store.claimForAdoption({ + receipt: rotated!.receipt, + nextConfigRevision: "second-restart-generation", + recovery: rotatedRecovery, + }); + expect(restartedClaim?.receipt.recovery).toEqual(rotatedRecovery); + await expect( + test.store.rotateAdoptionRecovery(restartedClaim!, rotatedRecovery), + ).resolves.toMatchObject({ receipt: { recovery: rotatedRecovery } }); + }); + + it("does not let an older adoption release overwrite newer ownership", async () => { + const test = await fixture(); + await acquired(test); + const original = (await test.store.scan()).receipts[0]; + if (original?.state !== "acquired" || original.recovery === undefined) { + throw new Error("expected acquired recovery receipt"); + } + const first = await test.store.claimForAdoption({ + receipt: original, + nextConfigRevision: "replacement-a", + recovery: original.recovery, + }); + if (first === undefined) throw new Error("expected first adoption claim"); + const second = await test.store.claimForAdoption({ + receipt: first.receipt, + nextConfigRevision: "replacement-b", + recovery: original.recovery, + }); + if (second === undefined) throw new Error("expected successor adoption claim"); + + await expect(test.store.releaseAdoptionClaim(first, original)).resolves.toBe(false); + const current = (await test.store.scan()).receipts[0]; + expect(current?.currentConfigRevision).toBe("replacement-b"); + }); + + it("admits only one atomic claimant for an exact recovery receipt", async () => { + const test = await fixture(); + await acquired(test); + const receipt = (await test.store.scan()).receipts[0]; + if (receipt?.state !== "acquired" || receipt.recovery === undefined) { + throw new Error("expected acquired recovery receipt"); + } + + const attempts = await Promise.allSettled([ + test.store.claimForAdoption({ + receipt, + nextConfigRevision: "replacement-a", + recovery: receipt.recovery, + }), + test.store.claimForAdoption({ + receipt, + nextConfigRevision: "replacement-b", + recovery: receipt.recovery, + }), + ]); + const claims = attempts.flatMap((attempt) => + attempt.status === "fulfilled" && attempt.value !== undefined ? [attempt.value] : [], + ); + expect(claims).toHaveLength(1); + const scan = await test.store.scan(); + expect(scan).toMatchObject({ corrupt: false, receipts: [{ state: "acquired" }] }); + expect(scan.receipts[0]?.currentConfigRevision).toBe(claims[0]?.handle.currentConfigRevision); + }); + + it("reconciles a SIGKILL lock left at the durable mutation barrier", async () => { + const test = await fixture(); + const helperPath = NodePath.join(test.root, "crash-lock-helper.ts"); + const sourcePath = NodePath.join(import.meta.dirname, "PrimeAgentOwnershipReceipt.ts"); + await NodeFSP.writeFile( + helperPath, + [ + `import { PrimeAgentOwnershipReceiptStore } from ${JSON.stringify(sourcePath)};`, + `const store = new PrimeAgentOwnershipReceiptStore(${JSON.stringify(test.root)}, {`, + " platform: process.platform as NodeJS.Platform,", + " afterLockPersisted: async (identity) => {", + " console.log(identity.attemptId);", + " await new Promise(() => undefined);", + " },", + "});", + "await store.begin({", + ' instanceId: "prime-crashed",', + ' configRevision: "generation-crashed",', + ` effectiveHome: ${JSON.stringify(test.home)},`, + "});", + ].join("\n"), + { mode: 0o600 }, + ); + const child = NodeChildProcess.spawn("node", ["--experimental-strip-types", helperPath], { + cwd: NodePath.resolve("."), + stdio: ["ignore", "pipe", "pipe"], + }); + const childExit = new Promise((resolve) => { + child.once("exit", () => resolve()); + }); + const attemptId = await new Promise((resolve, reject) => { + // @effect-diagnostics-next-line globalTimers:off + const timeout = NodeTimers.setTimeout( + () => reject(new Error("lock helper readiness timed out")), + 5_000, + ); + let output = ""; + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + output += chunk; + const lineEnd = output.indexOf("\n"); + if (lineEnd < 0) return; + NodeTimers.clearTimeout(timeout); + resolve(output.slice(0, lineEnd).trim()); + }); + child.once("error", (cause) => { + NodeTimers.clearTimeout(timeout); + reject(cause); + }); + child.once("exit", () => { + NodeTimers.clearTimeout(timeout); + reject(new Error("lock helper exited before the durable barrier")); + }); + }); + const lockPath = await lockPathForAttempt(test.store, attemptId); + expect((await NodeFSP.stat(lockPath)).mode & 0o777).toBe(0o600); + expect(child.kill("SIGKILL")).toBe(true); + await new Promise((resolve, reject) => { + // @effect-diagnostics-next-line globalTimers:off + const timeout = NodeTimers.setTimeout( + () => reject(new Error("lock helper SIGKILL timed out")), + 5_000, + ); + void childExit.then(() => { + NodeTimers.clearTimeout(timeout); + resolve(); + }); + }); + + const replacement = new PrimeAgentOwnershipReceiptStore(test.root, { + inspectProcessIdentity: async () => undefined, + }); + await expect(replacement.scan()).resolves.toEqual({ + receipts: [], + quarantinedHomes: [], + quarantinedHomeDigests: [], + corrupt: false, + }); + await expect(NodeFSP.stat(lockPath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("scopes live, unknown, and wrong-mode locks to one home without blocking a sibling", async () => { + const test = await fixture(); + const homeB = NodePath.join(test.root, "prime-home-b"); + await NodeFSP.mkdir(homeB, { recursive: true }); + const barrier = controlledBarrier(); + const lockedStore = new PrimeAgentOwnershipReceiptStore(test.root, { + inspectProcessIdentity: async (pid) => `live:${pid}`, + afterLockPersisted: barrier.wait, + }); + const pendingA = lockedStore.begin({ + instanceId: "prime-a", + configRevision: "generation-a", + effectiveHome: test.home, + }); + const locked = await barrier.entered; + const lockPath = await lockPathForAttempt(test.store, locked.attemptId); + + const liveScan = await new PrimeAgentOwnershipReceiptStore(test.root, { + inspectProcessIdentity: async (pid) => `live:${pid}`, + }).scan(); + expect(liveScan).toEqual({ + receipts: [], + quarantinedHomes: [test.home], + quarantinedHomeDigests: [primeAgentOwnershipHomeLockDigest(test.home)], + corrupt: false, + }); + + const unknownScan = await new PrimeAgentOwnershipReceiptStore(test.root, { + inspectProcessIdentity: async () => undefined, + }).scan(); + expect(unknownScan).toEqual({ + receipts: [], + quarantinedHomes: [test.home], + quarantinedHomeDigests: [primeAgentOwnershipHomeLockDigest(test.home)], + corrupt: false, + }); + + if (process.getuid !== undefined) { + await NodeFSP.chmod(lockPath, 0o644); + const wrongMode = await new PrimeAgentOwnershipReceiptStore(test.root, { + inspectProcessIdentity: inspectTestProcessIdentity, + }).scan(); + expect(wrongMode).toEqual({ + receipts: [], + quarantinedHomes: [], + quarantinedHomeDigests: [primeAgentOwnershipHomeLockDigest(test.home)], + corrupt: false, + }); + await NodeFSP.chmod(lockPath, 0o600); + } + + const sibling = await new PrimeAgentOwnershipReceiptStore(test.root, { + inspectProcessIdentity: inspectTestProcessIdentity, + }).begin({ + instanceId: "prime-b", + configRevision: "generation-b", + effectiveHome: homeB, + }); + expect(sibling.effectiveHome).toBe(homeB); + barrier.release(); + await expect(pendingA).resolves.toMatchObject({ attemptId: locked.attemptId }); + }); + + it("never lets one attempt release another live lock and scopes a known symlink lock", async () => { + const test = await fixture(); + const homeB = NodePath.join(test.root, "prime-home-b"); + await NodeFSP.mkdir(homeB, { recursive: true }); + const releaseByHome = new Map void>(); + let enterA!: (value: { readonly attemptId: string; readonly effectiveHome: string }) => void; + let enterB!: (value: { readonly attemptId: string; readonly effectiveHome: string }) => void; + const enteredA = new Promise<{ readonly attemptId: string; readonly effectiveHome: string }>( + (resolve) => { + enterA = resolve; + }, + ); + const enteredB = new Promise<{ readonly attemptId: string; readonly effectiveHome: string }>( + (resolve) => { + enterB = resolve; + }, + ); + const store = new PrimeAgentOwnershipReceiptStore(test.root, { + inspectProcessIdentity: inspectTestProcessIdentity, + afterLockPersisted: (identity) => { + (identity.effectiveHome === test.home ? enterA : enterB)(identity); + return new Promise((resolve) => releaseByHome.set(identity.effectiveHome, resolve)); + }, + }); + const pendingA = store.begin({ + instanceId: "prime-a", + configRevision: "generation-a", + effectiveHome: test.home, + }); + const lockA = await enteredA; + const pendingB = store.begin({ + instanceId: "prime-b", + configRevision: "generation-b", + effectiveHome: homeB, + }); + const lockB = await enteredB; + + releaseByHome.get(test.home)?.(); + await pendingA; + const lockBPath = await lockPathForAttempt(test.store, lockB.attemptId); + await expect(NodeFSP.stat(lockBPath)).resolves.toMatchObject({}); + expect((await test.store.scan()).quarantinedHomes).toContain(homeB); + releaseByHome.get(homeB)?.(); + const handleB = await pendingB; + + const target = NodePath.join(test.root, "lock-target"); + await NodeFSP.writeFile(target, "not trusted", { mode: 0o600 }); + const symlinkPath = NodePath.join( + test.store.directory, + `${handleB.attemptId}.${primeAgentOwnershipHomeLockDigest(homeB)}.json.lock`, + ); + await NodeFSP.symlink(target, symlinkPath); + const orphanSymlinkPath = NodePath.join( + test.store.directory, + `00000000-0000-4000-8000-000000000000.${primeAgentOwnershipHomeLockDigest(test.home)}.json.lock`, + ); + await NodeFSP.symlink(target, orphanSymlinkPath); + const symlinkScan = await test.store.scan(); + expect(symlinkScan.corrupt).toBe(false); + expect(symlinkScan.quarantinedHomes).toContain(homeB); + expect(symlinkScan.quarantinedHomes).not.toContain(test.home); + expect(symlinkScan.quarantinedHomeDigests).toEqual( + expect.arrayContaining([ + primeAgentOwnershipHomeLockDigest(test.home), + primeAgentOwnershipHomeLockDigest(homeB), + ]), + ); + expect(lockA.effectiveHome).toBe(test.home); + }); + + it("fails closed for corrupt, symlinked, and non-private receipts", async () => { + const corrupt = await fixture(); + await NodeFSP.mkdir(corrupt.store.directory, { recursive: true, mode: 0o700 }); + await NodeFSP.writeFile(NodePath.join(corrupt.store.directory, "not-a-receipt.json"), "{}", { + mode: 0o600, + }); + await expect(corrupt.store.scan()).resolves.toMatchObject({ corrupt: true }); + + const linked = await fixture(); + await NodeFSP.mkdir(linked.store.directory, { recursive: true, mode: 0o700 }); + const target = NodePath.join(linked.root, "target"); + await NodeFSP.writeFile(target, "{}", { mode: 0o600 }); + await NodeFSP.symlink( + target, + NodePath.join(linked.store.directory, "00000000-0000-4000-8000-000000000000.json"), + ); + await expect(linked.store.scan()).resolves.toMatchObject({ corrupt: true }); + + if (process.getuid !== undefined) { + const loose = await fixture(); + const looseHandle = await loose.store.begin({ + instanceId: "prime-loose", + configRevision: "generation-loose", + effectiveHome: loose.home, + }); + await NodeFSP.chmod( + NodePath.join(loose.store.directory, `${looseHandle.attemptId}.json`), + 0o644, + ); + await expect(loose.store.scan()).resolves.toMatchObject({ corrupt: true }); + } + }); + + it("detects equal, ancestor, and descendant homes without treating siblings as overlap", () => { + expect(primeAgentOwnershipHomesOverlap("/srv/prime", "/srv/prime", "linux")).toBe(true); + expect(primeAgentOwnershipHomesOverlap("/srv/prime", "/srv/prime/child", "linux")).toBe(true); + expect(primeAgentOwnershipHomesOverlap("/srv/prime/child", "/srv/prime", "linux")).toBe(true); + expect(primeAgentOwnershipHomesOverlap("/srv/prime-a", "/srv/prime-b", "linux")).toBe(false); + }); +}); diff --git a/apps/server/src/provider/prime/PrimeAgentOwnershipReceipt.ts b/apps/server/src/provider/prime/PrimeAgentOwnershipReceipt.ts new file mode 100644 index 000000000..5629b34d1 --- /dev/null +++ b/apps/server/src/provider/prime/PrimeAgentOwnershipReceipt.ts @@ -0,0 +1,1061 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeCrypto from "node:crypto"; +import * as NodeChildProcess from "node:child_process"; +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; +import * as NodeFS from "node:fs"; + +import * as Predicate from "effect/Predicate"; +import * as Schema from "effect/Schema"; + +import type { + PrimeAgentOwnedSessionContractProof, + PrimeAgentOwnedSessionDisposeResult, +} from "./PrimeAgentDaemonBridge.ts"; + +const RECEIPT_VERSION = 1 as const; +const LOCK_VERSION = 1 as const; +const RECEIPT_DIRECTORY = "native-ownership"; +const RECEIPT_FILE_SUFFIX = ".json"; +const RECEIPT_LOCK_SUFFIX = ".json.lock"; +const RECEIPT_MAX_BYTES = 64 * 1024; +const LOCK_MAX_BYTES = 16 * 1024; +const PRIVATE_DIRECTORY_MODE = 0o700; +const PRIVATE_FILE_MODE = 0o600; +const PROCESS_OWNER_ID = NodeCrypto.randomUUID(); +const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu; + +const NonNegativeInt = Schema.Number.pipe( + Schema.check(Schema.isInt(), Schema.isGreaterThanOrEqualTo(0)), +); +const DaemonIdentity = Schema.Struct({ + protocolName: Schema.String, + protocolVersion: Schema.Int, + schemaRevision: Schema.Int, + appVersion: Schema.optional(Schema.String), + buildId: Schema.optional(Schema.String), + supervisorGeneration: Schema.String, + transportGeneration: NonNegativeInt, +}); +const AttachProof = Schema.Struct({ + feature: Schema.Literal("caller_owned_session_environment_cleanup_v1"), + status: Schema.Literal("attached"), + daemon: DaemonIdentity, +}); +const RecoveryIdentity = Schema.Struct({ + threadId: Schema.String, + sessionIncarnationId: Schema.String, + admissionRequestId: Schema.String, + recoveryHandle: Schema.String, + ownershipGeneration: NonNegativeInt, +}); +const PendingReceipt = Schema.Struct({ + version: Schema.Literal(RECEIPT_VERSION), + state: Schema.Literal("pending"), + attemptId: Schema.String, + instanceId: Schema.String, + creationConfigRevision: Schema.String, + currentConfigRevision: Schema.String, + effectiveHome: Schema.String, + ownerProcessId: Schema.String, +}); +const AcquiredReceipt = Schema.Struct({ + version: Schema.Literal(RECEIPT_VERSION), + state: Schema.Literal("acquired"), + attemptId: Schema.String, + instanceId: Schema.String, + creationConfigRevision: Schema.String, + currentConfigRevision: Schema.String, + effectiveHome: Schema.String, + ownerProcessId: Schema.String, + activeSessionId: Schema.String, + nativeSessionId: Schema.String, + attachProof: AttachProof, + recovery: Schema.optional(RecoveryIdentity), +}); +const OwnershipReceipt = Schema.Union([PendingReceipt, AcquiredReceipt]); +export type PrimeAgentOwnershipReceipt = typeof OwnershipReceipt.Type; +export type PrimeAgentAcquiredOwnershipReceipt = typeof AcquiredReceipt.Type; +const decodeOwnershipReceipt = Schema.decodeUnknownSync(OwnershipReceipt); + +const OwnershipMutationLock = Schema.Struct({ + version: Schema.Literal(LOCK_VERSION), + state: Schema.Literal("locked"), + token: Schema.String, + attemptId: Schema.String, + effectiveHome: Schema.String, + ownerProcessId: Schema.String, + ownerPid: Schema.Int, + ownerStartIdentity: Schema.String, +}); +type PrimeAgentOwnershipMutationLock = typeof OwnershipMutationLock.Type; +const decodeOwnershipMutationLock = Schema.decodeUnknownSync(OwnershipMutationLock); + +export interface PrimeAgentOwnershipRecoveryIdentity { + readonly threadId: string; + readonly sessionIncarnationId: string; + readonly admissionRequestId: string; + readonly recoveryHandle: string; + readonly ownershipGeneration: number; +} + +export interface PrimeAgentOwnershipReceiptScan { + readonly receipts: ReadonlyArray; + /** Valid or scope-attributable live/unknown locks quarantine only overlapping homes. */ + readonly quarantinedHomes: ReadonlyArray; + /** Invalid known lock files retain a filename-bound home scope without trusting contents. */ + readonly quarantinedHomeDigests: ReadonlyArray; + /** Entries that cannot be attributed to one canonical home still fail closed globally. */ + readonly corrupt: boolean; +} + +export interface PrimeAgentOwnershipReceiptStoreOptions { + readonly platform?: NodeJS.Platform; + readonly inspectProcessIdentity?: (pid: number) => Promise; + /** Test barrier after the durable lock becomes visible and before its mutation starts. */ + readonly afterLockPersisted?: (lock: { + readonly attemptId: string; + readonly effectiveHome: string; + }) => Promise; +} + +export interface PrimeAgentOwnershipReceiptHandle { + readonly attemptId: string; + readonly instanceId: string; + readonly creationConfigRevision: string; + readonly currentConfigRevision: string; + readonly effectiveHome: string; +} + +export interface PrimeAgentOwnershipAdoptionClaim { + readonly receipt: PrimeAgentAcquiredOwnershipReceipt; + readonly handle: PrimeAgentOwnershipReceiptHandle; +} + +const liveSafeAttempts = new Set(); + +function sameDaemonIdentity( + left: typeof DaemonIdentity.Type, + right: typeof DaemonIdentity.Type, +): boolean { + return ( + left.protocolName === right.protocolName && + left.protocolVersion === right.protocolVersion && + left.schemaRevision === right.schemaRevision && + left.appVersion === right.appVersion && + left.buildId === right.buildId && + left.supervisorGeneration === right.supervisorGeneration && + left.transportGeneration === right.transportGeneration + ); +} + +function sameRecoveryIdentity( + left: PrimeAgentOwnershipRecoveryIdentity, + right: PrimeAgentOwnershipRecoveryIdentity, +): boolean { + return ( + left.threadId === right.threadId && + left.sessionIncarnationId === right.sessionIncarnationId && + left.admissionRequestId === right.admissionRequestId && + left.recoveryHandle === right.recoveryHandle && + left.ownershipGeneration === right.ownershipGeneration + ); +} + +function sameAcquiredReceipt( + left: PrimeAgentAcquiredOwnershipReceipt, + right: PrimeAgentAcquiredOwnershipReceipt, +): boolean { + return ( + left.version === right.version && + left.state === right.state && + left.attemptId === right.attemptId && + left.instanceId === right.instanceId && + left.creationConfigRevision === right.creationConfigRevision && + left.currentConfigRevision === right.currentConfigRevision && + left.effectiveHome === right.effectiveHome && + left.ownerProcessId === right.ownerProcessId && + left.activeSessionId === right.activeSessionId && + left.nativeSessionId === right.nativeSessionId && + left.attachProof.feature === right.attachProof.feature && + left.attachProof.status === right.attachProof.status && + sameDaemonIdentity(left.attachProof.daemon, right.attachProof.daemon) && + ((left.recovery === undefined && right.recovery === undefined) || + (left.recovery !== undefined && + right.recovery !== undefined && + sameRecoveryIdentity(left.recovery, right.recovery))) + ); +} + +function sameAttachProof( + left: PrimeAgentOwnedSessionContractProof, + right: typeof AttachProof.Type, +): boolean { + return ( + left.feature === right.feature && + left.status === right.status && + sameDaemonIdentity(left.daemon, right.daemon) + ); +} + +function validCleanupOutcome( + result: PrimeAgentOwnedSessionDisposeResult, + attachProof: typeof AttachProof.Type, +): boolean { + if ( + result.feature !== "caller_owned_session_environment_cleanup_v1" || + (result.status !== "completed" && + result.status !== "already_completed" && + result.status !== "replacement_settled") + ) { + return false; + } + if (!sameAttachProof(result.started, attachProof)) return false; + if (result.status === "replacement_settled") { + return ( + result.daemonReplaced === true && + result.observed.supervisorGeneration !== result.started.daemon.supervisorGeneration + ); + } + return ( + result.daemonReplaced === + (result.observed.supervisorGeneration !== result.started.daemon.supervisorGeneration) + ); +} + +async function ensurePrivateDirectory(directory: string): Promise { + await NodeFSP.mkdir(directory, { recursive: true, mode: PRIVATE_DIRECTORY_MODE }); + const info = await NodeFSP.lstat(directory); + const uid = process.getuid?.(); + if ( + !info.isDirectory() || + info.isSymbolicLink() || + (uid !== undefined && info.uid !== uid) || + (uid !== undefined && (info.mode & 0o077) !== 0) + ) { + throw new Error("Prime Agent ownership receipt directory is not private"); + } + if (uid !== undefined) { + await NodeFSP.chmod(directory, PRIVATE_DIRECTORY_MODE); + const restricted = await NodeFSP.lstat(directory); + if ( + !restricted.isDirectory() || + restricted.isSymbolicLink() || + restricted.dev !== info.dev || + restricted.ino !== info.ino || + restricted.uid !== uid || + (restricted.mode & 0o077) !== 0 + ) { + throw new Error("Prime Agent ownership receipt directory changed during validation"); + } + } +} + +async function readReceiptFile(filePath: string): Promise { + const handle = await NodeFSP.open( + filePath, + NodeFS.constants.O_RDONLY | (NodeFS.constants.O_NOFOLLOW ?? 0), + ); + try { + const info = await handle.stat(); + const uid = process.getuid?.(); + if ( + !info.isFile() || + (uid !== undefined && info.uid !== uid) || + (uid !== undefined && (info.mode & 0o177) !== 0) || + info.size <= 0 || + info.size > RECEIPT_MAX_BYTES + ) { + throw new Error("Prime Agent ownership receipt metadata is invalid"); + } + const contents = await handle.readFile({ encoding: "utf8" }); + return decodeOwnershipReceipt(JSON.parse(contents)); + } finally { + await handle.close(); + } +} + +interface ReadMutationLock { + readonly lock: PrimeAgentOwnershipMutationLock; + readonly dev: number; + readonly ino: number; + readonly metadataValid: boolean; +} + +async function readMutationLockFile(lockPath: string): Promise { + const handle = await NodeFSP.open( + lockPath, + NodeFS.constants.O_RDONLY | (NodeFS.constants.O_NOFOLLOW ?? 0), + ); + try { + const info = await handle.stat(); + if (!info.isFile() || info.size <= 0 || info.size > LOCK_MAX_BYTES) { + throw new Error("Prime Agent ownership mutation lock metadata is invalid"); + } + const contents = await handle.readFile({ encoding: "utf8" }); + const lock = decodeOwnershipMutationLock(JSON.parse(contents)); + const uid = process.getuid?.(); + return { + lock, + dev: info.dev, + ino: info.ino, + metadataValid: + (uid === undefined || info.uid === uid) && + (uid === undefined || (info.mode & 0o177) === 0) && + UUID_PATTERN.test(lock.token) && + UUID_PATTERN.test(lock.attemptId) && + UUID_PATTERN.test(lock.ownerProcessId) && + lock.ownerStartIdentity.length > 0 && + Number.isSafeInteger(lock.ownerPid) && + lock.ownerPid > 0, + }; + } finally { + await handle.close(); + } +} + +async function inspectNativeProcessIdentity( + pid: number, + platform: NodeJS.Platform, +): Promise { + if (!Number.isSafeInteger(pid) || pid <= 0) return undefined; + if (platform === "linux") { + try { + const stat = await NodeFSP.readFile(`/proc/${pid}/stat`, "utf8"); + const commandEnd = stat.lastIndexOf(")"); + if (commandEnd < 0) return undefined; + const startTicks = stat + .slice(commandEnd + 2) + .trim() + .split(/\s+/u)[19]; + return startTicks === undefined ? undefined : `${pid}:${startTicks}`; + } catch { + return undefined; + } + } + if (platform !== "darwin") return undefined; + return await new Promise((resolve) => { + NodeChildProcess.execFile( + "/bin/ps", + ["-o", "lstart=", "-p", String(pid)], + { timeout: 1_000, maxBuffer: 1_024, encoding: "utf8" }, + (error, stdout) => { + const started = stdout.trim(); + resolve(error === null && started.length > 0 ? `${pid}:${started}` : undefined); + }, + ); + }); +} + +async function classifyLockOwner( + lock: PrimeAgentOwnershipMutationLock, + inspectProcessIdentity: (pid: number) => Promise, +): Promise<"live" | "dead" | "unknown"> { + const current = await inspectProcessIdentity(lock.ownerPid).catch(() => undefined); + if (current === lock.ownerStartIdentity) return "live"; + if (current !== undefined) return "dead"; + try { + process.kill(lock.ownerPid, 0); + return "unknown"; + } catch (cause) { + return Predicate.isObject(cause) && cause.code === "ESRCH" ? "dead" : "unknown"; + } +} + +async function removeExactMutationLock( + lockPath: string, + expected: ReadMutationLock, +): Promise { + let current: ReadMutationLock; + try { + current = await readMutationLockFile(lockPath); + } catch { + return false; + } + if ( + current.lock.token !== expected.lock.token || + current.dev !== expected.dev || + current.ino !== expected.ino + ) { + return false; + } + const pathInfo = await NodeFSP.lstat(lockPath).catch(() => undefined); + if ( + pathInfo === undefined || + !pathInfo.isFile() || + pathInfo.isSymbolicLink() || + pathInfo.dev !== expected.dev || + pathInfo.ino !== expected.ino + ) { + return false; + } + await NodeFSP.unlink(lockPath); + return true; +} + +async function canonicalizeStoredHome(candidate: string): Promise { + if (!NodePath.isAbsolute(candidate)) + throw new Error("Prime Agent ownership home is not absolute"); + let ancestor = NodePath.normalize(candidate); + const suffix: string[] = []; + while (true) { + try { + const resolved = await NodeFSP.realpath(ancestor); + const info = await NodeFSP.stat(resolved); + if (!info.isDirectory()) + throw new Error("Prime Agent ownership home ancestor is not a directory"); + return suffix.length === 0 ? resolved : NodePath.join(resolved, ...suffix); + } catch (cause) { + if (!Predicate.isObject(cause) || cause.code !== "ENOENT") throw cause; + const parent = NodePath.dirname(ancestor); + if (parent === ancestor) throw cause; + suffix.unshift(NodePath.basename(ancestor)); + ancestor = parent; + } + } +} + +async function syncDirectory(directory: string): Promise { + const handle = await NodeFSP.open(directory, NodeFS.constants.O_RDONLY); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +async function writeAtomicallyUnlocked(input: { + readonly directory: string; + readonly filePath: string; + readonly receipt: PrimeAgentOwnershipReceipt; + readonly replace: boolean; + readonly expected?: PrimeAgentOwnershipReceipt; +}): Promise { + await ensurePrivateDirectory(input.directory); + if (input.replace) { + const previous = await readReceiptFile(input.filePath); + if ( + previous.attemptId !== input.receipt.attemptId || + previous.instanceId !== input.receipt.instanceId || + previous.creationConfigRevision !== input.receipt.creationConfigRevision || + (input.expected !== undefined && JSON.stringify(previous) !== JSON.stringify(input.expected)) + ) { + throw new Error("Prime Agent ownership receipt owner changed"); + } + } + const tempPath = NodePath.join( + input.directory, + `.${input.receipt.attemptId}.${NodeCrypto.randomUUID()}.tmp`, + ); + const handle = await NodeFSP.open( + tempPath, + NodeFS.constants.O_WRONLY | + NodeFS.constants.O_CREAT | + NodeFS.constants.O_EXCL | + (NodeFS.constants.O_NOFOLLOW ?? 0), + PRIVATE_FILE_MODE, + ); + try { + await handle.writeFile(`${JSON.stringify(input.receipt)}\n`, { encoding: "utf8" }); + await handle.sync(); + } finally { + await handle.close(); + } + try { + if (!input.replace) { + try { + await NodeFSP.lstat(input.filePath); + throw new Error("Prime Agent ownership receipt already exists"); + } catch (cause) { + if (!Predicate.isObject(cause) || cause.code !== "ENOENT") throw cause; + } + } + await NodeFSP.rename(tempPath, input.filePath); + await syncDirectory(input.directory); + } catch (cause) { + await NodeFSP.rm(tempPath, { force: true }).catch(() => undefined); + throw cause; + } +} + +interface ReceiptLockRuntime { + readonly inspectProcessIdentity: (pid: number) => Promise; + readonly afterLockPersisted?: PrimeAgentOwnershipReceiptStoreOptions["afterLockPersisted"]; +} + +async function persistMutationLock( + directory: string, + lockPath: string, + lock: PrimeAgentOwnershipMutationLock, +): Promise { + const tempPath = NodePath.join(directory, `.ownership-lock.${lock.token}.tmp`); + const handle = await NodeFSP.open( + tempPath, + NodeFS.constants.O_WRONLY | + NodeFS.constants.O_CREAT | + NodeFS.constants.O_EXCL | + (NodeFS.constants.O_NOFOLLOW ?? 0), + PRIVATE_FILE_MODE, + ); + try { + await handle.writeFile( + `${JSON.stringify(lock)} +`, + { encoding: "utf8" }, + ); + await handle.sync(); + } finally { + await handle.close(); + } + try { + try { + await NodeFSP.link(tempPath, lockPath); + } catch (cause) { + if (Predicate.isObject(cause) && cause.code === "EEXIST") return undefined; + throw cause; + } + await syncDirectory(directory); + return await readMutationLockFile(lockPath); + } finally { + await NodeFSP.rm(tempPath, { force: true }).catch(() => undefined); + } +} + +async function withReceiptLock( + directory: string, + identity: { readonly attemptId: string; readonly effectiveHome: string }, + runtime: ReceiptLockRuntime, + work: () => Promise, +): Promise { + await ensurePrivateDirectory(directory); + const lockPath = NodePath.join( + directory, + `${identity.attemptId}.${primeAgentOwnershipHomeLockDigest(identity.effectiveHome)}${RECEIPT_LOCK_SUFFIX}`, + ); + const ownerStartIdentity = await runtime.inspectProcessIdentity(process.pid); + if (ownerStartIdentity === undefined) { + throw new Error("Prime Agent ownership lock process identity is unavailable"); + } + const mutationLock: PrimeAgentOwnershipMutationLock = { + version: LOCK_VERSION, + state: "locked", + token: NodeCrypto.randomUUID(), + attemptId: identity.attemptId, + effectiveHome: identity.effectiveHome, + ownerProcessId: PROCESS_OWNER_ID, + ownerPid: process.pid, + ownerStartIdentity, + }; + + let persisted = await persistMutationLock(directory, lockPath, mutationLock); + if (persisted === undefined) { + let existing: ReadMutationLock; + try { + existing = await readMutationLockFile(lockPath); + } catch { + throw new Error( + `Prime Agent ownership mutation for '${identity.effectiveHome}' is locked by an unknown owner`, + ); + } + const exactKnownLock = + existing.metadataValid && + existing.lock.attemptId === identity.attemptId && + existing.lock.effectiveHome === identity.effectiveHome; + const ownerState = exactKnownLock + ? await classifyLockOwner(existing.lock, runtime.inspectProcessIdentity) + : "unknown"; + if (ownerState !== "dead" || !(await removeExactMutationLock(lockPath, existing))) { + throw new Error( + `Prime Agent ownership mutation for '${identity.effectiveHome}' is already locked`, + ); + } + await syncDirectory(directory); + persisted = await persistMutationLock(directory, lockPath, mutationLock); + if (persisted === undefined) { + throw new Error( + `Prime Agent ownership mutation for '${identity.effectiveHome}' was claimed concurrently`, + ); + } + } + + try { + await runtime.afterLockPersisted?.(identity); + return await work(); + } finally { + await removeExactMutationLock(lockPath, persisted).catch(() => false); + await syncDirectory(directory).catch(() => undefined); + } +} + +async function writeAtomically( + input: { + readonly directory: string; + readonly filePath: string; + readonly receipt: PrimeAgentOwnershipReceipt; + readonly replace: boolean; + readonly expected?: PrimeAgentOwnershipReceipt; + }, + runtime: ReceiptLockRuntime, +): Promise { + return await withReceiptLock( + input.directory, + { attemptId: input.receipt.attemptId, effectiveHome: input.receipt.effectiveHome }, + runtime, + () => writeAtomicallyUnlocked(input), + ); +} + +function receiptHandle(receipt: PrimeAgentOwnershipReceipt): PrimeAgentOwnershipReceiptHandle { + return Object.freeze({ + attemptId: receipt.attemptId, + instanceId: receipt.instanceId, + creationConfigRevision: receipt.creationConfigRevision, + currentConfigRevision: receipt.currentConfigRevision, + effectiveHome: receipt.effectiveHome, + }); +} + +export function primeAgentOwnershipHomeLockDigest(effectiveHome: string): string { + return NodeCrypto.createHash("sha256").update(effectiveHome).digest("hex").slice(0, 32); +} + +export function primeAgentOwnershipHomesOverlap( + left: string, + right: string, + platform: NodeJS.Platform, +): boolean { + const normalize = (value: string) => + platform === "darwin" || platform === "win32" ? value.toLowerCase() : value; + const normalizedLeft = normalize(NodePath.normalize(left)); + const normalizedRight = normalize(NodePath.normalize(right)); + const relative = NodePath.relative(normalizedLeft, normalizedRight); + if (relative === "") return true; + if ( + !NodePath.isAbsolute(relative) && + relative !== ".." && + !relative.startsWith(`..${NodePath.sep}`) + ) { + return true; + } + const inverse = NodePath.relative(normalizedRight, normalizedLeft); + return ( + !NodePath.isAbsolute(inverse) && inverse !== ".." && !inverse.startsWith(`..${NodePath.sep}`) + ); +} + +export class PrimeAgentOwnershipReceiptStore { + readonly stateDir: string; + readonly directory: string; + private readonly lockRuntime: ReceiptLockRuntime; + + constructor(stateDir: string, options: PrimeAgentOwnershipReceiptStoreOptions = {}) { + this.stateDir = stateDir; + this.directory = NodePath.join(stateDir, "provider-sessions", "prime-agent", RECEIPT_DIRECTORY); + const platform = options.platform; + this.lockRuntime = { + inspectProcessIdentity: + options.inspectProcessIdentity ?? + (platform === undefined + ? async () => undefined + : (pid) => inspectNativeProcessIdentity(pid, platform)), + ...(options.afterLockPersisted === undefined + ? {} + : { afterLockPersisted: options.afterLockPersisted }), + }; + } + + private filePath(attemptId: string): string { + if (!UUID_PATTERN.test(attemptId)) { + throw new Error("Prime Agent ownership attempt identity is invalid"); + } + return NodePath.join(this.directory, `${attemptId}${RECEIPT_FILE_SUFFIX}`); + } + + async scan(): Promise { + try { + await ensurePrivateDirectory(this.directory); + const entries = await NodeFSP.readdir(this.directory, { withFileTypes: true }); + const receipts: PrimeAgentOwnershipReceipt[] = []; + const receiptHomes = new Map(); + const quarantinedHomes = new Set(); + const quarantinedHomeDigests = new Set(); + let corrupt = false; + + for (const entry of entries) { + if (!UUID_PATTERN.test(entry.name.slice(0, -RECEIPT_FILE_SUFFIX.length))) continue; + if (!entry.name.endsWith(RECEIPT_FILE_SUFFIX) || entry.name.endsWith(RECEIPT_LOCK_SUFFIX)) { + continue; + } + if (!entry.isFile() || entry.isSymbolicLink()) { + corrupt = true; + continue; + } + try { + const receipt = await readReceiptFile(NodePath.join(this.directory, entry.name)); + const canonicalHome = await canonicalizeStoredHome(receipt.effectiveHome); + if ( + `${receipt.attemptId}${RECEIPT_FILE_SUFFIX}` !== entry.name || + canonicalHome !== receipt.effectiveHome + ) { + corrupt = true; + } else { + receipts.push(receipt); + receiptHomes.set(receipt.attemptId, canonicalHome); + } + } catch { + corrupt = true; + } + } + + for (const entry of entries) { + const lockMatch = /^([0-9a-f-]{36})(?:\.([0-9a-f]{32}))?\.json\.lock$/iu.exec(entry.name); + const tempMatch = /^\.ownership-lock\.([0-9a-f-]{36})\.tmp$/iu.exec(entry.name); + if (lockMatch === null && tempMatch === null) { + const receiptName = entry.name.endsWith(RECEIPT_FILE_SUFFIX) + ? entry.name.slice(0, -RECEIPT_FILE_SUFFIX.length) + : ""; + if (!UUID_PATTERN.test(receiptName)) corrupt = true; + continue; + } + + const lockPath = NodePath.join(this.directory, entry.name); + const expectedAttemptId = lockMatch?.[1]; + const expectedHomeDigest = lockMatch?.[2]; + let fallbackHome = + expectedAttemptId === undefined ? undefined : receiptHomes.get(expectedAttemptId); + let read: ReadMutationLock | undefined; + try { + read = await readMutationLockFile(lockPath); + } catch { + // A known lock with an accompanying receipt can still be scoped without + // following a symlink or trusting malformed lock contents. + } + + const lockMatchesName = + read !== undefined && + (expectedAttemptId === undefined + ? tempMatch?.[1] === read.lock.token + : expectedAttemptId === read.lock.attemptId && + (expectedHomeDigest === undefined || + expectedHomeDigest === primeAgentOwnershipHomeLockDigest(read.lock.effectiveHome))); + if (read !== undefined && read.metadataValid && lockMatchesName) { + const canonicalHome = await canonicalizeStoredHome(read.lock.effectiveHome).catch( + () => undefined, + ); + if (canonicalHome !== undefined && canonicalHome === read.lock.effectiveHome) { + fallbackHome = canonicalHome; + const ownerState = await classifyLockOwner( + read.lock, + this.lockRuntime.inspectProcessIdentity, + ); + if (ownerState === "dead" && (await removeExactMutationLock(lockPath, read))) { + await syncDirectory(this.directory); + continue; + } + } + } + + if (fallbackHome !== undefined) quarantinedHomes.add(fallbackHome); + if (expectedHomeDigest !== undefined) { + quarantinedHomeDigests.add(expectedHomeDigest); + } + if (fallbackHome === undefined && expectedHomeDigest === undefined) corrupt = true; + } + return { + receipts, + quarantinedHomes: [...quarantinedHomes], + quarantinedHomeDigests: [...quarantinedHomeDigests], + corrupt, + }; + } catch { + return { + receipts: [], + quarantinedHomes: [], + quarantinedHomeDigests: [], + corrupt: true, + }; + } + } + + async begin(input: { + readonly instanceId: string; + readonly configRevision: string; + readonly effectiveHome: string; + }): Promise { + const receipt: PrimeAgentOwnershipReceipt = { + version: RECEIPT_VERSION, + state: "pending", + attemptId: NodeCrypto.randomUUID(), + instanceId: input.instanceId, + creationConfigRevision: input.configRevision, + currentConfigRevision: input.configRevision, + effectiveHome: input.effectiveHome, + ownerProcessId: PROCESS_OWNER_ID, + }; + await writeAtomically( + { + directory: this.directory, + filePath: this.filePath(receipt.attemptId), + receipt, + replace: false, + }, + this.lockRuntime, + ); + return receiptHandle(receipt); + } + + async markAcquired( + handle: PrimeAgentOwnershipReceiptHandle, + input: { + readonly activeSessionId: string; + readonly nativeSessionId: string; + readonly attachProof: PrimeAgentOwnedSessionContractProof; + readonly recovery?: PrimeAgentOwnershipRecoveryIdentity; + }, + ): Promise { + const filePath = this.filePath(handle.attemptId); + const previous = await readReceiptFile(filePath); + if ( + previous.state !== "pending" || + previous.instanceId !== handle.instanceId || + previous.creationConfigRevision !== handle.creationConfigRevision || + previous.currentConfigRevision !== handle.currentConfigRevision || + previous.effectiveHome !== handle.effectiveHome || + previous.ownerProcessId !== PROCESS_OWNER_ID + ) { + throw new Error("Prime Agent ownership receipt is not owned by this attempt"); + } + const receipt: PrimeAgentAcquiredOwnershipReceipt = { + ...previous, + state: "acquired", + activeSessionId: input.activeSessionId, + nativeSessionId: input.nativeSessionId, + attachProof: input.attachProof, + ...(input.recovery === undefined ? {} : { recovery: input.recovery }), + }; + await writeAtomically( + { + directory: this.directory, + filePath, + receipt, + replace: true, + expected: previous, + }, + this.lockRuntime, + ); + liveSafeAttempts.add(receipt.attemptId); + return receiptHandle(receipt); + } + + async refreshAttachProof( + handle: PrimeAgentOwnershipReceiptHandle, + attachProof: PrimeAgentOwnedSessionContractProof, + ): Promise { + const filePath = this.filePath(handle.attemptId); + const previous = await readReceiptFile(filePath); + if ( + previous.state !== "acquired" || + previous.currentConfigRevision !== handle.currentConfigRevision || + previous.ownerProcessId !== PROCESS_OWNER_ID + ) { + throw new Error("Prime Agent ownership receipt is not owned by this generation"); + } + await writeAtomically( + { + directory: this.directory, + filePath, + receipt: { ...previous, attachProof }, + replace: true, + expected: previous, + }, + this.lockRuntime, + ); + } + + markUnsafe(handle: PrimeAgentOwnershipReceiptHandle): void { + liveSafeAttempts.delete(handle.attemptId); + } + + async proveNeverAcquired(handle: PrimeAgentOwnershipReceiptHandle): Promise { + const filePath = this.filePath(handle.attemptId); + return await withReceiptLock( + this.directory, + { attemptId: handle.attemptId, effectiveHome: handle.effectiveHome }, + this.lockRuntime, + async () => { + const previous = await readReceiptFile(filePath); + if ( + previous.state !== "pending" || + previous.instanceId !== handle.instanceId || + previous.creationConfigRevision !== handle.creationConfigRevision || + previous.currentConfigRevision !== handle.currentConfigRevision || + previous.effectiveHome !== handle.effectiveHome || + previous.ownerProcessId !== PROCESS_OWNER_ID + ) { + return false; + } + await NodeFSP.unlink(filePath); + await syncDirectory(this.directory); + liveSafeAttempts.delete(handle.attemptId); + return true; + }, + ); + } + + async clearAfterCleanup( + handle: PrimeAgentOwnershipReceiptHandle, + input: { + readonly activeSessionId: string; + readonly nativeSessionId: string; + readonly result: PrimeAgentOwnedSessionDisposeResult; + }, + ): Promise { + const filePath = this.filePath(handle.attemptId); + return await withReceiptLock( + this.directory, + { attemptId: handle.attemptId, effectiveHome: handle.effectiveHome }, + this.lockRuntime, + async () => { + const previous = await readReceiptFile(filePath); + if ( + previous.state !== "acquired" || + previous.instanceId !== handle.instanceId || + previous.creationConfigRevision !== handle.creationConfigRevision || + previous.currentConfigRevision !== handle.currentConfigRevision || + previous.effectiveHome !== handle.effectiveHome || + previous.ownerProcessId !== PROCESS_OWNER_ID || + previous.activeSessionId !== input.activeSessionId || + previous.nativeSessionId !== input.nativeSessionId || + !validCleanupOutcome(input.result, previous.attachProof) + ) { + liveSafeAttempts.delete(handle.attemptId); + return false; + } + await NodeFSP.unlink(filePath); + await syncDirectory(this.directory); + liveSafeAttempts.delete(handle.attemptId); + return true; + }, + ); + } + + async claimForAdoption(input: { + readonly receipt: PrimeAgentAcquiredOwnershipReceipt; + readonly nextConfigRevision: string; + readonly recovery: PrimeAgentOwnershipRecoveryIdentity; + }): Promise { + const filePath = this.filePath(input.receipt.attemptId); + const previous = await readReceiptFile(filePath); + if ( + previous.state !== "acquired" || + previous.attemptId !== input.receipt.attemptId || + previous.instanceId !== input.receipt.instanceId || + previous.creationConfigRevision !== input.receipt.creationConfigRevision || + previous.currentConfigRevision !== input.receipt.currentConfigRevision || + previous.effectiveHome !== input.receipt.effectiveHome || + previous.activeSessionId !== input.receipt.activeSessionId || + previous.nativeSessionId !== input.receipt.nativeSessionId || + previous.recovery === undefined || + previous.recovery.threadId !== input.recovery.threadId || + previous.recovery.sessionIncarnationId !== input.recovery.sessionIncarnationId || + previous.recovery.admissionRequestId !== input.recovery.admissionRequestId || + previous.recovery.recoveryHandle !== input.recovery.recoveryHandle || + previous.recovery.ownershipGeneration !== input.recovery.ownershipGeneration + ) { + return undefined; + } + const claimed: PrimeAgentAcquiredOwnershipReceipt = { + ...previous, + currentConfigRevision: input.nextConfigRevision, + ownerProcessId: PROCESS_OWNER_ID, + }; + await writeAtomically( + { + directory: this.directory, + filePath, + receipt: claimed, + replace: true, + expected: previous, + }, + this.lockRuntime, + ); + liveSafeAttempts.add(claimed.attemptId); + return { receipt: claimed, handle: receiptHandle(claimed) }; + } + + async rotateAdoptionRecovery( + claim: PrimeAgentOwnershipAdoptionClaim, + recovery: PrimeAgentOwnershipRecoveryIdentity, + ): Promise { + const filePath = this.filePath(claim.handle.attemptId); + const current = await readReceiptFile(filePath); + if (current.state !== "acquired") return undefined; + + if (current.recovery !== undefined && sameRecoveryIdentity(current.recovery, recovery)) { + const priorIdentity = claim.receipt.recovery; + if ( + priorIdentity !== undefined && + sameAcquiredReceipt({ ...current, recovery: priorIdentity }, claim.receipt) + ) { + return { receipt: current, handle: receiptHandle(current) }; + } + } + const previousRecovery = claim.receipt.recovery; + if ( + previousRecovery === undefined || + previousRecovery.threadId !== recovery.threadId || + previousRecovery.sessionIncarnationId !== recovery.sessionIncarnationId || + previousRecovery.admissionRequestId !== recovery.admissionRequestId || + previousRecovery.recoveryHandle === recovery.recoveryHandle || + previousRecovery.ownershipGeneration >= recovery.ownershipGeneration || + !sameAcquiredReceipt(current, claim.receipt) + ) { + return undefined; + } + + const rotated: PrimeAgentAcquiredOwnershipReceipt = { ...current, recovery }; + await writeAtomically( + { + directory: this.directory, + filePath, + receipt: rotated, + replace: true, + expected: current, + }, + this.lockRuntime, + ); + liveSafeAttempts.add(rotated.attemptId); + return { receipt: rotated, handle: receiptHandle(rotated) }; + } + + async releaseAdoptionClaim( + claim: PrimeAgentOwnershipAdoptionClaim, + previous: PrimeAgentAcquiredOwnershipReceipt, + ): Promise { + const filePath = this.filePath(claim.handle.attemptId); + const current = await readReceiptFile(filePath); + if ( + current.state !== "acquired" || + current.currentConfigRevision !== claim.handle.currentConfigRevision || + current.ownerProcessId !== PROCESS_OWNER_ID + ) { + return false; + } + await writeAtomically( + { + directory: this.directory, + filePath, + receipt: previous, + replace: true, + expected: current, + }, + this.lockRuntime, + ); + liveSafeAttempts.delete(current.attemptId); + return true; + } +} + +export function primeAgentOwnershipReceiptIsSafeLive(receipt: PrimeAgentOwnershipReceipt): boolean { + return receipt.ownerProcessId === PROCESS_OWNER_ID && liveSafeAttempts.has(receipt.attemptId); +} diff --git a/apps/server/src/provider/prime/PrimeAgentRealDaemon.integration.test.ts b/apps/server/src/provider/prime/PrimeAgentRealDaemon.integration.test.ts index 1d7490f9d..54f240070 100644 --- a/apps/server/src/provider/prime/PrimeAgentRealDaemon.integration.test.ts +++ b/apps/server/src/provider/prime/PrimeAgentRealDaemon.integration.test.ts @@ -13,6 +13,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Stream from "effect/Stream"; import type { PrimeDaemonEvent } from "./PrimeAgentDaemonEvents.ts"; +import { sanitizePrimeAgentDaemonEnvironment } from "./PrimeAgentDaemonBridge.ts"; import { makePrimeAgentDaemonManager } from "./PrimeAgentDaemonManager.ts"; import { makePrimeAgentDaemonSessionRuntime, @@ -23,6 +24,48 @@ const configuredExecutable = process.env.PYLON_REAL_PRIME_AGENT?.trim(); const configuredAuthHome = process.env.PYLON_REAL_PRIME_AGENT_AUTH_HOME?.trim(); const providerInstanceId = ProviderInstanceId.make("prime-real-integration"); +const makeTestIdentity = (agentHomePath: string) => ({ + instanceId: providerInstanceId, + generation: { _tag: "PrimeAgentRuntimeGeneration" as const }, + configRevision: "real-integration-test", + effectiveHome: agentHomePath, + nativeMultipleInstancesRequired: false, + launchEnv: sanitizePrimeAgentDaemonEnvironment({ + ...Object.fromEntries( + Object.entries(process.env).filter( + (entry): entry is [string, string] => typeof entry[1] === "string", + ), + ), + PRIME_AGENT_HOME: agentHomePath, + PRIME_AGENT_CODING_AGENT_DIR: agentHomePath, + }), + settings: { + enabled: true, + binaryPath: configuredExecutable ?? "prime-agent", + agentHomePath, + launchArgs: "", + customModels: [], + }, +}); + +const makeTestRuntimeContext = (identity: ReturnType) => ({ + ...identity, + backendKind: "daemon" as const, + backendIdentity: { + kind: "daemon" as const, + proof: { + sdkFeatures: [ + "negotiated_daemon_session_capabilities_v1", + "caller_owned_session_environment_cleanup_v1", + ], + requiredServerCapabilities: [ + "caller_owned_session_environment_cleanup_v1", + "authoritative_owned_session_cleanup_v1", + ] as const, + }, + }, +}); + type TurnCompleted = Extract; function drainEvents(input: { @@ -137,18 +180,19 @@ it.live.skipIf(!configuredExecutable)( const first = yield* Effect.scoped( Effect.gen(function* () { + const identity = makeTestIdentity(agentHomePath); + const runtimeContext = makeTestRuntimeContext(identity); const manager = yield* makePrimeAgentDaemonManager({ executablePath: configuredExecutable, - settings: { agentHomePath }, - environment: process.env, + identity, stateDir, - providerInstanceId, tempDir: "/tmp", }); yield* manager.prepare(); const primary = yield* makePrimeAgentDaemonSessionRuntime({ manager, + runtimeContext, cwd: root, sessionDir: NodePath.join(manager.sessionDir, "phase-1-primary"), thinkingLevel: "off", @@ -156,6 +200,7 @@ it.live.skipIf(!configuredExecutable)( }); const interrupted = yield* makePrimeAgentDaemonSessionRuntime({ manager, + runtimeContext, cwd: root, sessionDir: NodePath.join(manager.sessionDir, "phase-1-interrupted"), thinkingLevel: "off", @@ -236,6 +281,7 @@ it.live.skipIf(!configuredExecutable)( }; const reconnecting = yield* makePrimeAgentDaemonSessionRuntime({ manager: reconnectManager, + runtimeContext, cwd: root, sessionDir: NodePath.join(manager.sessionDir, "phase-1-reconnecting"), thinkingLevel: "off", @@ -336,16 +382,17 @@ it.live.skipIf(!configuredExecutable)( const restarted = yield* Effect.scoped( Effect.gen(function* () { + const identity = makeTestIdentity(agentHomePath); + const runtimeContext = makeTestRuntimeContext(identity); const manager = yield* makePrimeAgentDaemonManager({ executablePath: configuredExecutable, - settings: { agentHomePath }, - environment: process.env, + identity, stateDir, - providerInstanceId, tempDir: "/tmp", }); const runtime = yield* makePrimeAgentDaemonSessionRuntime({ manager, + runtimeContext, cwd: root, sessionDir: NodePath.join(manager.sessionDir, "phase-1-primary"), disableExtensionDiscovery: true, diff --git a/apps/server/src/provider/prime/PrimeAgentRecoveryLedger.test.ts b/apps/server/src/provider/prime/PrimeAgentRecoveryLedger.test.ts index 3393101cc..0c9c960f0 100644 --- a/apps/server/src/provider/prime/PrimeAgentRecoveryLedger.test.ts +++ b/apps/server/src/provider/prime/PrimeAgentRecoveryLedger.test.ts @@ -10,6 +10,7 @@ import * as NodeSqliteClient from "../../persistence/NodeSqliteClient.ts"; import { make, PRIME_AGENT_RECOVERY_ADOPTION_MAX_ATTEMPTS, + primeAgentRecoveryOwnershipIdentities, type PrimeAgentRecoveryAdoptionProof, type PrimeAgentRecoveryAuthority, type PrimeAgentRecoveryLedgerShape, @@ -121,6 +122,15 @@ layer("PrimeAgentRecoveryLedger", (it) => { assert.equal(claimed.adoptionRequestId, "a".repeat(48)); assert.equal(claimed.adoptionPhase, "claimed"); assert.equal(claimed.adoptionAttempt, 0); + assert.deepEqual(primeAgentRecoveryOwnershipIdentities(claimed), [ + { + threadId: authority.threadId, + sessionIncarnationId: authority.sessionIncarnationId, + admissionRequestId: authority.admissionRequestId, + recoveryHandle: authority.recoveryHandle, + ownershipGeneration: authority.ownershipGeneration, + }, + ]); assert.isTrue( Option.isNone( yield* ledger.claim({ @@ -161,6 +171,22 @@ layer("PrimeAgentRecoveryLedger", (it) => { assert.equal(committed.ownerToken, authority.ownerToken); assert.equal(committed.adoptionRecoveryHandle, "private-handle-2"); assert.deepEqual(committed.adoptionProof, proof); + assert.deepEqual(primeAgentRecoveryOwnershipIdentities(committed), [ + { + threadId: authority.threadId, + sessionIncarnationId: authority.sessionIncarnationId, + admissionRequestId: authority.admissionRequestId, + recoveryHandle: authority.recoveryHandle, + ownershipGeneration: authority.ownershipGeneration, + }, + { + threadId: authority.threadId, + sessionIncarnationId: authority.sessionIncarnationId, + admissionRequestId: authority.admissionRequestId, + recoveryHandle: "private-handle-2", + ownershipGeneration: proof.ownershipGeneration, + }, + ]); const retried = Option.getOrThrow( yield* ledger.beginAdoptionAttempt({ @@ -203,6 +229,10 @@ layer("PrimeAgentRecoveryLedger", (it) => { ); assert.equal(confirming.adoptionPhase, "confirming"); assert.equal(confirming.adoptionAttempt, 3); + assert.deepEqual( + primeAgentRecoveryOwnershipIdentities(confirming), + primeAgentRecoveryOwnershipIdentities(committed), + ); const confirmationRetried = Option.getOrThrow( yield* ledger.beginAdoptionConfirmation({ threadId: authority.threadId, @@ -232,6 +262,15 @@ layer("PrimeAgentRecoveryLedger", (it) => { assert.equal(adopted.mcpOwnerId, proof.mcpOwnerId); assert.equal(adopted.adoptionRequestId, null); assert.equal(adopted.adoptionProof, null); + assert.deepEqual(primeAgentRecoveryOwnershipIdentities(adopted), [ + { + threadId: authority.threadId, + sessionIncarnationId: authority.sessionIncarnationId, + admissionRequestId: authority.admissionRequestId, + recoveryHandle: "private-handle-2", + ownershipGeneration: proof.ownershipGeneration, + }, + ]); }), ); @@ -424,4 +463,27 @@ layer("PrimeAgentRecoveryLedger", (it) => { assert.isTrue(Option.isNone(yield* ledger.get(authority.threadId))); }), ); + + it.effect("skips mutations after a private commit guard retires", () => + Effect.gen(function* () { + yield* migration050; + const ledger = yield* make; + yield* ledger.putPrepared(authority, { commitGuard: Effect.succeed(false) }); + assert.isTrue(Option.isNone(yield* ledger.get(authority.threadId))); + + yield* ledger.putPrepared(authority, { commitGuard: Effect.succeed(true) }); + assert.isFalse( + yield* ledger.markAdmitted( + { + threadId: authority.threadId, + ownerToken: authority.ownerToken, + turnId: "turn-retired", + updatedAt: "2026-01-01T00:00:08.000Z", + }, + { commitGuard: Effect.succeed(false) }, + ), + ); + assert.equal(Option.getOrThrow(yield* ledger.get(authority.threadId)).state, "prepared"); + }), + ); }); diff --git a/apps/server/src/provider/prime/PrimeAgentRecoveryLedger.ts b/apps/server/src/provider/prime/PrimeAgentRecoveryLedger.ts index bc27622c4..03566bf2e 100644 --- a/apps/server/src/provider/prime/PrimeAgentRecoveryLedger.ts +++ b/apps/server/src/provider/prime/PrimeAgentRecoveryLedger.ts @@ -75,6 +75,58 @@ export const PrimeAgentRecoveryAuthority = Schema.Struct({ }); export type PrimeAgentRecoveryAuthority = typeof PrimeAgentRecoveryAuthority.Type; +export interface PrimeAgentRecoveryOwnershipIdentity { + readonly threadId: string; + readonly sessionIncarnationId: string; + readonly admissionRequestId: string; + readonly recoveryHandle: string; + readonly ownershipGeneration: number; +} + +/** + * The current identity remains valid while adoption is durable. The staged identity + * is valid only after the complete rotated proof was committed and before finalization. + */ +export function primeAgentRecoveryOwnershipIdentities( + authority: PrimeAgentRecoveryAuthority, +): ReadonlyArray { + const current = { + threadId: authority.threadId, + sessionIncarnationId: authority.sessionIncarnationId, + admissionRequestId: authority.admissionRequestId, + recoveryHandle: authority.recoveryHandle, + ownershipGeneration: authority.ownershipGeneration, + }; + const proof = authority.adoptionProof; + const stagedHandle = authority.adoptionRecoveryHandle; + const exactStaged = + authority.state === "adopting" && + (authority.adoptionPhase === "committed" || authority.adoptionPhase === "confirming") && + stagedHandle !== null && + stagedHandle !== authority.recoveryHandle && + proof !== null && + proof.supervisorGeneration === authority.supervisorGeneration && + proof.ownershipGeneration > authority.ownershipGeneration && + proof.activeSessionId === authority.activeSessionId && + proof.sessionId === authority.nativeSessionId && + proof.correlationId === authority.correlationId && + proof.mcpOwnerId === authority.adoptionMcpOwnerId && + proof.cursor.generation === authority.cursor.generation && + proof.cursor.sequence >= authority.cursor.sequence; + return exactStaged + ? [ + current, + { + threadId: authority.threadId, + sessionIncarnationId: authority.sessionIncarnationId, + admissionRequestId: authority.admissionRequestId, + recoveryHandle: stagedHandle, + ownershipGeneration: proof.ownershipGeneration, + }, + ] + : [current]; +} + export class PrimeAgentRecoveryLedgerError extends Schema.TaggedErrorClass()( "PrimeAgentRecoveryLedgerError", { @@ -89,9 +141,14 @@ export class PrimeAgentRecoveryLedgerError extends Schema.TaggedErrorClass | undefined; +} + export interface PrimeAgentRecoveryLedgerShape { readonly putPrepared: ( authority: PrimeAgentRecoveryAuthority, + options?: PrimeAgentRecoveryCommitOptions, ) => Effect.Effect; readonly get: ( threadId: string, @@ -100,98 +157,125 @@ export interface PrimeAgentRecoveryLedgerShape { ReadonlyArray, PrimeAgentRecoveryLedgerError >; - readonly markAdmitted: (input: { - readonly threadId: string; - readonly ownerToken: string; - readonly turnId: string; - readonly updatedAt: string; - }) => Effect.Effect; - readonly discardPrepared: (input: { - readonly threadId: string; - readonly ownerToken: string; - }) => Effect.Effect; - readonly updateTranscriptProgress: (input: { - readonly threadId: string; - readonly ownerToken: string; - readonly cursor: typeof RecoveryCursor.Type; - readonly messageCount: number; - readonly fingerprints: ReadonlyArray; - readonly updatedAt: string; - }) => Effect.Effect; + readonly markAdmitted: ( + input: { + readonly threadId: string; + readonly ownerToken: string; + readonly turnId: string; + readonly updatedAt: string; + }, + options?: PrimeAgentRecoveryCommitOptions, + ) => Effect.Effect; + readonly discardPrepared: ( + input: { readonly threadId: string; readonly ownerToken: string }, + options?: PrimeAgentRecoveryCommitOptions, + ) => Effect.Effect; + readonly updateTranscriptProgress: ( + input: { + readonly threadId: string; + readonly ownerToken: string; + readonly cursor: typeof RecoveryCursor.Type; + readonly messageCount: number; + readonly fingerprints: ReadonlyArray; + readonly updatedAt: string; + }, + options?: PrimeAgentRecoveryCommitOptions, + ) => Effect.Effect; /** * Compare-and-swap one stable adoption route without replacing the current native authority. * The old owner and bearer handle remain current until native confirmation succeeds. */ - readonly claim: (input: { - readonly threadId: string; - readonly expectedOwnerToken: string; - readonly nextOwnerToken: string; - readonly requestId: string; - readonly mcpOwnerId: string; - readonly updatedAt: string; - }) => Effect.Effect, PrimeAgentRecoveryLedgerError>; + readonly claim: ( + input: { + readonly threadId: string; + readonly expectedOwnerToken: string; + readonly nextOwnerToken: string; + readonly requestId: string; + readonly mcpOwnerId: string; + readonly updatedAt: string; + }, + options?: PrimeAgentRecoveryCommitOptions, + ) => Effect.Effect, PrimeAgentRecoveryLedgerError>; /** Records an attempt before the native prepare request can begin. */ - readonly beginAdoptionAttempt: (input: { - readonly threadId: string; - readonly ownerToken: string; - readonly requestId: string; - readonly updatedAt: string; - }) => Effect.Effect, PrimeAgentRecoveryLedgerError>; + readonly beginAdoptionAttempt: ( + input: { + readonly threadId: string; + readonly ownerToken: string; + readonly requestId: string; + readonly updatedAt: string; + }, + options?: PrimeAgentRecoveryCommitOptions, + ) => Effect.Effect, PrimeAgentRecoveryLedgerError>; /** Releases only a never-started claim. Any native ambiguity keeps the durable route. */ - readonly releaseClaim: (input: { - readonly threadId: string; - readonly ownerToken: string; - readonly previousOwnerToken: string; - readonly requestId: string; - readonly updatedAt: string; - }) => Effect.Effect; + readonly releaseClaim: ( + input: { + readonly threadId: string; + readonly ownerToken: string; + readonly previousOwnerToken: string; + readonly requestId: string; + readonly updatedAt: string; + }, + options?: PrimeAgentRecoveryCommitOptions, + ) => Effect.Effect; /** Persist the complete rotated receipt separately before the SDK confirmation step. */ - readonly commitAdoption: (input: { - readonly threadId: string; - readonly ownerToken: string; - readonly requestId: string; - readonly recoveryHandle: string; - readonly proof: PrimeAgentRecoveryAdoptionProof; - readonly updatedAt: string; - }) => Effect.Effect; + readonly commitAdoption: ( + input: { + readonly threadId: string; + readonly ownerToken: string; + readonly requestId: string; + readonly recoveryHandle: string; + readonly proof: PrimeAgentRecoveryAdoptionProof; + readonly updatedAt: string; + }, + options?: PrimeAgentRecoveryCommitOptions, + ) => Effect.Effect; /** Records the confirmation attempt before it can close the old-handle retry window. */ - readonly beginAdoptionConfirmation: (input: { - readonly threadId: string; - readonly ownerToken: string; - readonly requestId: string; - readonly updatedAt: string; - }) => Effect.Effect, PrimeAgentRecoveryLedgerError>; + readonly beginAdoptionConfirmation: ( + input: { + readonly threadId: string; + readonly ownerToken: string; + readonly requestId: string; + readonly updatedAt: string; + }, + options?: PrimeAgentRecoveryCommitOptions, + ) => Effect.Effect, PrimeAgentRecoveryLedgerError>; /** Promotes the staged receipt only after exact native confirmation. */ - readonly finalizeAdoption: (input: { - readonly threadId: string; - readonly ownerToken: string; - readonly requestId: string; - readonly recoveryHandle: string; - readonly proof: PrimeAgentRecoveryAdoptionProof; - readonly updatedAt: string; - }) => Effect.Effect; - readonly quarantineAdoption: (input: { - readonly threadId: string; - readonly ownerToken: string; - readonly requestId: string; - readonly updatedAt: string; - }) => Effect.Effect; - readonly markNativeCleanup: (input: { - readonly threadId: string; - readonly ownerToken: string; - readonly updatedAt: string; - }) => Effect.Effect; - readonly markTerminalProjected: (input: { - readonly threadId: string; - readonly updatedAt: string; - }) => Effect.Effect; - readonly markCheckpointQuiesced: (input: { - readonly threadId: string; - readonly updatedAt: string; - }) => Effect.Effect; + readonly finalizeAdoption: ( + input: { + readonly threadId: string; + readonly ownerToken: string; + readonly requestId: string; + readonly recoveryHandle: string; + readonly proof: PrimeAgentRecoveryAdoptionProof; + readonly updatedAt: string; + }, + options?: PrimeAgentRecoveryCommitOptions, + ) => Effect.Effect; + readonly quarantineAdoption: ( + input: { + readonly threadId: string; + readonly ownerToken: string; + readonly requestId: string; + readonly updatedAt: string; + }, + options?: PrimeAgentRecoveryCommitOptions, + ) => Effect.Effect; + readonly markNativeCleanup: ( + input: { readonly threadId: string; readonly ownerToken: string; readonly updatedAt: string }, + options?: PrimeAgentRecoveryCommitOptions, + ) => Effect.Effect; + readonly markTerminalProjected: ( + input: { readonly threadId: string; readonly updatedAt: string }, + options?: PrimeAgentRecoveryCommitOptions, + ) => Effect.Effect; + readonly markCheckpointQuiesced: ( + input: { readonly threadId: string; readonly updatedAt: string }, + options?: PrimeAgentRecoveryCommitOptions, + ) => Effect.Effect; /** Deletes only after native cleanup, terminal projection, and checkpoint quiescence all hold. */ readonly deleteIfSettled: ( threadId: string, + options?: PrimeAgentRecoveryCommitOptions, ) => Effect.Effect; } @@ -383,10 +467,12 @@ export const make = Effect.gen(function* () { const get: PrimeAgentRecoveryLedgerShape["get"] = (threadId) => queryByThread(threadId).pipe(Effect.map((rows) => Option.fromNullishOr(rows[0]))); - const putPrepared: PrimeAgentRecoveryLedgerShape["putPrepared"] = (authority) => - sql - .unsafe( - `INSERT INTO prime_agent_recovery_ledger ( + const putPrepared: PrimeAgentRecoveryLedgerShape["putPrepared"] = (authority, options) => + Effect.flatMap(options?.commitGuard ?? Effect.succeed(true), (current) => + current + ? sql + .unsafe( + `INSERT INTO prime_agent_recovery_ledger ( thread_id, provider_instance_id, session_incarnation_id, admission_request_id, turn_id, package_root, package_version, managed_build_id, sdk_features_json, daemon_capabilities_json, protocol_name, protocol_version, schema_revision, active_session_id, native_session_id, @@ -399,50 +485,52 @@ export const make = Effect.gen(function* () { updated_at ) VALUES (${Array.from({ length: 40 }, () => "?").join(",")}) `, - [ - authority.threadId, - authority.providerInstanceId, - authority.sessionIncarnationId, - authority.admissionRequestId, - authority.turnId, - authority.packageRoot, - authority.packageVersion, - authority.managedBuildId, - JSON.stringify(authority.sdkFeatures), - JSON.stringify(authority.daemonCapabilities), - authority.protocolName, - authority.protocolVersion, - authority.schemaRevision, - authority.activeSessionId, - authority.nativeSessionId, - authority.recoveryHandle, - authority.supervisorGeneration, - authority.ownershipGeneration, - authority.cursor.generation, - authority.cursor.sequence, - authority.correlationId, - authority.mcpOwnerId, - JSON.stringify(authority.recoveryConfig), - JSON.stringify(authority.launchEnvironment), - authority.transcriptMessageCount, - JSON.stringify(authority.transcriptFingerprints), - authority.ownerToken, - authority.state, - authority.adoptionPreviousOwnerToken, - authority.adoptionOwnerToken, - authority.adoptionRequestId, - authority.adoptionMcpOwnerId, - authority.adoptionPhase, - authority.adoptionAttempt, - authority.adoptionRecoveryHandle, - authority.adoptionProof === null ? null : JSON.stringify(authority.adoptionProof), - authority.nativeCleanupProven ? 1 : 0, - authority.terminalProjected ? 1 : 0, - authority.checkpointQuiesced ? 1 : 0, - authority.updatedAt, - ], - ) - .pipe(Effect.mapError(mapSqlError("putPrepared")), Effect.asVoid); + [ + authority.threadId, + authority.providerInstanceId, + authority.sessionIncarnationId, + authority.admissionRequestId, + authority.turnId, + authority.packageRoot, + authority.packageVersion, + authority.managedBuildId, + JSON.stringify(authority.sdkFeatures), + JSON.stringify(authority.daemonCapabilities), + authority.protocolName, + authority.protocolVersion, + authority.schemaRevision, + authority.activeSessionId, + authority.nativeSessionId, + authority.recoveryHandle, + authority.supervisorGeneration, + authority.ownershipGeneration, + authority.cursor.generation, + authority.cursor.sequence, + authority.correlationId, + authority.mcpOwnerId, + JSON.stringify(authority.recoveryConfig), + JSON.stringify(authority.launchEnvironment), + authority.transcriptMessageCount, + JSON.stringify(authority.transcriptFingerprints), + authority.ownerToken, + authority.state, + authority.adoptionPreviousOwnerToken, + authority.adoptionOwnerToken, + authority.adoptionRequestId, + authority.adoptionMcpOwnerId, + authority.adoptionPhase, + authority.adoptionAttempt, + authority.adoptionRecoveryHandle, + authority.adoptionProof === null ? null : JSON.stringify(authority.adoptionProof), + authority.nativeCleanupProven ? 1 : 0, + authority.terminalProjected ? 1 : 0, + authority.checkpointQuiesced ? 1 : 0, + authority.updatedAt, + ], + ) + .pipe(Effect.mapError(mapSqlError("putPrepared")), Effect.asVoid) + : Effect.void, + ); const listActive: PrimeAgentRecoveryLedgerShape["listActive"] = () => sql @@ -458,30 +546,38 @@ export const make = Effect.gen(function* () { operation: string, statement: string, parameters: ReadonlyArray, + options?: PrimeAgentRecoveryCommitOptions, ) => - sql.unsafe(statement, parameters).pipe( - Effect.mapError(mapSqlError(operation)), - Effect.map((rows) => Array.isArray(rows) && rows.length === 1), + Effect.flatMap(options?.commitGuard ?? Effect.succeed(true), (current) => + current + ? sql.unsafe(statement, parameters).pipe( + Effect.mapError(mapSqlError(operation)), + Effect.map((rows) => Array.isArray(rows) && rows.length === 1), + ) + : Effect.succeed(false), ); - const markAdmitted: PrimeAgentRecoveryLedgerShape["markAdmitted"] = (input) => + const markAdmitted: PrimeAgentRecoveryLedgerShape["markAdmitted"] = (input, options) => conditionalUpdate( "markAdmitted", `UPDATE prime_agent_recovery_ledger SET turn_id=?, state='active', updated_at=? WHERE thread_id=? AND owner_token=? AND state='prepared' RETURNING thread_id`, [input.turnId, input.updatedAt, input.threadId, input.ownerToken], + options, ); - const discardPrepared: PrimeAgentRecoveryLedgerShape["discardPrepared"] = (input) => + const discardPrepared: PrimeAgentRecoveryLedgerShape["discardPrepared"] = (input, options) => conditionalUpdate( "discardPrepared", `DELETE FROM prime_agent_recovery_ledger WHERE thread_id=? AND owner_token=? AND state='prepared' RETURNING thread_id`, [input.threadId, input.ownerToken], + options, ); const updateTranscriptProgress: PrimeAgentRecoveryLedgerShape["updateTranscriptProgress"] = ( input, + options, ) => conditionalUpdate( "updateTranscriptProgress", @@ -498,9 +594,10 @@ export const make = Effect.gen(function* () { input.threadId, input.ownerToken, ], + options, ); - const claim: PrimeAgentRecoveryLedgerShape["claim"] = (input) => + const claim: PrimeAgentRecoveryLedgerShape["claim"] = (input, options) => conditionalUpdate( "claim", `UPDATE prime_agent_recovery_ledger @@ -518,11 +615,15 @@ export const make = Effect.gen(function* () { input.threadId, input.expectedOwnerToken, ], + options, ).pipe( Effect.flatMap((claimed) => (claimed ? get(input.threadId) : Effect.succeed(Option.none()))), ); - const beginAdoptionAttempt: PrimeAgentRecoveryLedgerShape["beginAdoptionAttempt"] = (input) => + const beginAdoptionAttempt: PrimeAgentRecoveryLedgerShape["beginAdoptionAttempt"] = ( + input, + options, + ) => conditionalUpdate( "beginAdoptionAttempt", `UPDATE prime_agent_recovery_ledger @@ -539,11 +640,12 @@ export const make = Effect.gen(function* () { input.requestId, PRIME_AGENT_RECOVERY_ADOPTION_MAX_ATTEMPTS, ], + options, ).pipe( Effect.flatMap((started) => (started ? get(input.threadId) : Effect.succeed(Option.none()))), ); - const releaseClaim: PrimeAgentRecoveryLedgerShape["releaseClaim"] = (input) => + const releaseClaim: PrimeAgentRecoveryLedgerShape["releaseClaim"] = (input, options) => conditionalUpdate( "releaseClaim", `UPDATE prime_agent_recovery_ledger @@ -563,9 +665,10 @@ export const make = Effect.gen(function* () { input.ownerToken, input.requestId, ], + options, ); - const commitAdoption: PrimeAgentRecoveryLedgerShape["commitAdoption"] = (input) => { + const commitAdoption: PrimeAgentRecoveryLedgerShape["commitAdoption"] = (input, options) => { const proofJson = encodeAdoptionProof(input.proof); return conditionalUpdate( "commitAdoption", @@ -587,11 +690,13 @@ export const make = Effect.gen(function* () { input.recoveryHandle, proofJson, ], + options, ); }; const beginAdoptionConfirmation: PrimeAgentRecoveryLedgerShape["beginAdoptionConfirmation"] = ( input, + options, ) => conditionalUpdate( "beginAdoptionConfirmation", @@ -609,11 +714,12 @@ export const make = Effect.gen(function* () { input.requestId, PRIME_AGENT_RECOVERY_ADOPTION_MAX_ATTEMPTS, ], + options, ).pipe( Effect.flatMap((started) => (started ? get(input.threadId) : Effect.succeed(Option.none()))), ); - const finalizeAdoption: PrimeAgentRecoveryLedgerShape["finalizeAdoption"] = (input) => { + const finalizeAdoption: PrimeAgentRecoveryLedgerShape["finalizeAdoption"] = (input, options) => { const proofJson = encodeAdoptionProof(input.proof); return conditionalUpdate( "finalizeAdoption", @@ -641,10 +747,14 @@ export const make = Effect.gen(function* () { input.recoveryHandle, proofJson, ], + options, ); }; - const quarantineAdoption: PrimeAgentRecoveryLedgerShape["quarantineAdoption"] = (input) => + const quarantineAdoption: PrimeAgentRecoveryLedgerShape["quarantineAdoption"] = ( + input, + options, + ) => conditionalUpdate( "quarantineAdoption", `UPDATE prime_agent_recovery_ledger @@ -659,40 +769,49 @@ export const make = Effect.gen(function* () { input.requestId, PRIME_AGENT_RECOVERY_ADOPTION_MAX_ATTEMPTS, ], + options, ); - const markNativeCleanup: PrimeAgentRecoveryLedgerShape["markNativeCleanup"] = (input) => + const markNativeCleanup: PrimeAgentRecoveryLedgerShape["markNativeCleanup"] = (input, options) => conditionalUpdate( "markNativeCleanup", `UPDATE prime_agent_recovery_ledger SET native_cleanup_proven=1, state='terminal', updated_at=? WHERE thread_id=? AND owner_token=? RETURNING thread_id`, [input.updatedAt, input.threadId, input.ownerToken], + options, ); - const markTerminalProjected: PrimeAgentRecoveryLedgerShape["markTerminalProjected"] = (input) => - sql - .unsafe( - `UPDATE prime_agent_recovery_ledger SET terminal_projected=1, updated_at=? WHERE thread_id=?`, - [input.updatedAt, input.threadId], - ) - .pipe(Effect.mapError(mapSqlError("markTerminalProjected")), Effect.asVoid); + const markTerminalProjected: PrimeAgentRecoveryLedgerShape["markTerminalProjected"] = ( + input, + options, + ) => + conditionalUpdate( + "markTerminalProjected", + `UPDATE prime_agent_recovery_ledger SET terminal_projected=1, updated_at=? WHERE thread_id=? RETURNING thread_id`, + [input.updatedAt, input.threadId], + options, + ).pipe(Effect.asVoid); - const markCheckpointQuiesced: PrimeAgentRecoveryLedgerShape["markCheckpointQuiesced"] = (input) => - sql - .unsafe( - `UPDATE prime_agent_recovery_ledger SET checkpoint_quiesced=1, updated_at=? WHERE thread_id=?`, - [input.updatedAt, input.threadId], - ) - .pipe(Effect.mapError(mapSqlError("markCheckpointQuiesced")), Effect.asVoid); + const markCheckpointQuiesced: PrimeAgentRecoveryLedgerShape["markCheckpointQuiesced"] = ( + input, + options, + ) => + conditionalUpdate( + "markCheckpointQuiesced", + `UPDATE prime_agent_recovery_ledger SET checkpoint_quiesced=1, updated_at=? WHERE thread_id=? RETURNING thread_id`, + [input.updatedAt, input.threadId], + options, + ).pipe(Effect.asVoid); - const deleteIfSettled: PrimeAgentRecoveryLedgerShape["deleteIfSettled"] = (threadId) => + const deleteIfSettled: PrimeAgentRecoveryLedgerShape["deleteIfSettled"] = (threadId, options) => conditionalUpdate( "deleteIfSettled", `DELETE FROM prime_agent_recovery_ledger WHERE thread_id=? AND native_cleanup_proven=1 AND terminal_projected=1 AND checkpoint_quiesced=1 RETURNING thread_id`, [threadId], + options, ); return { diff --git a/apps/server/src/provider/prime/PrimeAgentRestartAdoption.real.test.mjs b/apps/server/src/provider/prime/PrimeAgentRestartAdoption.real.test.mjs index 34bdf1fce..1eef44f5a 100644 --- a/apps/server/src/provider/prime/PrimeAgentRestartAdoption.real.test.mjs +++ b/apps/server/src/provider/prime/PrimeAgentRestartAdoption.real.test.mjs @@ -1,4 +1,4 @@ -/* eslint-disable t3code/no-manual-effect-runtime-in-tests -- This opt-in POSIX proof drives two external server processes through their public RPC boundary. */ +/* eslint-disable t3code/no-manual-effect-runtime-in-tests -- This opt-in POSIX proof drives repeated external server processes through their public RPC boundary. */ import * as NodeChildProcess from "node:child_process"; import * as NodeCrypto from "node:crypto"; import * as NodeFSP from "node:fs/promises"; @@ -23,11 +23,11 @@ import { describe, expect, it } from "vite-plus/test"; import { persistPrimeManagedReceipt } from "./PrimeAgentDistributionVerifier.ts"; const packageRoot = NodeProcess.env.PRIME_AGENT_REAL_PACKAGE_ROOT?.trim(); -const exactHead = "507a52239d3ace7bb2b2965ade7779988fdb6344"; +const exactHead = "a3dd5ce633fef161d30ded9474f75a609a9e7a2a"; const skipReason = NodeProcess.platform === "win32" ? "native Windows is unsupported; run the POSIX proof in WSL2 with a Linux PRIME_AGENT_REAL_PACKAGE_ROOT" - : "set PRIME_AGENT_REAL_PACKAGE_ROOT to the built exact Prime checkout at 507a52239d3ace7bb2b2965ade7779988fdb6344"; + : `set PRIME_AGENT_REAL_PACKAGE_ROOT to the built exact Prime checkout at ${exactHead}`; const enabled = NodeProcess.platform !== "win32" && Boolean(packageRoot); const outerSafetyMs = 180_000; const maximumOutputBytes = 2 * 1024 * 1024; @@ -654,6 +654,17 @@ const readLedger = (databasePath, threadId) => { } }; +const readOwnershipReceipt = async (stateDir) => { + const directory = NodePath.join(stateDir, "provider-sessions", "prime-agent", "native-ownership"); + const entries = (await NodeFSP.readdir(directory)).filter( + (name) => name.endsWith(".json") && !name.endsWith(".json.lock"), + ); + if (entries.length !== 1) { + throw new Error(`expected one Prime ownership receipt, found ${entries.length}`); + } + return JSON.parse(await NodeFSP.readFile(NodePath.join(directory, entries[0]), "utf8")); +}; + const readProviderSessionRuntime = (databasePath, threadId) => { const database = new NodeSqlite.DatabaseSync(databasePath, { readOnly: true }); try { @@ -890,7 +901,7 @@ const runRestartedTurn = ({ wsUrl, threadId, fixture, onRecoveredActivity }) => ); describe.skipIf(!enabled)( - `Prime Agent two-Pylon-server restart adoption (${enabled ? "enabled" : skipReason})`, + `Prime Agent repeated Pylon-server restart adoption (${enabled ? "enabled" : skipReason})`, () => { it( "adopts one live owned worker across the real server boundary and cleans it authoritatively", @@ -918,7 +929,7 @@ describe.skipIf(!enabled)( expect(sourceHead).toBe(exactHead); const temp = await NodeFSP.realpath( - await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "pylon-two-server-adoption-")), + await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "pylon-repeated-server-adoption-")), ); const home = NodePath.join(temp, "home"); const baseDir = NodePath.join(temp, "server-home"); @@ -1019,7 +1030,7 @@ describe.skipIf(!enabled)( type: "project.create", commandId: `project-command-${NodeCrypto.randomUUID()}`, projectId, - title: "Two-server adoption proof", + title: "Repeated-server adoption proof", workspaceRoot: projectDir, defaultModelSelection: modelSelection, createdAt, @@ -1100,6 +1111,19 @@ describe.skipIf(!enabled)( turn_id: activeSnapshot.thread.session.activeTurnId, state: "active", }); + const receiptA = await readOwnershipReceipt(stateDir); + expect(receiptA).toMatchObject({ + state: "acquired", + activeSessionId: ledgerA.active_session_id, + nativeSessionId: ledgerA.native_session_id, + recovery: { + threadId, + sessionIncarnationId: ledgerA.session_incarnation_id, + admissionRequestId: ledgerA.admission_request_id, + recoveryHandle: ledgerA.recovery_handle, + ownershipGeneration: ledgerA.ownership_generation, + }, + }); const sessionFilesBefore = await listSessionJsonlFiles(stateDir); expect(sessionFilesBefore).toHaveLength(1); expect(await NodeFSP.lstat(daemonSocket)).toMatchObject({}); @@ -1130,18 +1154,28 @@ describe.skipIf(!enabled)( stage: "after-claim-persisted", phase: "claimed", attempt: 0, + receipt: "current", }, { label: "server C after native response", stage: "after-native-response-before-commit", phase: "requested", attempt: 1, + receipt: "current", }, { - label: "server D after durable rotated receipt", - stage: "after-commit-before-confirm", + label: "server D after durable ledger commit", + stage: "after-commit-before-receipt", phase: "committed", attempt: 2, + receipt: "current", + }, + { + label: "server E after durable receipt rotation", + stage: "after-receipt-before-confirm", + phase: "committed", + attempt: 2, + receipt: "staged", }, ]; let adoptionRequestId; @@ -1195,6 +1229,24 @@ describe.skipIf(!enabled)( expect(crashLedger.adoption_recovery_handle).toBeNull(); expect(crashLedger.adoption_proof_json).toBeNull(); } + const crashReceipt = await readOwnershipReceipt(stateDir); + expect(crashReceipt).toMatchObject({ + attemptId: receiptA.attemptId, + activeSessionId: ledgerA.active_session_id, + nativeSessionId: ledgerA.native_session_id, + }); + if (crashCase.receipt === "staged") { + const adoptionProof = JSON.parse(crashLedger.adoption_proof_json); + expect(crashReceipt.recovery).toMatchObject({ + threadId, + sessionIncarnationId: ledgerA.session_incarnation_id, + admissionRequestId: ledgerA.admission_request_id, + recoveryHandle: crashLedger.adoption_recovery_handle, + ownershipGeneration: adoptionProof.ownershipGeneration, + }); + } else { + expect(crashReceipt.recovery).toEqual(receiptA.recovery); + } expect(fixture.records).toHaveLength(1); expect(processExists(workerPid)).toBe(true); } @@ -1206,13 +1258,13 @@ describe.skipIf(!enabled)( projectDir, home, port: portB, - label: "server E", + label: "server F", }); const wsB = await issueWebSocketUrl(serverB.baseUrl, bearerToken); await runRpc( wsB, (client) => client[WS_METHODS.serverProbe]({}), - "server E command readiness after repeated adoption recovery", + "server F command readiness after repeated adoption recovery", ); expect(fixture.records).toHaveLength(1); const ledgerAfterRepeatedCrash = await waitForLedger( @@ -1239,6 +1291,13 @@ describe.skipIf(!enabled)( ownerRotated: true, handleRotated: true, }); + expect((await readOwnershipReceipt(stateDir)).recovery).toMatchObject({ + threadId, + sessionIncarnationId: ledgerAfterRepeatedCrash.session_incarnation_id, + admissionRequestId: ledgerAfterRepeatedCrash.admission_request_id, + recoveryHandle: ledgerAfterRepeatedCrash.recovery_handle, + ownershipGeneration: ledgerAfterRepeatedCrash.ownership_generation, + }); if (!processExists(workerPid)) { throw new Error( `captured Prime worker ${workerPid} exited before recovered activity\n${serverB.output()}`, diff --git a/apps/server/src/provider/prime/PrimeAgentRuntimeContext.test.ts b/apps/server/src/provider/prime/PrimeAgentRuntimeContext.test.ts new file mode 100644 index 000000000..16757ffee --- /dev/null +++ b/apps/server/src/provider/prime/PrimeAgentRuntimeContext.test.ts @@ -0,0 +1,223 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import { PrimeAgentSettings, ProviderInstanceId } from "@t3tools/contracts"; +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import { + bindPrimeAgentRuntimeContext, + materializePrimeAgentIdentities, + PRIME_AGENT_COMPAT_HOME_ENV, + PRIME_AGENT_HOME_ENV, + type PrimeAgentIdentityInput, + type PrimeAgentIdentityPreparation, +} from "./PrimeAgentRuntimeContext.ts"; +import { PrimeAgentOwnershipReceiptStore } from "./PrimeAgentOwnershipReceipt.ts"; + +const decodeSettings = Schema.decodeSync(PrimeAgentSettings); + +const input = ( + id: string, + agentHomePath: string, + environment: PrimeAgentIdentityInput["environment"] = [], +): PrimeAgentIdentityInput => ({ + instanceId: ProviderInstanceId.make(id), + environment, + enabled: true, + config: decodeSettings({ agentHomePath }), +}); + +const readyIdentity = ( + result: ReadonlyMap, + id: string, +) => { + const prepared = result.get(ProviderInstanceId.make(id)); + expect(prepared?.kind).toBe("ready"); + if (prepared?.kind !== "ready") throw new Error("expected ready identity"); + return prepared.identity; +}; + +it.layer(NodeServices.layer)("PrimeAgentRuntimeContext", (it) => { + it.effect("materializes one frozen exact environment with explicit-home precedence", () => + Effect.scoped( + Effect.gen(function* () { + const root = yield* Effect.acquireRelease( + Effect.promise(() => NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "prime-context-"))), + (directory) => + Effect.promise(() => NodeFSP.rm(directory, { recursive: true, force: true })), + ); + const canonicalRoot = yield* Effect.promise(() => NodeFSP.realpath(root)); + const explicitHome = NodePath.join(root, "explicit"); + const environmentHome = NodePath.join(root, "environment"); + yield* Effect.promise(() => NodeFSP.mkdir(explicitHome, { recursive: true })); + + const prepared = yield* materializePrimeAgentIdentities([ + input("prime-a", explicitHome, [ + { name: "HOME", value: NodePath.join(root, "first-home"), sensitive: false }, + { name: "HOME", value: NodePath.join(root, "effective-home"), sensitive: false }, + { name: PRIME_AGENT_HOME_ENV, value: environmentHome, sensitive: false }, + { + name: PRIME_AGENT_COMPAT_HOME_ENV, + value: NodePath.join(root, "compat"), + sensitive: false, + }, + { name: "PRIME_AGENT_INTERNAL_TOKEN", value: "secret-a", sensitive: false }, + { name: "rlm_depth", value: "9", sensitive: false }, + { name: "force_color", value: "1", sensitive: false }, + { name: "KEEP_ME", value: "yes", sensitive: false }, + ]), + ]).pipe( + Effect.provideService(HostProcessEnvironment, { + HOME: NodePath.join(root, "host-home"), + PRIME_AGENT_INTERNAL_ROLE: "server", + RLM_CHILD_ID: "secret-b", + }), + ); + const identity = readyIdentity(prepared, "prime-a"); + expect(identity.nativeMultipleInstancesRequired).toBe(false); + + const canonicalExplicitHome = NodePath.join(canonicalRoot, "explicit"); + expect(identity.effectiveHome).toBe(canonicalExplicitHome); + expect(identity.settings.agentHomePath).toBe(canonicalExplicitHome); + expect(identity.launchEnv).toMatchObject({ + HOME: NodePath.join(root, "effective-home"), + KEEP_ME: "yes", + [PRIME_AGENT_HOME_ENV]: canonicalExplicitHome, + [PRIME_AGENT_COMPAT_HOME_ENV]: canonicalExplicitHome, + }); + expect( + Object.keys(identity.launchEnv).some( + (name) => + name.toUpperCase().startsWith("PRIME_AGENT_INTERNAL_") || + name.toUpperCase().startsWith("RLM_") || + name.toUpperCase() === "FORCE_COLOR", + ), + ).toBe(false); + expect(Object.isFrozen(identity)).toBe(true); + expect(Object.isFrozen(identity.launchEnv)).toBe(true); + expect(Object.isFrozen(identity.settings)).toBe(true); + expect(Object.isFrozen(identity.settings.customModels)).toBe(true); + const runtimeContext = bindPrimeAgentRuntimeContext(identity, { + kind: "acp", + fallbackCategory: "daemon-setup", + }); + expect(Object.isFrozen(runtimeContext)).toBe(true); + expect(Object.isFrozen(runtimeContext.backendIdentity)).toBe(true); + + const ownership = { + store: new PrimeAgentOwnershipReceiptStore(root), + adoptableReceipts: [], + }; + const identityWithOwnership = { ...identity, nativeOwnership: ownership }; + const acpContext = bindPrimeAgentRuntimeContext(identityWithOwnership, { kind: "acp" }); + expect(acpContext.nativeOwnership).toBeUndefined(); + const daemonContext = bindPrimeAgentRuntimeContext(identityWithOwnership, { + kind: "daemon", + proof: { + sdkFeatures: [], + requiredServerCapabilities: [ + "caller_owned_session_environment_cleanup_v1", + "authoritative_owned_session_cleanup_v1", + ], + }, + }); + expect(daemonContext.nativeOwnership).toBe(ownership); + }), + ), + ); + + it.effect("uses environment home and HOME-derived default without ambient fallback", () => + Effect.scoped( + Effect.gen(function* () { + const root = yield* Effect.acquireRelease( + Effect.promise(() => NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "prime-home-"))), + (directory) => + Effect.promise(() => NodeFSP.rm(directory, { recursive: true, force: true })), + ); + const canonicalRoot = yield* Effect.promise(() => NodeFSP.realpath(root)); + const environmentHome = NodePath.join(root, "environment-agent"); + const effectiveOsHome = NodePath.join(root, "os-home"); + yield* Effect.promise(() => + Promise.all([ + NodeFSP.mkdir(environmentHome, { recursive: true }), + NodeFSP.mkdir(effectiveOsHome, { recursive: true }), + ]), + ); + + const prepared = yield* materializePrimeAgentIdentities([ + input("prime-env", "", [ + { name: PRIME_AGENT_COMPAT_HOME_ENV, value: environmentHome, sensitive: false }, + ]), + input("prime-default", "", [{ name: "HOME", value: effectiveOsHome, sensitive: false }]), + input("prime-relative", "relative/home", [ + { name: "HOME", value: effectiveOsHome, sensitive: false }, + ]), + ]).pipe( + Effect.provideService(HostProcessEnvironment, { HOME: NodePath.join(root, "ambient") }), + ); + + expect(readyIdentity(prepared, "prime-env").effectiveHome).toBe( + NodePath.join(canonicalRoot, "environment-agent"), + ); + expect(readyIdentity(prepared, "prime-default").effectiveHome).toBe( + NodePath.join(canonicalRoot, "os-home", ".prime", "agent"), + ); + const invalid = prepared.get(ProviderInstanceId.make("prime-relative")); + expect(invalid?.kind).toBe("unavailable"); + if (invalid?.kind === "unavailable") { + expect(invalid.error.detail).not.toContain("relative/home"); + expect(invalid.error.detail).not.toContain(root); + } + }), + ), + ); + + it.effect("rejects every enabled participant in nested and symlink-aliased overlap groups", () => + Effect.scoped( + Effect.gen(function* () { + const root = yield* Effect.acquireRelease( + Effect.promise(() => NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "prime-overlap-"))), + (directory) => + Effect.promise(() => NodeFSP.rm(directory, { recursive: true, force: true })), + ); + const canonicalRoot = yield* Effect.promise(() => NodeFSP.realpath(root)); + const shared = NodePath.join(root, "shared"); + const alias = NodePath.join(root, "alias"); + const separate = NodePath.join(root, "separate"); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(shared, { recursive: true }); + await NodeFSP.mkdir(separate, { recursive: true }); + await NodeFSP.symlink(shared, alias, "dir"); + }); + + const prepared = yield* materializePrimeAgentIdentities([ + input("prime-parent", shared), + input("prime-child", NodePath.join(shared, "nested", "future")), + input("prime-alias", alias), + input("prime-separate", separate), + ]).pipe( + Effect.provideService(HostProcessEnvironment, { HOME: root }), + Effect.provideService(HostProcessPlatform, "linux"), + ); + + for (const id of ["prime-parent", "prime-child", "prime-alias"]) { + const result = prepared.get(ProviderInstanceId.make(id)); + expect(result?.kind).toBe("unavailable"); + if (result?.kind === "unavailable") { + expect(result.error.detail).toContain("distinct homes"); + expect(result.error.detail).not.toContain(root); + } + } + const separateIdentity = readyIdentity(prepared, "prime-separate"); + expect(separateIdentity.effectiveHome).toBe(NodePath.join(canonicalRoot, "separate")); + expect(separateIdentity.nativeMultipleInstancesRequired).toBe(true); + }), + ), + ); +}); diff --git a/apps/server/src/provider/prime/PrimeAgentRuntimeContext.ts b/apps/server/src/provider/prime/PrimeAgentRuntimeContext.ts new file mode 100644 index 000000000..9e23342c9 --- /dev/null +++ b/apps/server/src/provider/prime/PrimeAgentRuntimeContext.ts @@ -0,0 +1,360 @@ +import { + type PrimeAgentSettings, + type ProviderInstanceEnvironment, + type ProviderInstanceId, +} from "@t3tools/contracts"; +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as Crypto from "effect/Crypto"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Result from "effect/Result"; +import * as Schema from "effect/Schema"; + +import { ProviderDriverError } from "../Errors.ts"; +import type { ProviderRuntimeFence } from "../ProviderDriver.ts"; +import type { + PrimeAgentAcquiredOwnershipReceipt, + PrimeAgentOwnershipReceiptStore, +} from "./PrimeAgentOwnershipReceipt.ts"; + +export const PRIME_AGENT_HOME_ENV = "PRIME_AGENT_CODING_AGENT_DIR" as const; +export const PRIME_AGENT_COMPAT_HOME_ENV = "PRIME_AGENT_HOME" as const; +export const PRIME_AGENT_CALLER_OWNED_SESSION_FEATURE = + "caller_owned_session_environment_cleanup_v1" as const; +export const PRIME_AGENT_AUTHORITATIVE_CLEANUP_CAPABILITY = + "authoritative_owned_session_cleanup_v1" as const; +export const PRIME_AGENT_NEGOTIATED_CAPABILITIES_FEATURE = + "negotiated_daemon_session_capabilities_v1" as const; + +const DRIVER_KIND = "primeAgent" as const; +const DISTINCT_HOME_REASON = + "Prime Agent instances need distinct homes. Configure each enabled instance with a home that is neither equal to nor nested inside another Prime Agent instance home."; +const INVALID_HOME_REASON = + "Prime Agent needs a safe absolute home. Configure an absolute Agent home path, or set an absolute HOME for the instance."; +const HOME_RESOLUTION_FAILED_REASON = + "Prime Agent could not safely resolve this instance home. Check the configured home and permissions."; + +const reservedPrimeAgentEnvironmentName = (name: string): boolean => { + const normalized = name.toUpperCase(); + return ( + normalized.startsWith("PRIME_AGENT_INTERNAL_") || + normalized.startsWith("RLM_") || + normalized === PRIME_AGENT_HOME_ENV || + normalized === PRIME_AGENT_COMPAT_HOME_ENV || + normalized === "FORCE_COLOR" + ); +}; + +const freezeStringRecord = (record: Record): Readonly> => + Object.freeze({ ...record }); + +class PrimeAgentHomeCanonicalizationError extends Schema.TaggedErrorClass()( + "PrimeAgentHomeCanonicalizationError", + {}, +) {} + +export interface PrimeAgentRuntimeGeneration { + readonly _tag: "PrimeAgentRuntimeGeneration"; +} + +export interface PrimeAgentMaterializedIdentity { + readonly instanceId: ProviderInstanceId; + readonly generation: PrimeAgentRuntimeGeneration; + readonly configRevision: string; + readonly effectiveHome: string; + readonly launchEnv: Readonly>; + readonly settings: Readonly; + /** True only for an enabled participant in a set with more than one enabled Prime instance. */ + readonly nativeMultipleInstancesRequired: boolean; + /** Server-private durable native ownership admission. Never serialize this field. */ + readonly nativeOwnership?: { + readonly store: PrimeAgentOwnershipReceiptStore; + readonly adoptableReceipts: ReadonlyArray; + }; +} + +export interface PrimeAgentNativeProofIdentity { + readonly sdkFeatures: ReadonlyArray; + readonly requiredServerCapabilities: readonly [ + typeof PRIME_AGENT_CALLER_OWNED_SESSION_FEATURE, + typeof PRIME_AGENT_AUTHORITATIVE_CLEANUP_CAPABILITY, + ]; +} + +export type PrimeAgentRuntimeBackendIdentity = + | { + readonly kind: "acp"; + readonly fallbackCategory?: "launch-args" | "binary-resolution" | "daemon-setup"; + } + | { + readonly kind: "daemon"; + readonly proof: PrimeAgentNativeProofIdentity; + }; + +export interface PrimeAgentRuntimeContext extends PrimeAgentMaterializedIdentity { + readonly backendKind: PrimeAgentRuntimeBackendIdentity["kind"]; + readonly runtimeFence?: ProviderRuntimeFence | undefined; + readonly backendIdentity: PrimeAgentRuntimeBackendIdentity; +} + +export interface PrimeAgentIdentityInput { + readonly instanceId: ProviderInstanceId; + readonly environment: ProviderInstanceEnvironment; + readonly enabled: boolean; + readonly config: PrimeAgentSettings; +} + +export type PrimeAgentIdentityPreparation = + | { readonly kind: "ready"; readonly identity: PrimeAgentMaterializedIdentity } + | { readonly kind: "unavailable"; readonly error: ProviderDriverError }; + +function driverError(instanceId: ProviderInstanceId, detail: string): ProviderDriverError { + return new ProviderDriverError({ driver: DRIVER_KIND, instanceId, detail }); +} + +function mergeEnvironment( + hostEnvironment: NodeJS.ProcessEnv, + overrides: ProviderInstanceEnvironment, +): Record { + const merged: Record = { ...hostEnvironment }; + for (const variable of overrides) merged[variable.name] = variable.value; + return merged; +} + +function readEnvironmentValue( + environment: Readonly>, + name: string, +): string | undefined { + const normalizedName = name.toUpperCase(); + let result: string | undefined; + for (const [candidate, value] of Object.entries(environment)) { + if (candidate.toUpperCase() === normalizedName) result = value; + } + return result; +} + +function expandHomeCandidate(input: { + readonly configured: string; + readonly effectiveOsHome: string | undefined; + readonly path: Path.Path; +}): string | undefined { + const configured = input.configured.trim(); + if (configured.length === 0) return undefined; + const expanded = + configured === "~" + ? input.effectiveOsHome + : configured.startsWith("~/") + ? input.effectiveOsHome === undefined + ? undefined + : input.path.join(input.effectiveOsHome, configured.slice(2)) + : configured; + return expanded !== undefined && input.path.isAbsolute(expanded) + ? input.path.normalize(expanded) + : undefined; +} + +const canonicalizePrimeAgentHome = Effect.fn("canonicalizePrimeAgentHome")(function* ( + candidate: string, +) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + let ancestor = path.normalize(candidate); + const suffix: string[] = []; + + while (true) { + const resolved = yield* Effect.result(fileSystem.realPath(ancestor)); + if (Result.isSuccess(resolved)) { + const info = yield* fileSystem.stat(resolved.success); + if (info.type !== "Directory") { + return yield* new PrimeAgentHomeCanonicalizationError(); + } + return suffix.length === 0 ? resolved.success : path.join(resolved.success, ...suffix); + } + const failure = resolved.failure; + if (failure._tag !== "PlatformError" || failure.reason._tag !== "NotFound") { + return yield* failure; + } + const parent = path.dirname(ancestor); + if (parent === ancestor) return yield* failure; + suffix.unshift(path.basename(ancestor)); + ancestor = parent; + } +}); + +function homesOverlap( + path: Path.Path, + left: string, + right: string, + caseInsensitive: boolean, +): boolean { + const normalizedLeft = caseInsensitive ? left.toLowerCase() : left; + const normalizedRight = caseInsensitive ? right.toLowerCase() : right; + const relative = path.relative(normalizedLeft, normalizedRight); + if (relative === "") return true; + const rightInsideLeft = + !path.isAbsolute(relative) && relative !== ".." && !relative.startsWith(`..${path.sep}`); + if (rightInsideLeft) return true; + const inverse = path.relative(normalizedRight, normalizedLeft); + return !path.isAbsolute(inverse) && inverse !== ".." && !inverse.startsWith(`..${path.sep}`); +} + +/** + * Materialize every Prime identity in one pass before any Prime process, import, + * network request, or probe. Explicit settings win over the merged environment; + * the merged environment wins over the default derived from its effective HOME. + */ +export const materializePrimeAgentIdentities = Effect.fn("materializePrimeAgentIdentities")( + function* ( + inputs: ReadonlyArray, + ): Effect.fn.Return< + ReadonlyMap, + never, + Crypto.Crypto | FileSystem.FileSystem | Path.Path + > { + const hostEnvironment = { ...(yield* HostProcessEnvironment) }; + const crypto = yield* Crypto.Crypto; + const path = yield* Path.Path; + const platform = yield* HostProcessPlatform; + const prepared = new Map(); + const ready = new Map(); + const enabledInstanceCount = inputs.filter((input) => input.enabled).length; + + for (const input of inputs) { + const merged = mergeEnvironment(hostEnvironment, input.environment); + const environmentAgentHome = + readEnvironmentValue(merged, PRIME_AGENT_HOME_ENV)?.trim() || + readEnvironmentValue(merged, PRIME_AGENT_COMPAT_HOME_ENV)?.trim(); + const effectiveOsHome = expandHomeCandidate({ + configured: readEnvironmentValue(merged, "HOME")?.trim() ?? "", + effectiveOsHome: undefined, + path, + }); + const configured = + input.config.agentHomePath.trim() || + environmentAgentHome || + (effectiveOsHome === undefined ? "" : path.join(effectiveOsHome, ".prime", "agent")); + const candidate = expandHomeCandidate({ configured, effectiveOsHome, path }); + if (candidate === undefined) { + prepared.set(input.instanceId, { + kind: "unavailable", + error: driverError(input.instanceId, INVALID_HOME_REASON), + }); + continue; + } + + const canonical = yield* Effect.result(canonicalizePrimeAgentHome(candidate)); + if (Result.isFailure(canonical)) { + prepared.set(input.instanceId, { + kind: "unavailable", + error: driverError(input.instanceId, HOME_RESOLUTION_FAILED_REASON), + }); + continue; + } + + const launchEnv: Record = {}; + for (const [name, value] of Object.entries(merged)) { + if (typeof value === "string" && !reservedPrimeAgentEnvironmentName(name)) { + launchEnv[name] = value; + } + } + launchEnv[PRIME_AGENT_HOME_ENV] = canonical.success; + launchEnv[PRIME_AGENT_COMPAT_HOME_ENV] = canonical.success; + const configRevision = yield* crypto.randomUUIDv4.pipe(Effect.orDie); + const identity = Object.freeze({ + instanceId: input.instanceId, + generation: Object.freeze({ _tag: "PrimeAgentRuntimeGeneration" as const }), + configRevision, + effectiveHome: canonical.success, + launchEnv: freezeStringRecord(launchEnv), + settings: Object.freeze({ + ...input.config, + enabled: input.enabled, + agentHomePath: canonical.success, + customModels: Object.freeze([...input.config.customModels]), + }), + nativeMultipleInstancesRequired: input.enabled && enabledInstanceCount > 1, + }) satisfies PrimeAgentMaterializedIdentity; + ready.set(input.instanceId, identity); + prepared.set(input.instanceId, { kind: "ready", identity }); + } + + const overlapping = new Set(); + const enabled = inputs.filter((input) => input.enabled); + for (let leftIndex = 0; leftIndex < enabled.length; leftIndex += 1) { + const leftInput = enabled[leftIndex]; + if (leftInput === undefined) continue; + const left = ready.get(leftInput.instanceId); + if (left === undefined) continue; + for (let rightIndex = leftIndex + 1; rightIndex < enabled.length; rightIndex += 1) { + const rightInput = enabled[rightIndex]; + if (rightInput === undefined) continue; + const right = ready.get(rightInput.instanceId); + if ( + right !== undefined && + homesOverlap( + path, + left.effectiveHome, + right.effectiveHome, + platform === "darwin" || platform === "win32", + ) + ) { + overlapping.add(left.instanceId); + overlapping.add(right.instanceId); + } + } + } + for (const instanceId of overlapping) { + prepared.set(instanceId, { + kind: "unavailable", + error: driverError(instanceId, DISTINCT_HOME_REASON), + }); + } + return prepared; + }, +); + +export function bindPrimeAgentRuntimeContext( + identity: PrimeAgentMaterializedIdentity, + backendIdentity: PrimeAgentRuntimeBackendIdentity, + runtimeFence?: ProviderRuntimeFence, +): PrimeAgentRuntimeContext { + if (runtimeFence !== undefined && runtimeFence.generation !== identity.generation) { + throw new Error("Prime Agent runtime fence does not match its materialized generation."); + } + const immutableBackendIdentity: PrimeAgentRuntimeBackendIdentity = + backendIdentity.kind === "daemon" + ? Object.freeze({ + kind: "daemon", + proof: Object.freeze({ + sdkFeatures: Object.freeze([...backendIdentity.proof.sdkFeatures]), + requiredServerCapabilities: Object.freeze([ + ...backendIdentity.proof.requiredServerCapabilities, + ]) as PrimeAgentNativeProofIdentity["requiredServerCapabilities"], + }), + }) + : Object.freeze({ + kind: "acp", + ...(backendIdentity.fallbackCategory === undefined + ? {} + : { fallbackCategory: backendIdentity.fallbackCategory }), + }); + const runtimeIdentity = + immutableBackendIdentity.kind === "daemon" || identity.nativeOwnership === undefined + ? identity + : { + instanceId: identity.instanceId, + generation: identity.generation, + configRevision: identity.configRevision, + effectiveHome: identity.effectiveHome, + launchEnv: identity.launchEnv, + settings: identity.settings, + nativeMultipleInstancesRequired: identity.nativeMultipleInstancesRequired, + }; + return Object.freeze({ + ...runtimeIdentity, + ...(runtimeFence === undefined ? {} : { runtimeFence }), + backendKind: immutableBackendIdentity.kind, + backendIdentity: immutableBackendIdentity, + }); +} diff --git a/apps/server/src/provider/primeAgentBackends.test.ts b/apps/server/src/provider/primeAgentBackends.test.ts index 870ded91d..ad2b5e502 100644 --- a/apps/server/src/provider/primeAgentBackends.test.ts +++ b/apps/server/src/provider/primeAgentBackends.test.ts @@ -16,7 +16,6 @@ import * as ChildProcess from "effect/unstable/process/ChildProcess"; import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; import { - primeAgentCredentialFingerprint, primeAgentSignInsFromAuthFile, readPrimeAgentBackends, readPrimeAgentCapacity, @@ -45,13 +44,6 @@ const AUTH_FILE = { }; describe("primeAgentSignInsFromAuthFile", () => { - it("builds stable secret-safe credential fingerprints", () => { - const fingerprint = primeAgentCredentialFingerprint("access-secret"); - assert.strictEqual(fingerprint, primeAgentCredentialFingerprint("access-secret")); - assert.notStrictEqual(fingerprint, primeAgentCredentialFingerprint("different-secret")); - assert.notInclude(fingerprint, "access-secret"); - }); - it("keeps each backend's identity and its token while fresh", () => { assert.deepStrictEqual(primeAgentSignInsFromAuthFile(JSON.stringify(AUTH_FILE), NOW), [ { backend: "anthropic", accessToken: "access-secret" }, @@ -301,6 +293,8 @@ it.layer(Layer.mergeAll(NodeServices.layer, UnreachableHttpClient))( { agentHomePath: home }, { sharedCacheDir: cacheDir, + instanceId: "primeAgent", + configRevision: "test-revision", readCodexWindows: (signIn) => Effect.sync(() => { reads.push(`${signIn.accountId}:${signIn.accessToken}`); @@ -331,6 +325,8 @@ it.layer(Layer.mergeAll(NodeServices.layer, UnreachableHttpClient))( { agentHomePath: home }, { sharedCacheDir: cacheDir, + instanceId: "primeAgent", + configRevision: "test-revision", readCodexWindows: () => Effect.die("must be served from the shared reading"), }, ); @@ -360,6 +356,8 @@ it.layer(Layer.mergeAll(NodeServices.layer, UnreachableHttpClient))( }); const options = { sharedCacheDir: cacheDir, + instanceId: "primeAgent", + configRevision: "test-revision", freshForMs: 0, readCodexWindows: read, }; @@ -400,6 +398,8 @@ it.layer(Layer.mergeAll(NodeServices.layer, UnreachableHttpClient))( let reads = 0; const options = { sharedCacheDir: cacheDir, + instanceId: "primeAgent", + configRevision: "test-revision", freshForMs: 0, readCodexWindows: () => Effect.sync(() => { @@ -420,7 +420,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, UnreachableHttpClient))( }).pipe(Effect.scoped), ); - it.effect("does not reuse Anthropic capacity after a token re-login", () => { + it.effect("does not reuse Anthropic capacity after its private config revision rotates", () => { let calls = 0; const httpClient = HttpClient.make((request) => Effect.sync(() => { @@ -451,16 +451,31 @@ it.layer(Layer.mergeAll(NodeServices.layer, UnreachableHttpClient))( yield* writeAuth("token-a"); const first = yield* readPrimeAgentCapacity( { agentHomePath: home }, - { sharedCacheDir: cacheDir, freshForMs: 0 }, + { + sharedCacheDir: cacheDir, + instanceId: "primeAgent", + configRevision: "revision-a", + freshForMs: 0, + }, ); yield* writeAuth("token-b"); const relogged = yield* readPrimeAgentCapacity( { agentHomePath: home }, - { sharedCacheDir: cacheDir, freshForMs: 0 }, + { + sharedCacheDir: cacheDir, + instanceId: "primeAgent", + configRevision: "revision-b", + freshForMs: 0, + }, ); const snapshot = yield* readPrimeAgentBackends( { agentHomePath: home }, - { sharedCacheDir: cacheDir, freshForMs: 0 }, + { + sharedCacheDir: cacheDir, + instanceId: "primeAgent", + configRevision: "revision-b", + freshForMs: 0, + }, ); assert.strictEqual(calls, 2); diff --git a/apps/server/src/provider/primeAgentBackends.ts b/apps/server/src/provider/primeAgentBackends.ts index 29a208496..3bfa809b9 100644 --- a/apps/server/src/provider/primeAgentBackends.ts +++ b/apps/server/src/provider/primeAgentBackends.ts @@ -107,13 +107,8 @@ export interface PrimeAgentBackendSignIn { readonly accessToken?: string | undefined; } -/** One-way identity for cache and retention keys; never sent or logged. */ -export function primeAgentCredentialFingerprint(credential: string): string { - return NodeCrypto.createHash("sha256").update(credential).digest("hex").slice(0, 24); -} - -const capacityRetentionIdentity = (backend: string, credentialIdentity: string) => - `${backend}:${primeAgentCredentialFingerprint(credentialIdentity)}`; +const capacityRetentionIdentity = (backend: string, configRevision: string) => + `${backend}:${configRevision}`; function freshAccessToken( entry: { @@ -285,7 +280,9 @@ function enforceFailedCapacityRetention( const readPrimeAgentCodexCapacity = Effect.fn("readPrimeAgentCodexCapacity")(function* (input: { readonly signIn: { readonly accessToken: string; readonly accountId: string }; - readonly homePath: string; + readonly cacheKey: string; + readonly configRevision: string; + readonly commitGuard: Effect.Effect | undefined; readonly nowMs: number; readonly freshForMs: number | undefined; readonly sharedCacheDir: string | undefined; @@ -295,9 +292,9 @@ const readPrimeAgentCodexCapacity = Effect.fn("readPrimeAgentCodexCapacity")(fun }) => Effect.Effect; }): Effect.fn.Return { const cacheDir = input.sharedCacheDir ?? (yield* resolveSharedUsageCacheDir); - const cacheKey = sharedUsageReadKey(["prime-codex", input.homePath, input.signIn.accountId]); + const cacheKey = input.cacheKey; const shared = decideSharedUsageRead( - yield* readSharedUsageEntry(cacheDir, cacheKey), + yield* readSharedUsageEntry(cacheDir, cacheKey, input.configRevision), input.nowMs, input.freshForMs, ); @@ -311,7 +308,7 @@ const readPrimeAgentCodexCapacity = Effect.fn("readPrimeAgentCodexCapacity")(fun }; } if (shared.kind === "throttled") { - const retained = yield* readSharedUsageEntry(cacheDir, cacheKey); + const retained = yield* readSharedUsageEntry(cacheDir, cacheKey, input.configRevision); return { ...(retained?.usageLimits ? { usageLimits: retained.usageLimits } : {}), didReadCapacity: false, @@ -319,7 +316,7 @@ const readPrimeAgentCodexCapacity = Effect.fn("readPrimeAgentCodexCapacity")(fun } const acquiredLock = yield* acquireSharedUsageLock(cacheDir, cacheKey, input.nowMs); if (!acquiredLock) { - const retained = yield* readSharedUsageEntry(cacheDir, cacheKey); + const retained = yield* readSharedUsageEntry(cacheDir, cacheKey, input.configRevision); return { ...(retained?.usageLimits ? { usageLimits: retained.usageLimits } : {}), didReadCapacity: false, @@ -332,11 +329,19 @@ const readPrimeAgentCodexCapacity = Effect.fn("readPrimeAgentCodexCapacity")(fun ? usageLimitsFromCodexRateLimits(response, readAt, "primeAgentCodex") : undefined; if (usageLimits) { - yield* writeSharedUsageEntry(cacheDir, cacheKey, { version: 1, readAt, usageLimits }); + yield* writeSharedUsageEntry( + cacheDir, + cacheKey, + { version: 1, configRevision: input.configRevision, readAt, usageLimits }, + input.commitGuard, + ); return { usageLimits, didReadCapacity: true }; } - yield* markSharedUsageReadFailed(cacheDir, cacheKey, readAt); - const retained = yield* readSharedUsageEntry(cacheDir, cacheKey); + yield* markSharedUsageReadFailed(cacheDir, cacheKey, readAt, { + configRevision: input.configRevision, + commitGuard: input.commitGuard, + }); + const retained = yield* readSharedUsageEntry(cacheDir, cacheKey, input.configRevision); return { ...(retained?.usageLimits ? { usageLimits: retained.usageLimits } : {}), didReadCapacity: false, @@ -346,6 +351,9 @@ const readPrimeAgentCodexCapacity = Effect.fn("readPrimeAgentCodexCapacity")(fun export interface ReadPrimeAgentBackendsOptions extends PrimeAgentHomeResolutionOptions { readonly sharedCacheDir?: string | undefined; + readonly instanceId?: string | undefined; + readonly configRevision?: string | undefined; + readonly commitGuard?: Effect.Effect | undefined; /** * How recent a shared reading must be to be served instead of read again. * The periodic probe leaves this at the shared window; the turn-end read @@ -390,6 +398,10 @@ export const readPrimeAgentCapacity = Effect.fn("readPrimeAgentCapacity")(functi if (Option.isNone(raw) || Option.isNone(decodePrimeAuthFile(raw.value))) return undefined; const nowMs = yield* Effect.clockWith((clock) => clock.currentTimeMillis); + // Missing revision is deliberately uncorrelated: legacy callers always start cold. + const configRevision = options?.configRevision ?? NodeCrypto.randomUUID(); + const cacheKeyForBackend = (backend: string) => + sharedUsageReadKey(["prime", options?.instanceId ?? "uncorrelated", backend, configRevision]); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const readWindows = options?.readCodexWindows ?? @@ -403,16 +415,17 @@ export const readPrimeAgentCapacity = Effect.fn("readPrimeAgentCapacity")(functi (signIn) => Effect.gen(function* () { if (signIn.backend === PRIME_AGENT_ANTHROPIC_BACKEND && signIn.accessToken) { - const credentialFingerprint = primeAgentCredentialFingerprint(signIn.accessToken); - const retentionIdentity = capacityRetentionIdentity(signIn.backend, signIn.accessToken); + const retentionIdentity = capacityRetentionIdentity(signIn.backend, configRevision); const read = yield* fetchOAuthUsageWithToken({ token: signIn.accessToken, - cacheKey: sharedUsageReadKey(["prime-anthropic", homePath, credentialFingerprint]), + cacheKey: cacheKeyForBackend(signIn.backend), checkedAt: DateTime.formatIso(DateTime.makeUnsafe(nowMs)), source: "primeAgentOAuth", sharedCacheDir: options?.sharedCacheDir, freshForMs: options?.freshForMs, shareFailures: true, + configRevision, + commitGuard: options?.commitGuard, }); return { backend: { @@ -430,7 +443,9 @@ export const readPrimeAgentCapacity = Effect.fn("readPrimeAgentCapacity")(functi ) { const read = yield* readPrimeAgentCodexCapacity({ signIn: { accessToken: signIn.accessToken, accountId: signIn.accountId }, - homePath, + cacheKey: cacheKeyForBackend(signIn.backend), + configRevision, + commitGuard: options?.commitGuard, nowMs, freshForMs: options?.freshForMs, sharedCacheDir: options?.sharedCacheDir, @@ -443,7 +458,7 @@ export const readPrimeAgentCapacity = Effect.fn("readPrimeAgentCapacity")(functi ...(read.usageLimits ? { usageLimits: read.usageLimits } : {}), }, didReadCapacity: read.didReadCapacity, - retentionIdentity: capacityRetentionIdentity(signIn.backend, signIn.accountId), + retentionIdentity: capacityRetentionIdentity(signIn.backend, configRevision), } satisfies ProviderBackendCapacityRead; } return { @@ -453,7 +468,7 @@ export const readPrimeAgentCapacity = Effect.fn("readPrimeAgentCapacity")(functi }, didReadCapacity: false, ...(signIn.accountId - ? { retentionIdentity: capacityRetentionIdentity(signIn.backend, signIn.accountId) } + ? { retentionIdentity: capacityRetentionIdentity(signIn.backend, configRevision) } : {}), } satisfies ProviderBackendCapacityRead; }), diff --git a/apps/server/src/provider/providerInstanceSettingsValidation.test.ts b/apps/server/src/provider/providerInstanceSettingsValidation.test.ts new file mode 100644 index 000000000..0c30c7533 --- /dev/null +++ b/apps/server/src/provider/providerInstanceSettingsValidation.test.ts @@ -0,0 +1,181 @@ +import { NodeServices } from "@effect/platform-node"; +import { + ProviderDriverKind, + ServerSettings, + type ProviderInstanceConfig, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import { describe, expect, it } from "@effect/vitest"; + +import { + getProviderMultipleInstanceSupport, + PRIME_AGENT_SUPPORTED_INSTANCE_LIMIT, + validateProviderInstanceSettings, +} from "./providerInstanceSettingsValidation.ts"; + +const decodeSettings = Schema.decodeUnknownSync(ServerSettings); +const prime = ProviderDriverKind.make("primeAgent"); +const instance = (home: string): ProviderInstanceConfig => ({ + driver: prime, + enabled: true, + config: { agentHomePath: home }, +}); + +function settingsWithInstances(providerInstances: Record) { + return decodeSettings({ providerInstances }); +} + +describe("provider multiple-instance settings validation", () => { + it("is explicit for built-ins, fail-closed for unknown drivers, and zero-work on native Windows", () => { + expect(getProviderMultipleInstanceSupport("codex", "darwin")).toEqual({ supported: true }); + expect(getProviderMultipleInstanceSupport("primeAgent", "linux")).toMatchObject({ + supported: false, + reason: expect.stringMatching(/N=1\/2\/4/u), + }); + expect(getProviderMultipleInstanceSupport("primeAgent", "win32")).toMatchObject({ + supported: false, + reason: expect.stringContaining("WSL2"), + }); + expect(getProviderMultipleInstanceSupport("forkDriver", "darwin")).toMatchObject({ + supported: false, + reason: expect.stringContaining("has not proved"), + }); + }); + + it.effect("keeps four independent Prime homes gated until graduation proof", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "pylon-prime-multi-settings-" }); + const providerInstances: Record = {}; + for (let index = 0; index < PRIME_AGENT_SUPPORTED_INSTANCE_LIMIT; index += 1) { + const home = path.join(root, `account-${index}`); + yield* fs.makeDirectory(home); + providerInstances[index === 0 ? "primeAgent" : `prime_${index}`] = + index === 1 + ? { + driver: prime, + enabled: true, + environment: [ + { name: "PRIME_AGENT_CODING_AGENT_DIR", value: home, sensitive: false }, + ], + config: {}, + } + : instance(home); + } + + const error = yield* validateProviderInstanceSettings({ + settings: settingsWithInstances(providerInstances), + platform: "darwin", + hostEnvironment: { HOME: root }, + }).pipe(Effect.flip); + expect(error.detail).toMatch(/N=1\/2\/4/u); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("rejects an ACP-only participant before persistence", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "pylon-prime-acp-settings-" }); + const first = path.join(root, "first"); + const second = path.join(root, "second"); + yield* fs.makeDirectory(first); + yield* fs.makeDirectory(second); + const error = yield* validateProviderInstanceSettings({ + settings: settingsWithInstances({ + primeAgent: instance(first), + prime_work: { + driver: prime, + enabled: true, + config: { agentHomePath: second, launchArgs: "--verbose" }, + }, + }), + platform: "darwin", + hostEnvironment: { HOME: root }, + }).pipe(Effect.flip); + + expect(error.detail).toContain("explicitly ACP-only"); + expect(error.detail).toContain("native-only"); + expect(error.detail).not.toContain(root); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("rejects equal, nested, and symlink-aliased Prime homes before persistence", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "pylon-prime-overlap-settings-" }); + const account = path.join(root, "account"); + const alias = path.join(root, "alias"); + yield* fs.makeDirectory(account); + yield* fs.symlink(account, alias); + + for (const other of [account, path.join(account, "nested"), alias]) { + const error = yield* validateProviderInstanceSettings({ + settings: settingsWithInstances({ + primeAgent: instance(account), + prime_work: instance(other), + }), + platform: "darwin", + hostEnvironment: { HOME: root }, + }).pipe(Effect.flip); + expect(error.detail).toMatch(/equal|nested|symlink/u); + expect(error.detail).toContain("separate Prime sign-in"); + } + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("rejects a fifth Prime instance and a second enabled unproved driver", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "pylon-prime-limit-settings-" }); + const tooMany: Record = {}; + for (let index = 0; index <= PRIME_AGENT_SUPPORTED_INSTANCE_LIMIT; index += 1) { + const home = path.join(root, `account-${index}`); + yield* fs.makeDirectory(home); + tooMany[index === 0 ? "primeAgent" : `prime_${index}`] = instance(home); + } + const limit = yield* validateProviderInstanceSettings({ + settings: settingsWithInstances(tooMany), + platform: "darwin", + hostEnvironment: { HOME: root }, + }).pipe(Effect.flip); + expect(limit.detail).toContain(`at most ${PRIME_AGENT_SUPPORTED_INSTANCE_LIMIT}`); + + const fork = ProviderDriverKind.make("forkDriver"); + const unsupported = yield* validateProviderInstanceSettings({ + settings: settingsWithInstances({ + fork_one: { driver: fork, enabled: true }, + fork_two: { driver: fork, enabled: true }, + }), + platform: "darwin", + hostEnvironment: { HOME: root }, + }).pipe(Effect.flip); + expect(unsupported.detail).toContain("has not proved"); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("keeps two ordinary Codex instances unchanged", () => + validateProviderInstanceSettings({ + settings: settingsWithInstances({ + codex: { driver: ProviderDriverKind.make("codex"), enabled: true }, + codex_work: { driver: ProviderDriverKind.make("codex"), enabled: true }, + }), + platform: "darwin", + hostEnvironment: { HOME: "/tmp" }, + }).pipe(Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/server/src/provider/providerInstanceSettingsValidation.ts b/apps/server/src/provider/providerInstanceSettingsValidation.ts new file mode 100644 index 000000000..717a172d8 --- /dev/null +++ b/apps/server/src/provider/providerInstanceSettingsValidation.ts @@ -0,0 +1,243 @@ +import { + PrimeAgentSettings, + ProviderDriverKind, + ProviderInstanceId, + resolveProviderInstanceEnabled, + type ProviderInstanceConfig, + type ProviderInstanceConfigMap, + type ServerSettings, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Result from "effect/Result"; +import * as Schema from "effect/Schema"; + +const PRIME_DRIVER = ProviderDriverKind.make("primeAgent"); +export const PRIME_AGENT_SUPPORTED_INSTANCE_LIMIT = 4; +export const PRIME_AGENT_DISTINCT_HOME_GUIDANCE = + "Each enabled Prime Agent instance needs a distinct, non-nested Agent home and a separate Prime sign-in. The home owns that instance's credentials, settings, models, sessions, sockets, checkpoints, and MCP state."; +export const PRIME_AGENT_GLOBAL_MAINTENANCE_GUIDANCE = + "Pylon manages only its instance-owned Prime processes. OS-user-global Prime update, doctor, shutdown, and stop-all maintenance stays external and is never run for an instance."; +export const NATIVE_WINDOWS_MULTIPLE_INSTANCES_REASON = + "Multiple Prime Agent instances are unavailable on native Windows. Run the Pylon server and Prime Agent inside WSL2, which uses the supported Linux runtime."; +export const PRIME_AGENT_ACP_ONLY_SETTINGS_REASON = + "Multiple Prime Agent instances are native-only, but custom Prime launch arguments require ACP compatibility. Remove the launch arguments or reduce the enabled Prime set to one."; +export const PRIME_AGENT_MULTIPLE_INSTANCES_GRADUATION_REASON = + "Multiple Prime Agent instances remain disabled until the signed-in N=1/2/4 macOS proof and enforced Linux/WSL2 hosted contract prove separate homes, credentials, catalogs, capacity, MCP bearer calls, canonical checkpoints, and conservative resource limits."; + +const MULTIPLE_INSTANCE_SUPPORT = new Map([ + ["codex", true], + ["claudeAgent", true], + ["cursor", true], + ["grok", true], + ["opencode", true], +]); + +export interface ProviderMultipleInstanceSupport { + readonly supported: boolean; + readonly reason?: string | undefined; +} + +export function getProviderMultipleInstanceSupport( + driver: string, + platform: NodeJS.Platform, +): ProviderMultipleInstanceSupport { + if (driver === PRIME_DRIVER) { + return { + supported: false, + reason: + platform === "win32" + ? NATIVE_WINDOWS_MULTIPLE_INSTANCES_REASON + : PRIME_AGENT_MULTIPLE_INSTANCES_GRADUATION_REASON, + }; + } + if (MULTIPLE_INSTANCE_SUPPORT.get(driver) === true) return { supported: true }; + return { + supported: false, + reason: `Driver '${driver}' has not proved support for multiple enabled instances.`, + }; +} + +function effectiveProviderInstances(settings: ServerSettings): ProviderInstanceConfigMap { + const merged: Record = { ...settings.providerInstances }; + for (const [driver, config] of Object.entries(settings.providers)) { + const id = ProviderInstanceId.make(driver); + if (id in merged) continue; + merged[id] = { driver: ProviderDriverKind.make(driver), config }; + } + return merged as ProviderInstanceConfigMap; +} + +function environmentValue( + environment: Readonly>, + name: string, +): string | undefined { + const normalized = name.toUpperCase(); + let value: string | undefined; + for (const [candidate, candidateValue] of Object.entries(environment)) { + if (candidate.toUpperCase() === normalized) value = candidateValue; + } + return value; +} + +const canonicalizeHome = Effect.fn("canonicalizeProviderInstanceHome")(function* ( + candidate: string, +) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + let ancestor = path.normalize(candidate); + const suffix: string[] = []; + while (true) { + const resolved = yield* Effect.result(fileSystem.realPath(ancestor)); + if (Result.isSuccess(resolved)) { + const info = yield* Effect.result(fileSystem.stat(resolved.success)); + if (Result.isFailure(info) || info.success.type !== "Directory") return null; + return suffix.length === 0 ? resolved.success : path.join(resolved.success, ...suffix); + } + const failure = resolved.failure; + if (failure._tag !== "PlatformError" || failure.reason._tag !== "NotFound") return null; + const parent = path.dirname(ancestor); + if (parent === ancestor) return null; + suffix.unshift(path.basename(ancestor)); + ancestor = parent; + } +}); + +const resolvePrimeHome = Effect.fn("resolvePrimeProviderInstanceHome")(function* (input: { + readonly instance: ProviderInstanceConfig; + readonly config: PrimeAgentSettings; + readonly hostEnvironment: NodeJS.ProcessEnv; +}) { + const path = yield* Path.Path; + const merged: Record = { ...input.hostEnvironment }; + for (const variable of input.instance.environment ?? []) merged[variable.name] = variable.value; + const effectiveOsHome = environmentValue(merged, "HOME")?.trim(); + const configured = input.config.agentHomePath.trim(); + const fromEnvironment = + environmentValue(merged, "PRIME_AGENT_CODING_AGENT_DIR")?.trim() || + environmentValue(merged, "PRIME_AGENT_HOME")?.trim(); + const candidate = + configured || + fromEnvironment || + (effectiveOsHome ? path.join(effectiveOsHome, ".prime", "agent") : ""); + const expanded = + candidate === "~" + ? effectiveOsHome + : candidate.startsWith("~/") && effectiveOsHome + ? path.join(effectiveOsHome, candidate.slice(2)) + : candidate; + if (!expanded || !path.isAbsolute(expanded)) return null; + return yield* canonicalizeHome(expanded); +}); + +function homesOverlap( + path: Path.Path, + left: string, + right: string, + caseInsensitive: boolean, +): boolean { + const a = caseInsensitive ? left.toLowerCase() : left; + const b = caseInsensitive ? right.toLowerCase() : right; + const relative = path.relative(a, b); + if (relative === "") return true; + if (!path.isAbsolute(relative) && relative !== ".." && !relative.startsWith(`..${path.sep}`)) { + return true; + } + const inverse = path.relative(b, a); + return !path.isAbsolute(inverse) && inverse !== ".." && !inverse.startsWith(`..${path.sep}`); +} + +export class ProviderInstanceSettingsValidationError extends Schema.TaggedErrorClass()( + "ProviderInstanceSettingsValidationError", + { detail: Schema.String }, +) { + override get message(): string { + return this.detail; + } +} + +/** Validate the complete next host configuration before it is persisted. */ +export const validateProviderInstanceSettings = Effect.fn("validateProviderInstanceSettings")( + function* (input: { + readonly settings: ServerSettings; + readonly platform: NodeJS.Platform; + readonly hostEnvironment: NodeJS.ProcessEnv; + }) { + const instances = effectiveProviderInstances(input.settings); + const enabledByDriver = new Map>(); + for (const [instanceId, instance] of Object.entries(instances)) { + if (!resolveProviderInstanceEnabled(instance)) continue; + const entries = enabledByDriver.get(instance.driver) ?? []; + entries.push([instanceId, instance]); + enabledByDriver.set(instance.driver, entries); + } + + const primeEntries = enabledByDriver.get(PRIME_DRIVER) ?? []; + if (primeEntries.length > PRIME_AGENT_SUPPORTED_INSTANCE_LIMIT) { + return yield* new ProviderInstanceSettingsValidationError({ + detail: `Prime Agent supports at most ${PRIME_AGENT_SUPPORTED_INSTANCE_LIMIT} enabled instances on one Pylon server.`, + }); + } + if (primeEntries.length >= 2) { + const decodePrime = Schema.decodeUnknownEffect(PrimeAgentSettings); + const homes: Array = []; + for (const [instanceId, instance] of primeEntries) { + const decoded = yield* decodePrime(instance.config ?? {}).pipe(Effect.result); + if (Result.isFailure(decoded)) { + return yield* new ProviderInstanceSettingsValidationError({ + detail: `Prime Agent instance '${instanceId}' has invalid settings.`, + }); + } + if (decoded.success.launchArgs.trim().length > 0) { + return yield* new ProviderInstanceSettingsValidationError({ + detail: `Prime Agent instance '${instanceId}' is explicitly ACP-only. ${PRIME_AGENT_ACP_ONLY_SETTINGS_REASON}`, + }); + } + const home = yield* resolvePrimeHome({ + instance, + config: decoded.success, + hostEnvironment: input.hostEnvironment, + }); + if (home === null) { + return yield* new ProviderInstanceSettingsValidationError({ + detail: `Prime Agent instance '${instanceId}' needs a safe absolute Agent home. ${PRIME_AGENT_DISTINCT_HOME_GUIDANCE}`, + }); + } + homes.push([instanceId, home]); + } + + const path = yield* Path.Path; + for (let left = 0; left < homes.length; left += 1) { + for (let right = left + 1; right < homes.length; right += 1) { + const a = homes[left]; + const b = homes[right]; + if ( + a && + b && + homesOverlap( + path, + a[1], + b[1], + input.platform === "darwin" || input.platform === "win32", + ) + ) { + return yield* new ProviderInstanceSettingsValidationError({ + detail: `Prime Agent instances '${a[0]}' and '${b[0]}' have equal, nested, or symlink-aliased homes. ${PRIME_AGENT_DISTINCT_HOME_GUIDANCE}`, + }); + } + } + } + } + + for (const [driver, entries] of enabledByDriver) { + if (entries.length < 2) continue; + const support = getProviderMultipleInstanceSupport(driver, input.platform); + if (!support.supported) { + return yield* new ProviderInstanceSettingsValidationError({ + detail: support.reason ?? `Driver '${driver}' supports only one enabled instance.`, + }); + } + } + }, +); diff --git a/apps/server/src/provider/providerMaintenanceRunner.ts b/apps/server/src/provider/providerMaintenanceRunner.ts index 33d9647ce..be039c803 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.ts @@ -297,6 +297,9 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { instanceId, provider, ); + const runtimeFence = yield* ( + providerRegistry.getProviderRuntimeFence?.(instanceId) ?? Effect.succeed(undefined) + ); const update = capabilities.update; if (!update) { return yield* new ServerProviderUpdateError({ @@ -310,6 +313,7 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { instanceId, action: "update", state, + runtimeFence, }); const setQueuedState = setUpdateState( makeUpdateState({ @@ -341,6 +345,17 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { const result = yield* runMaintenanceCommand(update.executable, update.args); const finishedAt = yield* nowIso; + if (runtimeFence !== undefined && !(yield* runtimeFence.isCurrent)) { + return yield* finish( + makeUpdateState({ + status: "unchanged", + startedAt, + finishedAt, + message: "The provider runtime changed while the update was running.", + output: null, + }), + ); + } if (result.timedOut || result.exitCode !== 0) { return yield* finish( makeUpdateState({ diff --git a/apps/server/src/provider/providerRuntimeFenceMetadata.ts b/apps/server/src/provider/providerRuntimeFenceMetadata.ts new file mode 100644 index 000000000..2dde4fc21 --- /dev/null +++ b/apps/server/src/provider/providerRuntimeFenceMetadata.ts @@ -0,0 +1,19 @@ +import type { ProviderRuntimeEvent } from "@t3tools/contracts"; + +import type { ProviderRuntimeFence } from "./ProviderDriver.ts"; + +const runtimeEventFences = new WeakMap(); + +export function attachProviderRuntimeEventFence( + event: ProviderRuntimeEvent, + runtimeFence: ProviderRuntimeFence | undefined, +): ProviderRuntimeEvent { + if (runtimeFence !== undefined) runtimeEventFences.set(event, runtimeFence); + return event; +} + +export function readProviderRuntimeEventFence( + event: ProviderRuntimeEvent, +): ProviderRuntimeFence | undefined { + return runtimeEventFences.get(event); +} diff --git a/apps/server/src/provider/providerStatusCache.test.ts b/apps/server/src/provider/providerStatusCache.test.ts index f9bb7ac91..232c9e5e9 100644 --- a/apps/server/src/provider/providerStatusCache.test.ts +++ b/apps/server/src/provider/providerStatusCache.test.ts @@ -10,6 +10,7 @@ import { assert, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Logger from "effect/Logger"; +import * as Ref from "effect/Ref"; import { hydrateCachedProvider, @@ -121,6 +122,42 @@ it.layer(NodeServices.layer)("providerStatusCache", (it) => { }), ); + it.effect("treats legacy, mismatched, and stale guarded Prime revisions as cold", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const tempDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-provider-cache-fence-" }); + const cachePath = `${tempDir}/primeAgent.json`; + const current = yield* Ref.make(true); + const currentProvider = makeProvider(ProviderDriverKind.make("primeAgent"), { + version: "2.0.0", + }); + yield* writeProviderStatusCache({ + filePath: cachePath, + provider: currentProvider, + configRevision: "revision-b", + commitGuard: Ref.get(current), + }); + + yield* Ref.set(current, false); + yield* writeProviderStatusCache({ + filePath: cachePath, + provider: { ...currentProvider, version: "1.0.0" }, + configRevision: "revision-a", + commitGuard: Ref.get(current), + }); + + assert.deepStrictEqual( + yield* readProviderStatusCache(cachePath, { configRevision: "revision-b" }), + currentProvider, + ); + assert.strictEqual( + yield* readProviderStatusCache(cachePath, { configRevision: "revision-a" }), + undefined, + ); + assert.strictEqual(yield* readProviderStatusCache(cachePath), undefined); + }), + ); + it("hydrates cached provider status while preserving current settings-derived models", () => { const cachedCodex = makeProvider(CODEX_DRIVER, { checkedAt: "2026-04-10T12:00:00.000Z", diff --git a/apps/server/src/provider/providerStatusCache.ts b/apps/server/src/provider/providerStatusCache.ts index 2fe0424b4..ff5f2c084 100644 --- a/apps/server/src/provider/providerStatusCache.ts +++ b/apps/server/src/provider/providerStatusCache.ts @@ -5,6 +5,7 @@ import { ServerProvider as ServerProviderSchema, } from "@t3tools/contracts"; import { causeErrorTag } from "@t3tools/shared/observability"; +import type * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; @@ -12,9 +13,17 @@ import * as Schema from "effect/Schema"; import { writeFileStringAtomically } from "../atomicWrite.ts"; -const decodeProviderStatusCache = Schema.decodeUnknownEffect( +const PrivateProviderStatusCache = Schema.Struct({ + version: Schema.Literal(1), + configRevision: Schema.String, + provider: ServerProviderSchema, +}); +const decodeLegacyProviderStatusCache = Schema.decodeUnknownEffect( Schema.fromJsonString(ServerProviderSchema), ); +const decodePrivateProviderStatusCache = Schema.decodeUnknownEffect( + Schema.fromJsonString(PrivateProviderStatusCache), +); const mergeProviderModels = ( fallbackModels: ReadonlyArray, @@ -115,7 +124,10 @@ export const resolveLegacyProviderStatusCachePath = Effect.fn( return path.join(input.cacheDir, `${input.provider}.json`); }); -export const readProviderStatusCache = (filePath: string) => +export const readProviderStatusCache = ( + filePath: string, + options?: { readonly configRevision?: string | undefined }, +) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const exists = yield* fs.exists(filePath).pipe(Effect.orElseSucceed(() => false)); @@ -129,25 +141,43 @@ export const readProviderStatusCache = (filePath: string) => return undefined; } - return yield* decodeProviderStatusCache(trimmed).pipe( + const expectedRevision = options?.configRevision; + const onDecodeFailure = (cause: Cause.Cause) => + Effect.logWarning("failed to parse provider status cache, ignoring", { + path: filePath, + errorTag: causeErrorTag(cause), + }).pipe(Effect.as(undefined)); + if (expectedRevision === undefined) { + return yield* decodeLegacyProviderStatusCache(trimmed).pipe( + Effect.matchCauseEffect({ + onFailure: onDecodeFailure, + onSuccess: Effect.succeed, + }), + ); + } + const decoded = yield* decodePrivateProviderStatusCache(trimmed).pipe( Effect.matchCauseEffect({ - onFailure: (cause) => - Effect.logWarning("failed to parse provider status cache, ignoring", { - path: filePath, - errorTag: causeErrorTag(cause), - }).pipe(Effect.as(undefined)), + onFailure: onDecodeFailure, onSuccess: Effect.succeed, }), ); + return decoded?.configRevision === expectedRevision ? decoded.provider : undefined; }); export const writeProviderStatusCache = (input: { readonly filePath: string; readonly provider: ServerProvider; + readonly configRevision?: string | undefined; + readonly commitGuard?: Effect.Effect; }) => { const { updateState: _updateState, ...cacheableProvider } = input.provider; + const contents = + input.configRevision === undefined + ? cacheableProvider + : { version: 1 as const, configRevision: input.configRevision, provider: cacheableProvider }; return writeFileStringAtomically({ filePath: input.filePath, - contents: `${JSON.stringify(cacheableProvider, null, 2)}\n`, + contents: `${JSON.stringify(contents, null, 2)}\n`, + ...(input.commitGuard === undefined ? {} : { commitGuard: input.commitGuard }), }); }; diff --git a/apps/server/src/provider/sharedUsageReadCache.test.ts b/apps/server/src/provider/sharedUsageReadCache.test.ts index 626d064a5..7713ef0c7 100644 --- a/apps/server/src/provider/sharedUsageReadCache.test.ts +++ b/apps/server/src/provider/sharedUsageReadCache.test.ts @@ -5,6 +5,7 @@ import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; +import * as Ref from "effect/Ref"; import { acquireSharedUsageLock, @@ -219,6 +220,32 @@ it.layer(NodeServices.layer)("shared usage read cache", (it) => { }).pipe(Effect.scoped), ); + it.effect("cold-misses mismatched revisions and rejects a retired write", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const dir = yield* fs.makeTempDirectoryScoped({ prefix: "pylon-usage-fence-" }); + const key = sharedUsageReadKey(["prime", "primeAgent", "anthropic", "revision-b"]); + const current = yield* Ref.make(true); + const entry = { + version: 1 as const, + configRevision: "revision-b", + readAt: at(0), + usageLimits: usage(at(0)), + }; + yield* writeSharedUsageEntry(dir, key, entry, Ref.get(current)); + yield* Ref.set(current, false); + yield* writeSharedUsageEntry( + dir, + key, + { ...entry, configRevision: "revision-a", readAt: at(1_000) }, + Ref.get(current), + ); + + assert.deepStrictEqual(yield* readSharedUsageEntry(dir, key, "revision-b"), entry); + assert.strictEqual(yield* readSharedUsageEntry(dir, key, "revision-a"), undefined); + }).pipe(Effect.scoped), + ); + // Several servers expiring together must read the endpoint once. it.effect("hands the lock to one holder at a time and breaks a dead one", () => Effect.gen(function* () { diff --git a/apps/server/src/provider/sharedUsageReadCache.ts b/apps/server/src/provider/sharedUsageReadCache.ts index d77b69864..5bea6ea7e 100644 --- a/apps/server/src/provider/sharedUsageReadCache.ts +++ b/apps/server/src/provider/sharedUsageReadCache.ts @@ -61,6 +61,8 @@ export const SharedUsageReadEntry = Schema.Struct({ throttledUntil: Schema.optional(Schema.String), /** Last failed attempt. It suppresses duplicate failures for a short window. */ failedAt: Schema.optional(Schema.String), + /** Private random correlation used only by fenced provider materializations. */ + configRevision: Schema.optional(Schema.String), }); export type SharedUsageReadEntry = typeof SharedUsageReadEntry.Type; @@ -192,6 +194,7 @@ const lockPath = (path: Path.Path, dir: string, key: string) => path.join(dir, ` export const readSharedUsageEntry = Effect.fn("readSharedUsageEntry")(function* ( dir: string, key: string, + expectedConfigRevision?: string, ): Effect.fn.Return { const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -200,7 +203,9 @@ export const readSharedUsageEntry = Effect.fn("readSharedUsageEntry")(function* Effect.catchCause(() => Effect.succeed(Option.none())), ); if (Option.isNone(raw)) return undefined; - return Option.getOrUndefined(decodeEntry(raw.value)); + const decoded = Option.getOrUndefined(decodeEntry(raw.value)); + if (expectedConfigRevision === undefined) return decoded; + return decoded?.configRevision === expectedConfigRevision ? decoded : undefined; }); /** @@ -212,6 +217,7 @@ export const writeSharedUsageEntry = Effect.fn("writeSharedUsageEntry")(function dir: string, key: string, entry: SharedUsageReadEntry, + commitGuard?: Effect.Effect, ): Effect.fn.Return { const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -220,6 +226,10 @@ export const writeSharedUsageEntry = Effect.fn("writeSharedUsageEntry")(function yield* Effect.gen(function* () { yield* fileSystem.makeDirectory(dir, { recursive: true }); yield* fileSystem.writeFileString(temp, encodeEntry(entry)); + if (commitGuard !== undefined && !(yield* commitGuard)) { + yield* fileSystem.remove(temp).pipe(Effect.ignore); + return; + } yield* fileSystem.rename(temp, target); }).pipe(Effect.catchCause(() => Effect.void)); }); @@ -232,15 +242,25 @@ export const markSharedUsageReadFailed = Effect.fn("markSharedUsageReadFailed")( dir: string, key: string, failedAt: string, + options?: { + readonly configRevision?: string | undefined; + readonly commitGuard?: Effect.Effect | undefined; + }, ): Effect.fn.Return { - const existing = yield* readSharedUsageEntry(dir, key); - yield* writeSharedUsageEntry(dir, key, { - version: 1, - readAt: existing?.readAt ?? failedAt, - ...(existing?.usageLimits ? { usageLimits: existing.usageLimits } : {}), - ...(existing?.throttledUntil ? { throttledUntil: existing.throttledUntil } : {}), - failedAt, - }); + const existing = yield* readSharedUsageEntry(dir, key, options?.configRevision); + yield* writeSharedUsageEntry( + dir, + key, + { + version: 1, + readAt: existing?.readAt ?? failedAt, + ...(options?.configRevision === undefined ? {} : { configRevision: options.configRevision }), + ...(existing?.usageLimits ? { usageLimits: existing.usageLimits } : {}), + ...(existing?.throttledUntil ? { throttledUntil: existing.throttledUntil } : {}), + failedAt, + }, + options?.commitGuard, + ); }); /** diff --git a/apps/server/src/provider/unavailableProviderSnapshot.ts b/apps/server/src/provider/unavailableProviderSnapshot.ts index 49e7b3929..93f4f83c4 100644 --- a/apps/server/src/provider/unavailableProviderSnapshot.ts +++ b/apps/server/src/provider/unavailableProviderSnapshot.ts @@ -74,6 +74,8 @@ export function buildUnavailableProviderSnapshot( : input.driverKind, availability: "unavailable", unavailableReason: input.reason, + supportsMultipleInstances: false, + multipleInstancesUnavailableReason: input.reason, }; }); } diff --git a/apps/server/src/serverSettings.providerInstances.test.ts b/apps/server/src/serverSettings.providerInstances.test.ts new file mode 100644 index 000000000..51d09680b --- /dev/null +++ b/apps/server/src/serverSettings.providerInstances.test.ts @@ -0,0 +1,164 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import { + ProviderDriverKind, + ProviderInstanceId, + ServerProviderInstancesMutationId, + type ProviderInstanceConfig, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import { describe, expect, it } from "@effect/vitest"; + +import { ServerSettingsService } from "./serverSettings.ts"; + +const codex = ProviderDriverKind.make("codex"); +const aId = ProviderInstanceId.make("codex_a"); +const bId = ProviderInstanceId.make("codex_b"); +const a: ProviderInstanceConfig = { driver: codex, enabled: true, displayName: "A" }; +const b: ProviderInstanceConfig = { driver: codex, enabled: true, displayName: "B" }; +const mutationId = (value: string) => ServerProviderInstancesMutationId.make(value); +const prime = ProviderDriverKind.make("primeAgent"); +const primeInstance = (home: string, launchArgs = ""): ProviderInstanceConfig => ({ + driver: prime, + enabled: true, + config: { agentHomePath: home, ...(launchArgs === "" ? {} : { launchArgs }) }, +}); + +describe("provider instance host CAS receipts", () => { + it.effect( + "is idempotent, rejects mutation-id reuse, and preserves unrelated instances on removal", + () => + Effect.gen(function* () { + const settings = yield* ServerSettingsService; + const create = { + mutationId: mutationId("create-a"), + expectedProviderInstances: {}, + patch: { providerInstances: { [aId]: a, [bId]: b } }, + } as const; + const applied = yield* settings.mutateProviderInstances(create); + const repeated = yield* settings.mutateProviderInstances(create); + expect(applied.disposition).toBe("applied"); + expect(repeated.disposition).toBe("already-applied"); + + const reused = yield* settings + .mutateProviderInstances({ + ...create, + patch: { providerInstances: { [aId]: { ...a, displayName: "Other" } } }, + }) + .pipe(Effect.flip); + expect(reused).toMatchObject({ + _tag: "ServerProviderInstancesMutationConflictError", + reason: "mutation-reused", + }); + + const removed = yield* settings.mutateProviderInstances({ + mutationId: mutationId("remove-a"), + expectedProviderInstances: { [aId]: a, [bId]: b }, + patch: { providerInstances: { [bId]: b } }, + }); + expect(removed.settings.providerInstances[aId]).toBeUndefined(); + expect(removed.settings.providerInstances[bId]).toEqual(b); + }).pipe(Effect.provide(ServerSettingsService.layerTest())), + ); + + it.effect("serializes two remote clients against one host snapshot", () => + Effect.gen(function* () { + const settings = yield* ServerSettingsService; + const exits = yield* Effect.all( + [ + settings.mutateProviderInstances({ + mutationId: mutationId("client-a"), + expectedProviderInstances: {}, + patch: { providerInstances: { [aId]: a } }, + }), + settings.mutateProviderInstances({ + mutationId: mutationId("client-b"), + expectedProviderInstances: {}, + patch: { providerInstances: { [bId]: b } }, + }), + ].map(Effect.exit), + { concurrency: "unbounded" }, + ); + expect(exits.filter(Exit.isSuccess)).toHaveLength(1); + expect(exits.filter(Exit.isFailure)).toHaveLength(1); + const current = yield* settings.getSettings; + expect(Object.keys(current.providerInstances)).toHaveLength(1); + }).pipe(Effect.provide(ServerSettingsService.layerTest())), + ); + + it.effect("rejects invalid native-only Prime sets without mutating the host CAS snapshot", () => + Effect.acquireUseRelease( + Effect.sync(() => NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "prime-host-cas-"))), + (root) => + Effect.gen(function* () { + const settings = yield* ServerSettingsService; + const home = (name: string) => { + const value = NodePath.join(root, name); + NodeFS.mkdirSync(value, { recursive: true }); + return value; + }; + const distinct = { + [ProviderInstanceId.make("prime_a")]: primeInstance(home("a")), + [ProviderInstanceId.make("prime_b")]: primeInstance(home("b")), + }; + const invalidSets = [ + { + ...distinct, + [ProviderInstanceId.make("prime_b")]: primeInstance(home("b"), "--acp-only"), + }, + { + [ProviderInstanceId.make("prime_a")]: primeInstance(home("shared")), + [ProviderInstanceId.make("prime_b")]: primeInstance(home("shared/nested")), + }, + Object.fromEntries( + Array.from({ length: 5 }, (_, index) => [ + ProviderInstanceId.make(index === 0 ? "primeAgent" : `prime_${index}`), + primeInstance(home(`limit-${index}`)), + ]), + ), + ]; + + for (const [index, providerInstances] of invalidSets.entries()) { + const rejected = yield* settings + .mutateProviderInstances({ + mutationId: mutationId(`invalid-prime-${index}`), + expectedProviderInstances: {}, + patch: { providerInstances }, + }) + .pipe(Effect.flip); + expect(rejected).toMatchObject({ _tag: "ServerSettingsError", operation: "normalize" }); + expect((yield* settings.getSettings).providerInstances).toEqual({}); + } + + const gated = yield* settings + .mutateProviderInstances({ + mutationId: mutationId("graduation-gated-prime-native-set"), + expectedProviderInstances: {}, + patch: { providerInstances: distinct }, + }) + .pipe(Effect.flip); + expect(gated).toMatchObject({ _tag: "ServerSettingsError", operation: "normalize" }); + expect(gated.message).toMatch(/N=1\/2\/4/u); + expect((yield* settings.getSettings).providerInstances).toEqual({}); + }), + (root) => Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), + ).pipe(Effect.provide(ServerSettingsService.layerTest())), + ); + + it.effect("recognizes an already-applied mutation after a server service restart", () => + Effect.gen(function* () { + const settings = yield* ServerSettingsService; + const receipt = yield* settings.mutateProviderInstances({ + mutationId: mutationId("retry-after-restart"), + expectedProviderInstances: {}, + patch: { providerInstances: { [aId]: a } }, + }); + expect(receipt.disposition).toBe("already-applied"); + expect(receipt.settings.providerInstances[aId]).toEqual(a); + }).pipe(Effect.provide(ServerSettingsService.layerTest({ providerInstances: { [aId]: a } }))), + ); +}); diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 2d89cb988..683e77341 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -4,6 +4,7 @@ import { ProviderDriverKind, ProviderInstanceId, resolveProviderInstanceEnabled, + ServerProviderInstancesMutationId, ServerSettings, ServerSettingsPatch, } from "@t3tools/contracts"; @@ -26,6 +27,26 @@ import * as ServerSettingsModule from "./serverSettings.ts"; const decodeSettingsPatch = Schema.decodeUnknownEffect(ServerSettingsPatch); const decodeServerSettings = Schema.decodeUnknownEffect(ServerSettings); +let providerMutationSequence = 0; +const updateSettingsWithProviderInstances = Effect.fn("updateSettingsWithProviderInstances")( + function* ( + serverSettings: ServerSettingsModule.ServerSettingsService["Service"], + patch: ServerSettingsPatch, + ) { + const current = ServerSettingsModule.redactServerSettingsForClient( + yield* serverSettings.getSettings, + ); + const receipt = yield* serverSettings.mutateProviderInstances({ + mutationId: ServerProviderInstancesMutationId.make( + `server-settings-test-${providerMutationSequence++}`, + ), + expectedProviderInstances: current.providerInstances, + patch, + }); + return receipt.settings; + }, +); + const makeServerSettingsLayer = () => ServerSettingsModule.layer.pipe( Layer.provide(ServerSecretStore.layer), @@ -314,7 +335,7 @@ it.layer(NodeServices.layer)("server settings", (it) => { Effect.gen(function* () { const serverSettings = yield* ServerSettingsModule.ServerSettingsService; - const next = yield* serverSettings.updateSettings({ + const next = yield* updateSettingsWithProviderInstances(serverSettings, { providerInstances: { [ProviderInstanceId.make("claude_openrouter")]: { driver: ProviderDriverKind.make("claudeAgent"), @@ -342,7 +363,7 @@ it.layer(NodeServices.layer)("server settings", (it) => { const serverSettings = yield* ServerSettingsModule.ServerSettingsService; const instanceId = ProviderInstanceId.make("claude_openrouter"); - const next = yield* serverSettings.updateSettings({ + const next = yield* updateSettingsWithProviderInstances(serverSettings, { providers: { claudeAgent: { enabled: false, @@ -373,7 +394,7 @@ it.layer(NodeServices.layer)("server settings", (it) => { const serverSettings = yield* ServerSettingsModule.ServerSettingsService; const instanceId = ProviderInstanceId.make("openrouter_text"); - const next = yield* serverSettings.updateSettings({ + const next = yield* updateSettingsWithProviderInstances(serverSettings, { providerInstances: { [instanceId]: { driver: ProviderDriverKind.make("openrouter"), @@ -407,7 +428,7 @@ it.layer(NodeServices.layer)("server settings", (it) => { model: "gpt-5.4-mini", }; - yield* serverSettings.updateSettings({ + yield* updateSettingsWithProviderInstances(serverSettings, { providerInstances: { [instanceId]: { driver: ProviderDriverKind.make("codex"), @@ -418,7 +439,7 @@ it.layer(NodeServices.layer)("server settings", (it) => { sourceControlWriterModelSelection, }); - const next = yield* serverSettings.updateSettings({ + const next = yield* updateSettingsWithProviderInstances(serverSettings, { providerInstances: { [instanceId]: { driver: ProviderDriverKind.make("codex"), @@ -445,7 +466,7 @@ it.layer(NodeServices.layer)("server settings", (it) => { sourceControlWriterModelSelection, ); - const restored = yield* serverSettings.updateSettings({ + const restored = yield* updateSettingsWithProviderInstances(serverSettings, { providerInstances: { [instanceId]: { driver: ProviderDriverKind.make("codex"), @@ -499,7 +520,7 @@ it.layer(NodeServices.layer)("server settings", (it) => { const serverSettings = yield* ServerSettingsModule.ServerSettingsService; const codexId = ProviderInstanceId.make("codex"); - yield* serverSettings.updateSettings({ + yield* updateSettingsWithProviderInstances(serverSettings, { providerInstances: { [codexId]: { driver: ProviderDriverKind.make("codex"), @@ -511,7 +532,7 @@ it.layer(NodeServices.layer)("server settings", (it) => { }, }); - const next = yield* serverSettings.updateSettings({ + const next = yield* updateSettingsWithProviderInstances(serverSettings, { providerInstances: { [codexId]: { driver: ProviderDriverKind.make("codex"), @@ -749,7 +770,7 @@ it.layer(NodeServices.layer)("server settings", (it) => { assert.isFalse(initial.providers.opencode.enabled); assert.isFalse(initial.providers.cursor.enabled); - const next = yield* serverSettings.updateSettings({ + const next = yield* updateSettingsWithProviderInstances(serverSettings, { addProjectBaseDirectory: "~/Development", providerInstances: { [ProviderInstanceId.make("grok")]: { @@ -817,7 +838,7 @@ it.layer(NodeServices.layer)("server settings", (it) => { const serverSettings = yield* ServerSettingsModule.ServerSettingsService; const grokId = ProviderInstanceId.make("grok"); - const next = yield* serverSettings.updateSettings({ + const next = yield* updateSettingsWithProviderInstances(serverSettings, { providerInstances: { [grokId]: { driver: ProviderDriverKind.make("grok"), @@ -1008,7 +1029,7 @@ it.layer(NodeServices.layer)("server settings", (it) => { const fileSystem = yield* FileSystem.FileSystem; const instanceId = ProviderInstanceId.make("codex_personal"); - const next = yield* serverSettings.updateSettings({ + const next = yield* updateSettingsWithProviderInstances(serverSettings, { providerInstances: { [instanceId]: { driver: ProviderDriverKind.make("codex"), @@ -1044,7 +1065,7 @@ it.layer(NodeServices.layer)("server settings", (it) => { { name: "ANTHROPIC_BASE_URL", value: "https://openrouter.ai/api", sensitive: false }, ]); - const roundTripped = yield* serverSettings.updateSettings({ + const roundTripped = yield* updateSettingsWithProviderInstances(serverSettings, { providerInstances: { [instanceId]: { driver: ProviderDriverKind.make("codex"), @@ -1062,6 +1083,65 @@ it.layer(NodeServices.layer)("server settings", (it) => { roundTripped.providerInstances[instanceId]?.environment?.[0]?.value, "sk-or-secret", ); + + const expectedProviderInstances = + ServerSettingsModule.redactServerSettingsForClient(roundTripped).providerInstances; + const mutated = yield* serverSettings.mutateProviderInstances({ + mutationId: ServerProviderInstancesMutationId.make("sensitive-provider-cas"), + expectedProviderInstances, + patch: { + providerInstances: { + ...expectedProviderInstances, + [instanceId]: { + ...expectedProviderInstances[instanceId]!, + displayName: "Codex Personal CAS", + }, + }, + }, + }); + assert.equal(mutated.disposition, "applied"); + assert.equal( + mutated.settings.providerInstances[instanceId]?.displayName, + "Codex Personal CAS", + ); + assert.equal( + mutated.settings.providerInstances[instanceId]?.environment?.[0]?.value, + "sk-or-secret", + ); + + const currentProviderInstances = ServerSettingsModule.redactServerSettingsForClient( + mutated.settings, + ).providerInstances; + for (const providerInstances of [{}, currentProviderInstances]) { + const rejected = yield* serverSettings + .updateSettings({ providerInstances }) + .pipe(Effect.flip); + assert.deepInclude(rejected, { + _tag: "ServerSettingsUpdateConflictError", + reason: "provider-instances-upgrade-required", + }); + } + + const stale = yield* serverSettings + .mutateProviderInstances({ + mutationId: ServerProviderInstancesMutationId.make("stale-secret-deletion"), + expectedProviderInstances: {}, + patch: { providerInstances: {} }, + }) + .pipe(Effect.flip); + assert.deepInclude(stale, { + _tag: "ServerProviderInstancesMutationConflictError", + reason: "stale", + }); + + const compatible = yield* serverSettings.updateSettings({ + addProjectBaseDirectory: "~/LegacyCompatible", + }); + assert.equal(compatible.addProjectBaseDirectory, "~/LegacyCompatible"); + assert.equal( + compatible.providerInstances[instanceId]?.environment?.[0]?.value, + "sk-or-secret", + ); }).pipe(Effect.provide(makeServerSettingsLayer())), ); it.effect( @@ -1070,13 +1150,19 @@ it.layer(NodeServices.layer)("server settings", (it) => { Effect.gen(function* () { const serverSettings = yield* ServerSettingsModule.ServerSettingsService; const instanceId = ProviderInstanceId.make("prime_work"); - yield* serverSettings.updateSettings({ - providerInstances: { - [instanceId]: { - driver: ProviderDriverKind.make("primeAgent"), - displayName: "Prime Work", - environment: [{ name: "PRIME_PROFILE", value: "work", sensitive: false }], - config: { binaryPath: "/opt/prime/stock", homePath: "/tmp/prime-work" }, + const initial = yield* serverSettings.getSettings; + yield* serverSettings.mutateProviderInstances({ + mutationId: ServerProviderInstancesMutationId.make("test-prime-managed-binding-create"), + expectedProviderInstances: initial.providerInstances, + patch: { + providerInstances: { + [instanceId]: { + driver: ProviderDriverKind.make("primeAgent"), + displayName: "Prime Work", + enabled: false, + environment: [{ name: "PRIME_PROFILE", value: "work", sensitive: false }], + config: { binaryPath: "/opt/prime/stock", agentHomePath: "/tmp/prime-work" }, + }, }, }, }); @@ -1102,14 +1188,18 @@ it.layer(NodeServices.layer)("server settings", (it) => { environment: [{ name: "PRIME_PROFILE", value: "work", sensitive: false }], config: { binaryPath: "/tmp/pylon-managed/.bin/prime-agent", - homePath: "/tmp/prime-work", + agentHomePath: "/tmp/prime-work", }, }); - yield* serverSettings.updateSettings({ - providerInstances: { - ...selected.providerInstances, - [instanceId]: { ...selected.providerInstances[instanceId]!, displayName: "Changed" }, + yield* serverSettings.mutateProviderInstances({ + mutationId: ServerProviderInstancesMutationId.make("test-prime-managed-binding-change"), + expectedProviderInstances: selected.providerInstances, + patch: { + providerInstances: { + ...selected.providerInstances, + [instanceId]: { ...selected.providerInstances[instanceId]!, displayName: "Changed" }, + }, }, }); assert.isUndefined( diff --git a/apps/server/src/serverSettings.ts b/apps/server/src/serverSettings.ts index f8ea230ec..ade8029af 100644 --- a/apps/server/src/serverSettings.ts +++ b/apps/server/src/serverSettings.ts @@ -19,12 +19,17 @@ import { type ProviderInstanceConfig, type ProviderInstanceEnvironmentVariable, ProviderDriverKind, + ServerProviderInstancesMutationConflictError, + type ServerProviderInstancesMutationInput, + type ServerProviderInstancesMutationReceipt, ProviderInstanceId, ServerSettings, ServerSettingsError, type ServerSettingsPatch, + ServerSettingsUpdateConflictError, } from "@t3tools/contracts"; import * as NodeCrypto from "node:crypto"; +import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Cache from "effect/Cache"; import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; @@ -45,6 +50,11 @@ import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as SqlClient from "effect/unstable/sql/SqlClient"; import { writeFileStringAtomically } from "./atomicWrite.ts"; +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import { + ProviderInstanceSettingsValidationError, + validateProviderInstanceSettings, +} from "./provider/providerInstanceSettingsValidation.ts"; import * as ServerConfig from "./config.ts"; import { type DeepPartial, deepMerge } from "@t3tools/shared/Struct"; import { fromJsonStringPretty, fromLenientJson } from "@t3tools/shared/schemaJson"; @@ -59,6 +69,9 @@ export { resolveSourceControlWriterModelSelection } from "@t3tools/shared/server const encodeServerSettings = Schema.encodeEffect(ServerSettings); const encodeServerSettingsJson = Schema.encodeUnknownEffect(fromJsonStringPretty(ServerSettings)); const decodeServerSettings = Schema.decodeUnknownEffect(ServerSettings); +const isProviderInstanceSettingsValidationError = Schema.is( + ProviderInstanceSettingsValidationError, +); const textEncoder = new TextEncoder(); const textDecoder = new TextDecoder(); @@ -127,6 +140,36 @@ const normalizeServerSettings = ( ), ); +const validateChangedProviderInstanceSettings = ( + current: ServerSettings, + next: ServerSettings, + settingsPath: string, +): Effect.Effect => { + if ( + Equal.equals(current.providerInstances, next.providerInstances) && + Equal.equals(current.providers, next.providers) + ) { + return Effect.void; + } + return Effect.gen(function* () { + const platform = yield* HostProcessPlatform; + const hostEnvironment = yield* HostProcessEnvironment; + yield* validateProviderInstanceSettings({ settings: next, platform, hostEnvironment }); + }).pipe( + Effect.mapError( + (cause) => + new ServerSettingsError({ + settingsPath, + operation: "normalize", + detail: isProviderInstanceSettingsValidationError(cause) + ? cause.message + : "Provider instance home validation failed.", + cause, + }), + ), + ); +}; + function providerEnvironmentSecretName(input: { readonly instanceId: string; readonly name: string; @@ -254,6 +297,20 @@ function patchPrimeAgentBinaryPath( return instanceId === "primeAgent" ? { providers: { primeAgent: { binaryPath } } } : undefined; } +function rejectLegacyProviderInstancesPatch( + patch: ServerSettingsPatch, +): Effect.Effect { + return patch.providerInstances === undefined + ? Effect.void + : Effect.fail( + new ServerSettingsUpdateConflictError({ + reason: "provider-instances-upgrade-required", + detail: + "This client cannot replace the provider instance map safely. Upgrade Pylon and retry through server.mutateProviderInstances.", + }), + ); +} + export class ServerSettingsService extends Context.Service< ServerSettingsService, { @@ -269,7 +326,15 @@ export class ServerSettingsService extends Context.Service< /** Patch settings and persist. Returns the new full settings object. */ readonly updateSettings: ( patch: ServerSettingsPatch, - ) => Effect.Effect; + ) => Effect.Effect; + + /** CAS/idempotent provider-instance mutation on the host server. */ + readonly mutateProviderInstances: ( + input: ServerProviderInstancesMutationInput, + ) => Effect.Effect< + ServerProviderInstancesMutationReceipt, + ServerSettingsError | ServerProviderInstancesMutationConflictError + >; /** Read the persisted complete Prime instance binding generation. */ readonly readPrimeAgentBinaryBinding?: ( @@ -319,17 +384,88 @@ const makeTest = (overrides: DeepPartial = {}) => : {}), }); const currentSettingsRef = yield* Ref.make(initialSettings); + const mutationReceipts = new Map< + string, + { + readonly input: ServerProviderInstancesMutationInput; + readonly receipt: ServerProviderInstancesMutationReceipt; + } + >(); + const mutationSemaphore = yield* Semaphore.make(1); return { start: Effect.void, ready: Effect.void, getSettings: Ref.get(currentSettingsRef).pipe(Effect.map(resolveTextGenerationProvider)), updateSettings: (patch) => - Ref.get(currentSettingsRef).pipe( - Effect.map((currentSettings) => applyServerSettingsPatch(currentSettings, patch)), - Effect.flatMap(normalizeServerSettings), - Effect.tap((nextSettings) => Ref.set(currentSettingsRef, nextSettings)), - Effect.map(resolveTextGenerationProvider), + Effect.gen(function* () { + yield* rejectLegacyProviderInstancesPatch(patch); + const current = yield* Ref.get(currentSettingsRef); + const next = yield* normalizeServerSettings(applyServerSettingsPatch(current, patch)); + yield* validateChangedProviderInstanceSettings(current, next, "").pipe( + Effect.provide(NodeServices.layer), + ); + yield* Ref.set(currentSettingsRef, next); + return resolveTextGenerationProvider(next); + }), + mutateProviderInstances: (input) => + mutationSemaphore.withPermits(1)( + Effect.gen(function* () { + const prior = mutationReceipts.get(input.mutationId); + if (prior !== undefined) { + if (!Equal.equals(prior.input, input)) { + return yield* new ServerProviderInstancesMutationConflictError({ + mutationId: input.mutationId, + reason: "mutation-reused", + detail: "This provider mutation id was already used for different input.", + }); + } + return { ...prior.receipt, disposition: "already-applied" as const }; + } + if (input.patch.providerInstances === undefined) { + return yield* new ServerProviderInstancesMutationConflictError({ + mutationId: input.mutationId, + reason: "invalid", + detail: "A provider instance mutation must include the complete next instance map.", + }); + } + const current = yield* Ref.get(currentSettingsRef); + if (!Equal.equals(current.providerInstances, input.expectedProviderInstances)) { + if (Equal.equals(current.providerInstances, input.patch.providerInstances)) { + const receipt = { + mutationId: input.mutationId, + disposition: "already-applied" as const, + settings: resolveTextGenerationProvider(current), + } satisfies ServerProviderInstancesMutationReceipt; + mutationReceipts.set(input.mutationId, { input, receipt }); + return receipt; + } + return yield* new ServerProviderInstancesMutationConflictError({ + mutationId: input.mutationId, + reason: "stale", + detail: + "Provider settings changed on the host. Reload the current instances and retry your edit.", + }); + } + const next = yield* normalizeServerSettings( + applyServerSettingsPatch(current, input.patch), + ); + yield* validateChangedProviderInstanceSettings(current, next, "").pipe( + Effect.provide(NodeServices.layer), + ); + yield* Ref.set(currentSettingsRef, next); + const receipt = { + mutationId: input.mutationId, + disposition: "applied" as const, + settings: resolveTextGenerationProvider(next), + } satisfies ServerProviderInstancesMutationReceipt; + mutationReceipts.set(input.mutationId, { input, receipt }); + if (mutationReceipts.size > 256) { + const oldest = mutationReceipts.keys().next().value; + if (oldest !== undefined) mutationReceipts.delete(oldest); + } + return receipt; + }), ), readPrimeAgentBinaryBinding: (instanceId) => Ref.get(currentSettingsRef).pipe( @@ -511,6 +647,13 @@ const make = Effect.gen(function* () { const secretStore = yield* ServerSecretStore.ServerSecretStore; const sql = yield* SqlClient.SqlClient; const writeSemaphore = yield* Semaphore.make(1); + const mutationReceipts = new Map< + string, + { + readonly input: ServerProviderInstancesMutationInput; + readonly receipt: ServerProviderInstancesMutationReceipt; + } + >(); const cacheKey = "settings" as const; const changesPubSub = yield* PubSub.unbounded(); const startedRef = yield* Ref.make(false); @@ -871,11 +1014,21 @@ const make = Effect.gen(function* () { updateSettings: (patch) => writeSemaphore.withPermits(1)( Effect.gen(function* () { + yield* rejectLegacyProviderInstancesPatch(patch); const current = yield* getSettingsFromCache; - const nextPersisted = yield* persistProviderEnvironmentSecrets( - current, - applyServerSettingsPatch(current, patch), + const currentMaterialized = yield* materializeProviderEnvironmentSecrets(current); + const candidate = yield* normalizeServerSettings( + applyServerSettingsPatch(currentMaterialized, patch), ); + yield* validateChangedProviderInstanceSettings( + currentMaterialized, + candidate, + settingsPath, + ).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, pathService), + ); + const nextPersisted = yield* persistProviderEnvironmentSecrets(current, candidate); const next = yield* normalizeServerSettings(nextPersisted); yield* writeSettingsAtomically(next); yield* Cache.set(settingsCache, cacheKey, next); @@ -913,6 +1066,82 @@ const make = Effect.gen(function* () { return primeAgentBinaryBinding(next, input.instanceId); }), ), + mutateProviderInstances: (input) => + writeSemaphore.withPermits(1)( + Effect.gen(function* () { + const prior = mutationReceipts.get(input.mutationId); + if (prior !== undefined) { + if (!Equal.equals(prior.input, input)) { + return yield* new ServerProviderInstancesMutationConflictError({ + mutationId: input.mutationId, + reason: "mutation-reused", + detail: "This provider mutation id was already used for different input.", + }); + } + return { ...prior.receipt, disposition: "already-applied" as const }; + } + if (input.patch.providerInstances === undefined) { + return yield* new ServerProviderInstancesMutationConflictError({ + mutationId: input.mutationId, + reason: "invalid", + detail: "A provider instance mutation must include the complete next instance map.", + }); + } + + const current = yield* getSettingsFromCache; + const currentMaterialized = yield* materializeProviderEnvironmentSecrets(current); + const currentClientProviderInstances = + redactServerSettingsForClient(currentMaterialized).providerInstances; + if (!Equal.equals(currentClientProviderInstances, input.expectedProviderInstances)) { + if (Equal.equals(currentClientProviderInstances, input.patch.providerInstances)) { + const receipt = { + mutationId: input.mutationId, + disposition: "already-applied" as const, + settings: resolveTextGenerationProvider(currentMaterialized), + } satisfies ServerProviderInstancesMutationReceipt; + mutationReceipts.set(input.mutationId, { input, receipt }); + return receipt; + } + return yield* new ServerProviderInstancesMutationConflictError({ + mutationId: input.mutationId, + reason: "stale", + detail: + "Provider settings changed on the host. Reload the current instances and retry your edit.", + }); + } + + const candidate = yield* normalizeServerSettings( + applyServerSettingsPatch(currentMaterialized, input.patch), + ); + yield* validateChangedProviderInstanceSettings( + currentMaterialized, + candidate, + settingsPath, + ).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, pathService), + ); + const nextPersisted = yield* persistProviderEnvironmentSecrets(current, candidate); + const next = yield* normalizeServerSettings(nextPersisted); + yield* writeSettingsAtomically(next); + yield* Cache.set(settingsCache, cacheKey, next); + yield* emitChange(next); + const materialized = resolveTextGenerationProvider( + yield* materializeProviderEnvironmentSecrets(next), + ); + const receipt = { + mutationId: input.mutationId, + disposition: "applied" as const, + settings: materialized, + } satisfies ServerProviderInstancesMutationReceipt; + mutationReceipts.set(input.mutationId, { input, receipt }); + if (mutationReceipts.size > 256) { + const oldest = mutationReceipts.keys().next().value; + if (oldest !== undefined) mutationReceipts.delete(oldest); + } + return receipt; + }), + ), get streamChanges() { return materializeChanges(Stream.fromPubSub(changesPubSub)); }, diff --git a/apps/server/src/textGeneration/PrimeAgentTextGeneration.ts b/apps/server/src/textGeneration/PrimeAgentTextGeneration.ts index 9fe097790..32b8c15b6 100644 --- a/apps/server/src/textGeneration/PrimeAgentTextGeneration.ts +++ b/apps/server/src/textGeneration/PrimeAgentTextGeneration.ts @@ -28,6 +28,7 @@ import { PRIME_AGENT_HOME_ENV, } from "../provider/acp/PrimeAgentAcpSupport.ts"; import { locatePrimeAgentPublicPackage } from "../provider/prime/PrimeAgentDaemonBridge.ts"; +import type { PrimeAgentRuntimeContext } from "../provider/prime/PrimeAgentRuntimeContext.ts"; import { PRIME_AGENT_INHERIT_MODEL_OPTION, resolvePrimeAgentTurnControls, @@ -75,6 +76,7 @@ export interface PrimeAgentTextGenerationOptions { readonly timeoutMs?: number | undefined; /** Test-only hook used to prove a validated image identity cannot be swapped before open. */ readonly beforeImageOpen?: ((filePath: string) => Effect.Effect) | undefined; + readonly runtimeContext?: PrimeAgentRuntimeContext; } function effectiveEnvironmentHome(environment: NodeJS.ProcessEnv): string { @@ -190,10 +192,11 @@ export const makePrimeAgentTextGeneration = Effect.fn("makePrimeAgentTextGenerat const path = yield* Path.Path; const commandSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const serverConfig = yield* ServerConfig.ServerConfig; - const resolvedEnvironment = makePrimeAgentEnvironment( - primeAgentSettings, - environment ?? process.env, - ); + const boundRuntimeContext = options.runtimeContext; + const effectiveSettings = boundRuntimeContext?.settings ?? primeAgentSettings; + const resolvedEnvironment = + boundRuntimeContext?.launchEnv ?? + makePrimeAgentEnvironment(effectiveSettings, environment ?? process.env); const helperEnvironment = Object.fromEntries( Object.entries(resolvedEnvironment).filter(([name]) => { const normalizedName = name.toUpperCase(); @@ -211,6 +214,18 @@ export const makePrimeAgentTextGeneration = Effect.fn("makePrimeAgentTextGenerat }), ); const timeoutMs = options.timeoutMs ?? PRIME_AGENT_TIMEOUT_MS; + const requireCurrentGeneration = (operation: PrimeAgentTextGenerationOperation) => + boundRuntimeContext?.runtimeFence === undefined + ? Effect.void + : boundRuntimeContext.runtimeFence.isCurrent.pipe( + Effect.flatMap((current) => + current + ? Effect.void + : Effect.fail( + textGenerationError(operation, "This Prime Agent runtime was replaced."), + ), + ), + ); const readBoundedImageFile = Effect.fn("readBoundedPrimeAgentImageFile")(function* ( filePath: string, @@ -315,6 +330,7 @@ export const makePrimeAgentTextGeneration = Effect.fn("makePrimeAgentTextGenerat readonly outputSchema: S; readonly modelSelection: ModelSelection; }): Effect.fn.Return { + yield* requireCurrentGeneration(input.operation); const controls = resolvePrimeAgentTurnControls(input.modelSelection); if (controls._tag === "Invalid") { return yield* textGenerationError(input.operation, controls.issue); @@ -326,17 +342,19 @@ export const makePrimeAgentTextGeneration = Effect.fn("makePrimeAgentTextGenerat const serviceTier = controls.serviceTier === PRIME_AGENT_INHERIT_MODEL_OPTION ? undefined : controls.serviceTier; - const agentDir = resolvePrimeAgentTextGenerationHomePath({ - environment: resolvedEnvironment, - cwd: input.cwd, - }); + const agentDir = + boundRuntimeContext?.effectiveHome ?? + resolvePrimeAgentTextGenerationHomePath({ + environment: resolvedEnvironment, + cwd: input.cwd, + }); const requestEnvironment = { ...helperEnvironment, NO_COLOR: "1", [PRIME_AGENT_HOME_ENV]: agentDir, }; const executablePath = yield* resolveCommandPath( - primeAgentSettings.binaryPath.trim() || "prime-agent", + effectiveSettings.binaryPath.trim() || "prime-agent", { env: requestEnvironment }, ).pipe( Effect.provideService(FileSystem.FileSystem, fileSystem), @@ -370,7 +388,11 @@ export const makePrimeAgentTextGeneration = Effect.fn("makePrimeAgentTextGenerat .makeTempFileScoped({ prefix: "pylon-prime-text-", suffix: ".mjs" }) .pipe( Effect.tap((filePath) => - fileSystem.writeFileString(filePath, PRIME_AGENT_TEXT_GENERATION_HELPER_SOURCE), + requireCurrentGeneration(input.operation).pipe( + Effect.andThen( + fileSystem.writeFileString(filePath, PRIME_AGENT_TEXT_GENERATION_HELPER_SOURCE), + ), + ), ), Effect.mapError(() => textGenerationError( @@ -405,6 +427,7 @@ export const makePrimeAgentTextGeneration = Effect.fn("makePrimeAgentTextGenerat killSignal: "SIGTERM", forceKillAfter: PRIME_AGENT_FORCE_KILL_AFTER, }); + yield* requireCurrentGeneration(input.operation); const child = yield* commandSpawner .spawn(command) .pipe( @@ -445,6 +468,7 @@ export const makePrimeAgentTextGeneration = Effect.fn("makePrimeAgentTextGenerat ); } + yield* requireCurrentGeneration(input.operation); const result = completed.value; if (result.stdout.truncated) { return yield* textGenerationError( diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 3ce1a01c6..118a18479 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -1943,6 +1943,19 @@ const makeWsRpcLayer = ( "rpc.aggregate": "server", }, ), + [WS_METHODS.serverMutateProviderInstances]: (input) => + observeRpcEffect( + WS_METHODS.serverMutateProviderInstances, + serverSettings.mutateProviderInstances(input).pipe( + Effect.map((receipt) => ({ + ...receipt, + settings: ServerSettings.redactServerSettingsForClient(receipt.settings), + })), + ), + { + "rpc.aggregate": "server", + }, + ), [WS_METHODS.serverDiscoverSourceControl]: (_input) => observeRpcEffect( WS_METHODS.serverDiscoverSourceControl, diff --git a/apps/web/src/components/settings/AddProviderInstanceDialog.environment.test.tsx b/apps/web/src/components/settings/AddProviderInstanceDialog.environment.test.tsx index 3c502c624..dfcde6232 100644 --- a/apps/web/src/components/settings/AddProviderInstanceDialog.environment.test.tsx +++ b/apps/web/src/components/settings/AddProviderInstanceDialog.environment.test.tsx @@ -1,10 +1,10 @@ -import { EnvironmentId } from "@t3tools/contracts"; +import { DEFAULT_SERVER_SETTINGS, EnvironmentId } from "@t3tools/contracts"; import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; import { reactHookHarness as hooks } from "../../test/reactHookHarness"; const settingsHooks = vi.hoisted(() => ({ - read: vi.fn(() => ({ providerInstances: {} })), + read: vi.fn(), update: vi.fn(() => vi.fn()), })); @@ -35,7 +35,8 @@ const remoteEnvironmentId = EnvironmentId.make("remote-device"); describe("AddProviderInstanceDialog environment routing", () => { beforeEach(() => { hooks.reset(); - settingsHooks.read.mockClear(); + settingsHooks.read.mockReset(); + settingsHooks.read.mockReturnValue(DEFAULT_SERVER_SETTINGS); settingsHooks.update.mockClear(); }); diff --git a/apps/web/src/components/settings/AddProviderInstanceDialog.tsx b/apps/web/src/components/settings/AddProviderInstanceDialog.tsx index 158908b5e..630924ce7 100644 --- a/apps/web/src/components/settings/AddProviderInstanceDialog.tsx +++ b/apps/web/src/components/settings/AddProviderInstanceDialog.tsx @@ -8,6 +8,7 @@ import { ProviderDriverKind, type EnvironmentId, type ProviderInstanceConfig, + type ServerProvider, } from "@t3tools/contracts"; import { useEnvironmentSettings, useUpdateEnvironmentSettings } from "../../hooks/useSettings"; @@ -36,6 +37,16 @@ import { type WizardNavigation, } from "./AddProviderInstanceDialog.logic"; import { AddProviderInstanceWizardSteps } from "./AddProviderInstanceWizardSteps"; +import { + countEnabledConfiguredInstances, + getDriverMultipleInstancePresentation, + PRIME_AGENT_ACP_GUIDANCE, + PRIME_AGENT_INSTANCE_GUIDANCE, + PRIME_AGENT_MAINTENANCE_GUIDANCE, + validatePrimeAgentAddHome, +} from "./providerMultipleInstances"; + +const EMPTY_SERVER_PROVIDERS: ReadonlyArray = []; const PROVIDER_ACCENT_SWATCHES = [ "#2563eb", @@ -119,6 +130,7 @@ interface AddProviderInstanceDialogProps { readonly open: boolean; readonly environmentId: EnvironmentId; readonly environmentLabel: string; + readonly serverProviders?: ReadonlyArray | undefined; readonly onOpenChange: (open: boolean) => void; } @@ -126,6 +138,7 @@ export function AddProviderInstanceDialog({ open, environmentId, environmentLabel, + serverProviders = EMPTY_SERVER_PROVIDERS, onOpenChange, }: AddProviderInstanceDialogProps) { const settings = useEnvironmentSettings(environmentId); @@ -160,6 +173,21 @@ export function AddProviderInstanceDialog({ const wizardStepSummaries = [driverOption.label, previewLabel, null] as const; const configDraft = configByDriver[driver] ?? EMPTY_CONFIG_DRAFT; + const multipleInstanceSupport = getDriverMultipleInstancePresentation({ + driver, + providers: serverProviders, + }); + const configuredEnabledCount = countEnabledConfiguredInstances(settings, driver); + const multipleInstancesBlocked = configuredEnabledCount > 0 && !multipleInstanceSupport.supported; + const primeHomeError = + driver === "primeAgent" && configuredEnabledCount > 0 + ? validatePrimeAgentAddHome({ draftConfig: configDraft, settings }) + : null; + const saveError = + instanceIdError ?? + (multipleInstancesBlocked + ? (multipleInstanceSupport.reason ?? "This driver supports only one enabled instance.") + : primeHomeError); const setConfigDraft = (config: Record | undefined) => { setConfigByDriver((existing) => { const next = { ...existing }; @@ -189,7 +217,7 @@ export function AddProviderInstanceDialog({ const handleSave = () => { setHasAttemptedSubmit(true); - if (instanceIdError !== null) return; + if (saveError !== null) return; const config = configByDriver[driver] ?? {}; const hasConfig = Object.keys(config).length > 0; @@ -263,11 +291,25 @@ export function AddProviderInstanceDialog({ > {DRIVER_OPTIONS.map((option) => { const IconComponent = option.icon; + const optionSupport = getDriverMultipleInstancePresentation({ + driver: option.value, + providers: serverProviders, + }); + const optionBlocked = + countEnabledConfiguredInstances(settings, option.value) > 0 && + !optionSupport.supported; return ( @@ -312,6 +354,11 @@ export function AddProviderInstanceDialog({ ); })} + {multipleInstancesBlocked ? ( +

+ {multipleInstanceSupport.reason} +

+ ) : null}