From ff116303739c21f0b4f7b49748cd9f0277916284 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 22 Aug 2026 00:16:38 -0700 Subject: [PATCH 1/2] Add test for CL-6640 boot-restore closure ENOENT quarantine Fabricates the exact wrapped-ENOENT shape readPackageJSON throws for a missing/incomplete closure staging directory and proves the boot restore loop quarantines it as a permanent failure without stopping, and that a sibling deployment's own restore is still attempted every boot. --- .../test/workflow-restore-quarantine.test.ts | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/apps/sidecar/test/workflow-restore-quarantine.test.ts b/apps/sidecar/test/workflow-restore-quarantine.test.ts index 1086126c..1db3a662 100644 --- a/apps/sidecar/test/workflow-restore-quarantine.test.ts +++ b/apps/sidecar/test/workflow-restore-quarantine.test.ts @@ -57,6 +57,10 @@ async function makeDataDir(): Promise { } const closureDefinitions = new Map(); +// CL-6640 fixture: deployment ids in this set fail materialization with +// the exact wrapped-ENOENT shape `readPackageJSON` (`@intx/workflow-host`) +// throws when a closure's staged `package.json` is missing. +const closureMissingStagingDir = new Set(); async function makeRouter( dataDir: string, @@ -87,6 +91,16 @@ async function makeRouter( unregisterDeployment: () => undefined, multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, materializeDeploymentClosure: ({ deploymentId }) => { + if (closureMissingStagingDir.has(deploymentId)) { + const rawEnoent = new Error( + "ENOENT: no such file or directory, open '.../store/@workbench-seed/last-30-days-research/0.0.0/package.json'", + ) as Error & { code: string }; + rawEnoent.code = "ENOENT"; + throw new Error( + `cannot read package.json for workflow package at ${dataDir}/workflow-definition-closures/${deploymentId}/packages/some-fresh-uuid/store/@workbench-seed/last-30-days-research/0.0.0`, + { cause: rawEnoent }, + ); + } const definition = closureDefinitions.get(deploymentId); if (definition === undefined) { throw new Error( @@ -234,3 +248,97 @@ test("a transiently unbuildable provider is retried every boot and never quarant expect(onDisk?.restoreFailure?.kind).toBe("transient"); expect(onDisk?.restoreFailure?.attempts).toBe(bootCount); }); + +// CL-6640: the sidecar crash-looped on boot because an ENOENT reading a +// closure's `package.json` -- observed for a UUID staging directory +// `applyFrozenWorkflowClosure` had just minted and populated moments +// earlier -- was falling through the boot loop's default "transient" +// classification, which never quarantines, so the record was retried +// forever and (by a mechanism outside this test's scope) the process +// itself died and was relaunched. This fabricates that exact failure +// shape -- `readPackageJSON`'s wrapped-ENOENT `Error` -- from the +// closure-materialization seam and proves: (a) the boot loop classifies +// it PERMANENT and quarantines it rather than retrying forever, (b) a +// sibling record's restore is still attempted every boot, unaffected, and +// (c) `restoreWorkflowDeployments()` itself never throws -- boot +// completes either way. +test("a missing/incomplete closure staging directory quarantines as a permanent failure without stopping the boot restore loop", async () => { + const dataDir = await makeDataDir(); + const brokenAgentAddress = "run_missing-staging-dir@example.com"; + const brokenDeploymentId = deriveDeploymentId(brokenAgentAddress); + closureMissingStagingDir.add(brokenDeploymentId); + + const healthyAgentAddress = "run_sibling-case@example.com"; + const healthyDeploymentId = deriveDeploymentId(healthyAgentAddress); + stageClosureDefinition(healthyDeploymentId); + + const router = await makeRouter(dataDir, { + assertSourceBuildable: () => undefined, + }); + + const brokenRecord: WorkflowDeploymentRecord = { + version: 1, + agentAddress: brokenAgentAddress, + definitionId: "def_1", + sources: {}, + approvedWireHash: "d".repeat(64), + sourceRef: SOURCE_REF, + }; + await writeWorkflowDeploymentRecord( + dataDir, + brokenDeploymentId, + brokenRecord, + ); + + const healthyRecord: WorkflowDeploymentRecord = { + version: 1, + agentAddress: healthyAgentAddress, + definitionId: "def_1", + // A buildable provider clears the source-admission gate and reaches + // `spawnWorkflowDeployment`, where the test's `recordingSpawner` + // refuses to launch a real child -- a plain `Error`, classified + // "transient" by the boot loop's default. What matters for this test + // is only that this record's own restore is attempted every boot, + // independent of the broken sibling record. + sources: { "step-1": [makeSource("buildable")] }, + approvedWireHash: "d".repeat(64), + sourceRef: SOURCE_REF, + }; + await writeWorkflowDeploymentRecord( + dataDir, + healthyDeploymentId, + healthyRecord, + ); + + for (let boot = 1; boot <= RESTORE_QUARANTINE_THRESHOLD; boot++) { + // Boot completes without throwing -- the ENOENT never escapes the + // restore loop, however it is classified. + await expect(router.restoreWorkflowDeployments()).resolves.toBeUndefined(); + const broken = await readWorkflowDeploymentRecord( + dataDir, + brokenDeploymentId, + ); + expect(broken?.restoreFailure?.kind).toBe("permanent"); + expect(broken?.restoreFailure?.attempts).toBe(boot); + + // The sibling deployment's own restore was attempted on every boot, + // unaffected by the broken record ahead of (or behind) it in the scan. + const healthy = await readWorkflowDeploymentRecord( + dataDir, + healthyDeploymentId, + ); + expect(healthy?.restoreFailure?.kind).toBe("transient"); + expect(healthy?.restoreFailure?.attempts).toBe(boot); + } + + // One more boot: quarantined now, so the attempt count stops moving -- + // the record is skipped rather than re-attempted and re-failed. + await router.restoreWorkflowDeployments(); + const afterQuarantine = await readWorkflowDeploymentRecord( + dataDir, + brokenDeploymentId, + ); + expect(afterQuarantine?.restoreFailure?.attempts).toBe( + RESTORE_QUARANTINE_THRESHOLD, + ); +}); From ecc9822ac01832d96e7025b359f56655dc079e35 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 22 Aug 2026 00:16:48 -0700 Subject: [PATCH 2/2] CL-6640: classify a missing closure staging dir as a permanent restore failure Boot restore was crash-looping: an ENOENT reading a freshly-staged closure's package.json fell through restoreDeploymentFromRecord's default "transient" classification, so it was retried every boot without ever quarantining. applyFrozenWorkflowClosure re-materializes into a brand-new deploy-id directory on every call and reads back from it in the same await chain, so a missing file there is never this boot's timing -- it is corrupt/incomplete persisted input (the tarball cache, or a durable source-asset checkout) that reproduces identically on every future retry. Classify it permanent so it quarantines after RESTORE_QUARANTINE_THRESHOLD attempts instead of warning forever. --- .../sidecar/src/workflow-host-wiring/index.ts | 63 ++++++++++++++++--- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring/index.ts b/apps/sidecar/src/workflow-host-wiring/index.ts index 766e8f76..a0ed5594 100644 --- a/apps/sidecar/src/workflow-host-wiring/index.ts +++ b/apps/sidecar/src/workflow-host-wiring/index.ts @@ -69,6 +69,7 @@ import { restoreAgentIdentity, snapshotAgentIdentity, } from "../hibernated-agent-identity-vault"; +import { isErrnoNotFound } from "../conversation-state"; import { computeWireDefinitionHash, validateWorkflowProjection, @@ -1478,6 +1479,28 @@ export function createSidecarDeployRouter(deps: { } } + /** + * Whether a closure-materialization failure is an ENOENT reading the + * workflow package's `package.json` (or a `package.json`-shaped read + * inside the staged closure) -- the CL-6640 crash-loop shape: + * `applyFrozenWorkflowClosure` stages fresh into a new deploy-id + * directory on every call and immediately reads back from that same + * directory in the same `await` chain, so a missing file there is never + * this boot's timing -- it is corrupt/incomplete PERSISTED input (the + * tarball cache, or a durable source-asset checkout) that reproduces + * identically on every future retry. `readPackageJSON` + * (`@intx/workflow-host`) wraps the raw ENOENT in a describing `Error` + * with `{ cause }`, so both the direct code and the wrapped cause's code + * are checked. + */ + function isMissingClosureStagingFailure(cause: unknown): boolean { + if (!(cause instanceof Error)) return false; + if (isErrnoNotFound(cause)) return true; + return ( + "cause" in cause && isErrnoNotFound((cause as { cause?: unknown }).cause) + ); + } + /** * Re-establish one persisted deployment from its on-disk record -- the * shared core of the boot-time restore loop and the CL-5477 idle-reap @@ -1488,9 +1511,10 @@ export function createSidecarDeployRouter(deps: { * the record is never deleted here. A failure that is intrinsic to the * record's own persisted bytes -- deterministic, will recur identically * on every future boot -- throws a `WorkflowRestoreFailure("permanent", - * ...)`: address-derivation mismatch and closure-derived-definition - * validation are the only two such gates below. Every other failure - * (an unbuildable inference provider, a closure-materialization miss, a + * ...)`: address-derivation mismatch, closure-derived-definition + * validation, and a missing/incomplete closure staging directory + * (CL-6640 -- see `isMissingClosureStagingFailure`) are the only such + * gates below. Every other failure (an unbuildable inference provider, a * spawn-core throw) is a plain `Error`, which the caller treats as * "transient" -- it depends on this boot's environment, not the * record's content, and may clear on a later boot with no change to the @@ -1534,12 +1558,33 @@ export function createSidecarDeployRouter(deps: { // `hubLink.connect()`. Asset-sourced entries read from the durable source // store the original deploy checked out, so no re-delivery is needed; a // store miss soft-fails the record (kept for the next boot). - const applied = await applyClosure({ - dataDir, - deploymentId, - pin: record.sourceRef, - substrateEnv: multistepSubstrateEnv, - }); + let applied: Awaited>; + try { + applied = await applyClosure({ + dataDir, + deploymentId, + pin: record.sourceRef, + substrateEnv: multistepSubstrateEnv, + }); + } catch (cause) { + if (isMissingClosureStagingFailure(cause)) { + // The closure re-materializes fresh into a brand-new staging + // directory on every single restore attempt (a new deploy-id is + // minted each call; see `applyFrozenWorkflowClosure`), so an ENOENT + // reading the package it JUST staged is not this boot's + // environment -- it is a corrupt/incomplete PERSISTED input (the + // pinned closure's tarball cache entry, or the durable source-asset + // checkout `resolveDeploymentAssetMounts` already validated) that + // re-derives byte-for-byte the same broken result on every future + // boot too. Permanent, so it quarantines instead of warning forever + // while silently never making progress. + throw new WorkflowRestoreFailure( + "permanent", + cause instanceof Error ? cause.message : String(cause), + ); + } + throw cause; + } const validatedDefinition = WorkflowProjectionDefinition( projectLiveToInert(applied.definition), );