From fb7194788384e5b9a6cb70a77775612d8ffab540 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Fri, 21 Aug 2026 19:46:44 -0700 Subject: [PATCH 1/2] Add tests for hibernated agent identity preservation CL-6581: the published @intx/hub-agent package destroys an agent's reconnect-challenge keypair on every undeploy, hibernate or not, which is the documented root cause of CL-6203/CL-6044. These tests exercise a snapshot-before-delete/restore-before-redeploy technique (proved viable in PR #291) against the real, unmodified @intx/hub-agent key/repo stores, and end-to-end through createSidecarDeployRouter's actual undeploy/deploy hooks: a hibernate teardown must preserve the identity across a real redeploy, a reclaiming teardown must still destroy it, a broken snapshot must report loudly instead of failing silent, and an orphaned snapshot must be reaped past its retention window. The shared lifecycle fixture gains an optional real-key-store override (needed because its default fake never touches disk) and its default fake now creates a real agentDir so an unrelated hibernate-teardown test does not trip the vault's ordering-broke detector. --- .../hibernated-agent-identity-vault.test.ts | 182 ++++++++++++++++++ .../support/workflow-lifecycle-fixture.ts | 32 ++- ...flow-suspend-identity-preservation.test.ts | 131 +++++++++++++ 3 files changed, 337 insertions(+), 8 deletions(-) create mode 100644 apps/sidecar/src/hibernated-agent-identity-vault.test.ts create mode 100644 apps/sidecar/test/workflow-suspend-identity-preservation.test.ts diff --git a/apps/sidecar/src/hibernated-agent-identity-vault.test.ts b/apps/sidecar/src/hibernated-agent-identity-vault.test.ts new file mode 100644 index 000000000..16571ac69 --- /dev/null +++ b/apps/sidecar/src/hibernated-agent-identity-vault.test.ts @@ -0,0 +1,182 @@ +// Exercises `hibernated-agent-identity-vault.ts` against the real, +// unmodified `@intx/hub-agent` key/repo stores -- the same technique +// `apps/sidecar/test/suspend-key-preservation.poc.test.ts` (PR #291) +// proved works, now against the actual snapshot/restore/reap module this +// repo ships. +import { describe, test, expect, mock } from "bun:test"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { + createAgentKeyStore, + createAgentRepoStore, + agentDir, +} from "@intx/hub-agent"; +import { generateKeyPair, signEd25519, verifySSHSignature } from "@intx/crypto"; + +import { + snapshotAgentIdentity, + restoreAgentIdentity, + reapExpiredHibernationSnapshots, +} from "./hibernated-agent-identity-vault"; + +const cryptoOps = { + generateKeyPair, + signEd25519, + verifySSHSig: verifySSHSignature, +}; + +async function makeTmpDataDir(): Promise { + return fs.mkdtemp(path.join(os.tmpdir(), "hibernated-agent-vault-")); +} + +async function modeOf(target: string): Promise { + const stat = await fs.stat(target); + return stat.mode & 0o777; +} + +describe("hibernated-agent-identity-vault", () => { + test("snapshot then restore preserves the real agent keypair with isNew: false", async () => { + const dataDir = await makeTmpDataDir(); + const keyStore = createAgentKeyStore({ dataDir, ...cryptoOps }); + const repoStore = createAgentRepoStore({ dataDir }); + const address = "run_hibernate-vault@example.com"; + + const { keyPair: original } = await keyStore.loadOrGenerateKey(address); + + const snapshot = await snapshotAgentIdentity(dataDir, address); + expect(snapshot.snapshotted).toBe(true); + + // The real, unmodified destructive call `hub-link.js` makes on every + // undeploy -- exercised here exactly as the PR #291 spike exercised it. + await repoStore.remove(address); + expect(await fs.readdir(dataDir)).not.toContain( + path.basename(agentDir(dataDir, address)), + ); + + const restore = await restoreAgentIdentity(dataDir, address); + expect(restore.restored).toBe(true); + + const { keyPair: restored, isNew } = + await keyStore.loadOrGenerateKey(address); + expect(isNew).toBe(false); + expect(restored.privateKey).toEqual(original.privateKey); + expect(restored.publicKey).toEqual(original.publicKey); + }); + + test("no snapshot means the destructive remove wins -- a fresh keypair comes back", async () => { + const dataDir = await makeTmpDataDir(); + const keyStore = createAgentKeyStore({ dataDir, ...cryptoOps }); + const repoStore = createAgentRepoStore({ dataDir }); + const address = "run_no-snapshot@example.com"; + + const { keyPair: original } = await keyStore.loadOrGenerateKey(address); + + // No snapshotAgentIdentity call: models the reclaimDirs: true (real + // undeploy) path, which never protects the identity directory. + await repoStore.remove(address); + + const restore = await restoreAgentIdentity(dataDir, address); + expect(restore.restored).toBe(false); + + const { keyPair: regenerated, isNew } = + await keyStore.loadOrGenerateKey(address); + expect(isNew).toBe(true); + expect(regenerated.privateKey).not.toEqual(original.privateKey); + }); + + test("the vault snapshot is hardened to owner-only permissions", async () => { + const dataDir = await makeTmpDataDir(); + const keyStore = createAgentKeyStore({ dataDir, ...cryptoOps }); + const address = "run_permissions@example.com"; + await keyStore.loadOrGenerateKey(address); + + await snapshotAgentIdentity(dataDir, address); + + const vaultEntry = path.join( + dataDir, + "hibernated-agent-identity", + path.basename(agentDir(dataDir, address)), + ); + expect(await modeOf(vaultEntry)).toBe(0o700); + const keysDir = path.join(vaultEntry, "keys"); + expect(await modeOf(keysDir)).toBe(0o700); + const privateKeyFile = path.join(keysDir, "id_ed25519"); + expect(await modeOf(privateKeyFile)).toBe(0o600); + }); + + test("a snapshot taken after the identity directory is already gone reports loudly and returns snapshotted: false", async () => { + const dataDir = await makeTmpDataDir(); + const address = "run_already-gone@example.com"; + + const reportErrorMock = mock( + (_error: unknown, _context: unknown) => "ref-test", + ); + mock.module("@corbits/error-sink", () => ({ + reportError: reportErrorMock, + })); + const { snapshotAgentIdentity: freshSnapshot } = + await import("./hibernated-agent-identity-vault"); + + const result = await freshSnapshot(dataDir, address); + + expect(result.snapshotted).toBe(false); + expect(reportErrorMock).toHaveBeenCalledTimes(1); + const context = reportErrorMock.mock.calls[0]?.[1]; + expect(context).toMatchObject({ + operation: "hibernated-agent-identity-vault.snapshot", + agentId: address, + }); + + mock.restore(); + }); + + test("reapExpiredHibernationSnapshots deletes only entries past retention", async () => { + const dataDir = await makeTmpDataDir(); + const keyStoreOld = createAgentKeyStore({ dataDir, ...cryptoOps }); + const keyStoreFresh = createAgentKeyStore({ dataDir, ...cryptoOps }); + const oldAddress = "run_old-orphan@example.com"; + const freshAddress = "run_fresh-hibernate@example.com"; + const retentionMs = 1_000; + + await keyStoreOld.loadOrGenerateKey(oldAddress); + await snapshotAgentIdentity(dataDir, oldAddress); + // Backdate the orphan's marker past retention; a real orphan is one + // nothing has redeployed (and thus restored) since it hibernated. + const oldVaultEntry = path.join( + dataDir, + "hibernated-agent-identity", + path.basename(agentDir(dataDir, oldAddress)), + ); + await fs.writeFile( + path.join(oldVaultEntry, ".snapshotted-at"), + new Date(Date.now() - retentionMs - 1).toISOString(), + { mode: 0o600 }, + ); + + await keyStoreFresh.loadOrGenerateKey(freshAddress); + await snapshotAgentIdentity(dataDir, freshAddress); + + const result = await reapExpiredHibernationSnapshots(dataDir, { + retentionMs, + }); + + expect(result.reapedEntries).toEqual([ + path.basename(agentDir(dataDir, oldAddress)), + ]); + await expect(fs.stat(oldVaultEntry)).rejects.toThrow(); + const freshVaultEntry = path.join( + dataDir, + "hibernated-agent-identity", + path.basename(agentDir(dataDir, freshAddress)), + ); + await expect(fs.stat(freshVaultEntry)).resolves.toBeDefined(); + }); + + test("reapExpiredHibernationSnapshots on a vault-less data dir is a no-op", async () => { + const dataDir = await makeTmpDataDir(); + const result = await reapExpiredHibernationSnapshots(dataDir); + expect(result.reapedEntries).toEqual([]); + }); +}); diff --git a/apps/sidecar/test/support/workflow-lifecycle-fixture.ts b/apps/sidecar/test/support/workflow-lifecycle-fixture.ts index ce6745f1e..976915ccb 100644 --- a/apps/sidecar/test/support/workflow-lifecycle-fixture.ts +++ b/apps/sidecar/test/support/workflow-lifecycle-fixture.ts @@ -12,6 +12,7 @@ import path from "node:path"; import { createEd25519Crypto, generateKeyPair } from "@intx/crypto"; import { hexEncode } from "@intx/types"; +import { agentDir } from "@intx/hub-agent"; import { createInMemoryTransport } from "@intx/mail-memory"; import type { RepoId, RepoStore } from "@intx/hub-sessions"; import { @@ -201,6 +202,13 @@ export async function makeLifecycleFixture(opts?: { * restart (boot-time restore). */ dataDir?: string; + /** + * Override the default in-memory fake key store with a real one (e.g. + * `@intx/hub-agent`'s `createAgentKeyStore` bound to the same + * `dataDir`), for a test that needs the actual on-disk key-persistence + * behavior the fake bypasses entirely. + */ + keyStore?: Parameters[0]["keyStore"]; }): Promise { const spawns: Spawn[] = []; const spawner: SubprocessSpawner = ({ env }) => { @@ -279,14 +287,22 @@ export async function makeLifecycleFixture(opts?: { initRepo: async () => undefined, } as unknown as Parameters[0]["sessions"], // Boundary type assertion: the single-step branch registers the agent's signing key (loadOrGenerateKey) and records the hub key (recordHubKey) at the head before spawn - keyStore: { - recordHubKey: () => undefined, - forgetAgent: () => undefined, - loadOrGenerateKey: async () => ({ - keyPair: await generateKeyPair(), - isNew: false, - }), - } as unknown as Parameters[0]["keyStore"], + keyStore: + opts?.keyStore ?? + ({ + recordHubKey: () => undefined, + forgetAgent: () => undefined, + // Mirrors the real `AgentKeyStore`'s on-disk contract just enough + // for `hibernated-agent-identity-vault.ts`'s snapshot step to find + // a real `agentDir` to preserve: a fixture-driven hibernate + // teardown must not spuriously trip its "ordering broke" report. + loadOrGenerateKey: async (address: string) => { + await fs.mkdir(agentDir(dataDir, address), { recursive: true }); + return { keyPair: await generateKeyPair(), isNew: false }; + }, + } as unknown as Parameters< + typeof createSidecarDeployRouter + >[0]["keyStore"]), transport, repoStore, signingKeySeed: keyPair.privateKey, diff --git a/apps/sidecar/test/workflow-suspend-identity-preservation.test.ts b/apps/sidecar/test/workflow-suspend-identity-preservation.test.ts new file mode 100644 index 000000000..4106d852b --- /dev/null +++ b/apps/sidecar/test/workflow-suspend-identity-preservation.test.ts @@ -0,0 +1,131 @@ +// End-to-end proof that the real `createSidecarDeployRouter` wiring -- +// not just the standalone `hibernated-agent-identity-vault.ts` module -- +// preserves an agent's reconnect-challenge keypair across a +// state-preserving "hibernate" teardown (reclaimDirs: false), and still +// lets a reclaiming (non-hibernate) teardown destroy it. Uses a REAL +// `@intx/hub-agent` key store bound to the fixture's data dir (via +// `makeLifecycleFixture`'s `keyStore` override) rather than the shared +// fixture's default in-memory fake, which never touches disk and so +// cannot exercise this path. +import { describe, test, expect } from "bun:test"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { + createAgentKeyStore, + createAgentRepoStore, + agentDir, +} from "@intx/hub-agent"; +import { generateKeyPair, signEd25519, verifySSHSignature } from "@intx/crypto"; +import { hexEncode } from "@intx/types"; + +import { + answerReadyHandshake, + makeLifecycleFixture, + makeWorkflowFrame, +} from "./support/workflow-lifecycle-fixture"; + +const cryptoOps = { + generateKeyPair, + signEd25519, + verifySSHSig: verifySSHSignature, +}; + +// `makeWorkflowFrame`'s default `hubPublicKey` ("hub-pk") is a placeholder +// the shared fixture's FAKE key store never validates. The real +// `AgentKeyStore.recordHubKey` this suite exercises hex-decodes it, so +// every frame here carries a real hex-encoded key instead. +async function makeFrameWithRealHubKey(agentAddress: string) { + const hubKeyPair = await generateKeyPair(); + return { + ...makeWorkflowFrame(agentAddress), + hubPublicKey: hexEncode(hubKeyPair.publicKey), + }; +} + +describe("hibernate/wake preserves the real deployed agent's identity", () => { + test("hibernate teardown, then redeploy: the same keypair comes back (isNew: false)", async () => { + const dataDir = await fs.mkdtemp( + path.join(os.tmpdir(), "sidecar-suspend-identity-"), + ); + const keyStore = createAgentKeyStore({ dataDir, ...cryptoOps }); + // The real, unmodified package's own repo store -- its `remove` is + // exactly the destructive call `@intx/hub-agent`'s `ws/hub-link.js` + // issues AFTER our `undeploy` hook returns for every undeploy, + // hibernate or not. + const realRepoStore = createAgentRepoStore({ dataDir }); + + const { router, spawns } = await makeLifecycleFixture({ + dataDir, + keyStore, + }); + const address = "run_suspend-hibernate@example.com"; + + const frame1 = await makeFrameWithRealHubKey(address); + const deploy1 = router.deploy(frame1); + await answerReadyHandshake(spawns, 0); + await deploy1; + + const { keyPair: original } = await keyStore.loadOrGenerateKey(address); + + await router.teardownDeployment(address, { reclaimDirs: false }); + // Models the published package's own post-hook delete. + await realRepoStore.remove(address); + expect(await fs.readdir(dataDir)).not.toContain( + path.basename(agentDir(dataDir, address)), + ); + + const frame2 = await makeFrameWithRealHubKey(address); + const deploy2 = router.deploy(frame2); + await answerReadyHandshake(spawns, 1); + await deploy2; + + // The real proof: deploy2's redeploy loaded back the SAME key bytes + // `original` held before the hibernate teardown + destructive remove, + // not a freshly minted keypair. + const { keyPair: restored } = await keyStore.loadOrGenerateKey(address); + expect(restored.privateKey).toEqual(original.privateKey); + expect(restored.publicKey).toEqual(original.publicKey); + }); + + test("reclaiming (non-hibernate) teardown, then redeploy: a fresh keypair comes back", async () => { + const dataDir = await fs.mkdtemp( + path.join(os.tmpdir(), "sidecar-suspend-identity-"), + ); + const keyStore = createAgentKeyStore({ dataDir, ...cryptoOps }); + const realRepoStore = createAgentRepoStore({ dataDir }); + + const { router, spawns } = await makeLifecycleFixture({ + dataDir, + keyStore, + }); + const address = "run_suspend-reclaim@example.com"; + + const frame1 = await makeFrameWithRealHubKey(address); + const deploy1 = router.deploy(frame1); + await answerReadyHandshake(spawns, 0); + await deploy1; + + const { keyPair: original } = await keyStore.loadOrGenerateKey(address); + + await router.teardownDeployment(address, { reclaimDirs: true }); + await realRepoStore.remove(address); + + const frame2 = await makeFrameWithRealHubKey(address); + const deploy2 = router.deploy(frame2); + await answerReadyHandshake(spawns, 1); + await deploy2; + + // A THIRD `loadOrGenerateKey` call for the same address always reads + // back `isNew: false` regardless of whether deploy2's own internal + // call (inside `spawnWorkflowDeployment`) minted a fresh key or + // restored an old one -- some key now exists on disk either way, so + // `isNew` here would not distinguish the two. The actual proof is the + // key material itself: a destructive (reclaimDirs: true) teardown + // must leave deploy2 minting a DIFFERENT keypair than `original`. + const { keyPair: regenerated } = await keyStore.loadOrGenerateKey(address); + expect(regenerated.privateKey).not.toEqual(original.privateKey); + expect(regenerated.publicKey).not.toEqual(original.publicKey); + }); +}); From 8c78891914f20f2607a5925222f0de712724342d Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Fri, 21 Aug 2026 19:46:56 -0700 Subject: [PATCH 2/2] Preserve a hibernated agent's identity keypair across undeploy/redeploy CL-6581, compensating for CL-6239 (still open -- the real fix is a non-destructive upstream undeploy). The published @intx/hub-agent package's handleAgentUndeploy unconditionally deletes an agent's reconnect-challenge keypair on every sendAgentUndeploy, hibernate or not, but only AFTER this sidecar's undeploy hook returns. New hibernated-agent-identity-vault.ts snapshots agentDir (hub-agent's only stable public export for this path) into this sidecar's own data dir, hardened to 0600/0700, before that delete runs, and restores it before the next deploy's loadOrGenerateKey call. Wired into createSidecarDeployRouter: teardownDeployment snapshots when reclaimDirs is false (the hibernate flavor); spawnWorkflowDeployment restores before minting/loading the deployment's key, and reports through reportError if a restored snapshot still yields a fresh keypair. A new reapExpiredHibernationSnapshots sweep (run once at boot) reclaims any snapshot whose address hibernated and was never redeployed, so an abandoned hibernate does not leak disk forever. A missing agentDir at snapshot time is reported the same way -- the signal that a future @intx/hub-agent release changed the delete/hook call ordering this technique depends on. --- apps/sidecar/package.json | 1 + .../src/hibernated-agent-identity-vault.ts | 224 ++++++++++++++++++ apps/sidecar/src/index.ts | 7 + .../sidecar/src/workflow-host-wiring/index.ts | 68 +++++- 4 files changed, 297 insertions(+), 3 deletions(-) create mode 100644 apps/sidecar/src/hibernated-agent-identity-vault.ts diff --git a/apps/sidecar/package.json b/apps/sidecar/package.json index a6d1b0def..d11a8cae1 100644 --- a/apps/sidecar/package.json +++ b/apps/sidecar/package.json @@ -17,6 +17,7 @@ "dependencies": { "@corbits/agent-lifecycle": "workspace:*", "@corbits/credential-providers": "workspace:*", + "@corbits/error-sink": "workspace:*", "@corbits/ollama-adapter": "workspace:*", "@corbits/workflow-host-actions": "workspace:*", "@intx/agent": "0.3.0", diff --git a/apps/sidecar/src/hibernated-agent-identity-vault.ts b/apps/sidecar/src/hibernated-agent-identity-vault.ts new file mode 100644 index 000000000..1239add03 --- /dev/null +++ b/apps/sidecar/src/hibernated-agent-identity-vault.ts @@ -0,0 +1,224 @@ +// Snapshot/restore for an agent's on-disk identity directory across a +// state-preserving "hibernate" teardown (CL-6581, compensating for +// CL-6239's still-open upstream ask). +// +// WHY THIS EXISTS: the published `@intx/hub-agent` package's +// `handleAgentUndeploy` (`ws/hub-link.js`) unconditionally destroys +// `agentDir(dataDir, address)` -- which holds the agent's +// reconnect-challenge Ed25519 keypair under its `keys/` subdirectory -- +// on EVERY `sendAgentUndeploy`, hibernate or not, and that call happens +// AFTER this sidecar's `undeploy` hook returns. Losing that keypair is the +// documented root cause of CL-6203/CL-6044: a woken agent mints a fresh +// identity, fails the hub's reconnect challenge, and the conversation goes +// silent. `agentDir` is `@intx/hub-agent`'s only stable public export for +// this path (its `keys/` subdirectory name is a documented internal +// implementation detail, not exported), so this module snapshots the whole +// directory before the destructive delete and restores it before the next +// deploy -- a workaround that depends on that call-ordering, not a fix. +// CL-6239 stays open for the real one: a non-destructive upstream undeploy. +import type { Dirent } from "node:fs"; +import fsp from "node:fs/promises"; +import path from "node:path"; + +import { agentDir } from "@intx/hub-agent"; +import { reportError } from "@corbits/error-sink"; + +import { isErrnoNotFound } from "./conversation-state"; + +const VAULT_DIRNAME = "hibernated-agent-identity"; +const MARKER_FILENAME = ".snapshotted-at"; + +/** + * How long an untouched snapshot may sit in the vault before + * `reapExpiredHibernationSnapshots` reclaims it. Generous relative to any + * expected idle-sleep duration (hours to low days): an agent that is + * legitimately woken always redeploys and restores its snapshot well + * inside this window. An agent hibernated and then never redeployed + * (channel deleted, member removed, workspace archived, ...) instead + * leaves an orphaned snapshot on disk -- this bounds that orphan's + * lifetime rather than letting it accumulate forever, per the same class + * of leak that left 62 dead deployment records unreaped. + */ +export const HIBERNATION_SNAPSHOT_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; + +function vaultRoot(dataDir: string): string { + return path.join(dataDir, VAULT_DIRNAME); +} + +function vaultEntryDir(dataDir: string, agentAddress: string): string { + return path.join( + vaultRoot(dataDir), + path.basename(agentDir(dataDir, agentAddress)), + ); +} + +async function pathExists(target: string): Promise { + try { + await fsp.access(target); + return true; + } catch (cause) { + if (isErrnoNotFound(cause)) return false; + throw cause; + } +} + +/** + * Recursively chmods every file 0600 and every directory 0700, key + * material's minimum-privilege mode -- matching the precedent in + * `workflow-deployment-record.ts`'s `writeWorkflowDeploymentRecord`. Run + * after a plain `fsp.cp`, which does not otherwise guarantee the copy's + * permission bits regardless of the source's. + */ +async function hardenPermissionsRecursive(root: string): Promise { + const entries = await fsp.readdir(root, { withFileTypes: true }); + for (const entry of entries) { + const entryPath = path.join(root, entry.name); + if (entry.isDirectory()) { + await hardenPermissionsRecursive(entryPath); + await fsp.chmod(entryPath, 0o700); + } else { + await fsp.chmod(entryPath, 0o600); + } + } +} + +export type SnapshotAgentIdentityResult = { snapshotted: boolean }; + +/** + * Copy `agentDir(dataDir, agentAddress)` into this sidecar's own vault + * directory, hardened to owner-only permissions, before the published + * package's destructive undeploy delete reaches it. Must be called and + * awaited from inside the `undeploy` hook, before it returns -- the + * published `handleAgentUndeploy`'s delete runs only after that hook + * settles, and that ordering is this technique's entire foundation. + * + * A missing source directory at this point means that ordering has + * already broken (a future `@intx/hub-agent` release deletes before + * calling this hook), so every wake from here on would silently mint a + * fresh identity -- the exact CL-6203 bug class. That is reported loudly + * through `reportError` rather than left to fail silent; the teardown + * itself still proceeds either way. + */ +export async function snapshotAgentIdentity( + dataDir: string, + agentAddress: string, +): Promise { + const source = agentDir(dataDir, agentAddress); + const dest = vaultEntryDir(dataDir, agentAddress); + + if (!(await pathExists(source))) { + reportError( + new Error( + "hibernate snapshot found no agent identity directory to preserve; " + + "@intx/hub-agent's undeploy call ordering may have changed, which " + + "would make every subsequent wake for this address mint a fresh " + + "identity and fail the hub's reconnect challenge", + ), + { + operation: "hibernated-agent-identity-vault.snapshot", + agentId: agentAddress, + }, + ); + return { snapshotted: false }; + } + + await fsp.rm(dest, { recursive: true, force: true }); + await fsp.mkdir(vaultRoot(dataDir), { recursive: true, mode: 0o700 }); + await fsp.cp(source, dest, { recursive: true }); + await hardenPermissionsRecursive(dest); + await fsp.chmod(dest, 0o700); + await fsp.writeFile( + path.join(dest, MARKER_FILENAME), + new Date().toISOString(), + { + mode: 0o600, + }, + ); + return { snapshotted: true }; +} + +export type RestoreAgentIdentityResult = { restored: boolean }; + +/** + * Restore a previously vaulted identity directory back to + * `agentDir(dataDir, agentAddress)`, if one exists, and remove it from the + * vault. A no-op returning `{ restored: false }` for an address that was + * never snapshotted (an ordinary fresh deploy) -- the caller's own + * `loadOrGenerateKey` call then mints a fresh identity exactly as it + * would with no vault involved. Must run BEFORE that `loadOrGenerateKey` + * call so it observes the restored files rather than minting a fresh key + * over an empty directory. + */ +export async function restoreAgentIdentity( + dataDir: string, + agentAddress: string, +): Promise { + const source = vaultEntryDir(dataDir, agentAddress); + if (!(await pathExists(source))) { + return { restored: false }; + } + + const dest = agentDir(dataDir, agentAddress); + await fsp.rm(path.join(source, MARKER_FILENAME), { force: true }); + await fsp.rm(dest, { recursive: true, force: true }); + await fsp.cp(source, dest, { recursive: true }); + await fsp.rm(source, { recursive: true, force: true }); + return { restored: true }; +} + +export type ReapExpiredHibernationSnapshotsResult = { + /** Opaque sanitized-address directory names reaped, for observability -- never the raw address. */ + reapedEntries: string[]; +}; + +/** + * Sweep the vault for entries older than `retentionMs` + * (`HIBERNATION_SNAPSHOT_RETENTION_MS` by default) and delete them. An + * entry with no marker (an interrupted snapshot write) is treated as + * immediately expired rather than kept forever with no way to age it. + * Intended to run once at boot, independent of and before/after the + * unrelated deployment-record boot-restore scan -- this function does not + * touch that scan or its concurrency. + */ +export async function reapExpiredHibernationSnapshots( + dataDir: string, + opts: { retentionMs?: number; nowMs?: number } = {}, +): Promise { + const retentionMs = opts.retentionMs ?? HIBERNATION_SNAPSHOT_RETENTION_MS; + const nowMs = opts.nowMs ?? Date.now(); + const root = vaultRoot(dataDir); + + let entries: Dirent[]; + try { + entries = await fsp.readdir(root, { withFileTypes: true }); + } catch (cause) { + if (isErrnoNotFound(cause)) return { reapedEntries: [] }; + throw cause; + } + + const reapedEntries: string[] = []; + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const entryPath = path.join(root, entry.name); + const markerPath = path.join(entryPath, MARKER_FILENAME); + + let snapshottedAtIso: string | undefined; + try { + snapshottedAtIso = await fsp.readFile(markerPath, "utf8"); + } catch (cause) { + if (!isErrnoNotFound(cause)) throw cause; + } + + const snapshottedAtMs = + snapshottedAtIso === undefined ? NaN : Date.parse(snapshottedAtIso); + const expired = Number.isNaN(snapshottedAtMs) + ? true + : nowMs - snapshottedAtMs >= retentionMs; + + if (expired) { + await fsp.rm(entryPath, { recursive: true, force: true }); + reapedEntries.push(entry.name); + } + } + return { reapedEntries }; +} diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index cea183a79..ba162410c 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -368,6 +368,13 @@ try { bootRestorePushHold.end(); } +// Independent of the deployment restore above: reclaims any +// hibernated-agent-identity snapshot (see +// `hibernated-agent-identity-vault.ts`) whose address hibernated and was +// never redeployed within the retention window, so a permanently +// abandoned hibernate does not leak disk forever. +await deployRouter.reapExpiredHibernationSnapshots(); + // The first connect bypasses the reconnect scheduler, so arm the stall // deadline by hand; the open path's getWorkflowAddresses disarms it. watchdog.armForBoot(); diff --git a/apps/sidecar/src/workflow-host-wiring/index.ts b/apps/sidecar/src/workflow-host-wiring/index.ts index 515181ccf..5402f6434 100644 --- a/apps/sidecar/src/workflow-host-wiring/index.ts +++ b/apps/sidecar/src/workflow-host-wiring/index.ts @@ -20,6 +20,7 @@ import type { DeployRouterResult, SessionManager, } from "@intx/hub-agent"; +import { reportError } from "@corbits/error-sink"; import { type DeriveStepAddress, type DispatchTimingMark, @@ -61,6 +62,11 @@ import { writeWorkflowDeploymentRecord, type WorkflowDeploymentRecord, } from "../workflow-deployment-record"; +import { + reapExpiredHibernationSnapshots as reapVaultSnapshots, + restoreAgentIdentity, + snapshotAgentIdentity, +} from "../hibernated-agent-identity-vault"; import { computeWireDefinitionHash, validateWorkflowProjection, @@ -191,6 +197,17 @@ export interface SidecarDeployRouter extends DeployRouter { * and is retried every boot for as long as the record exists. */ restoreWorkflowDeployments(): Promise; + /** + * Sweep this sidecar's hibernated-agent-identity vault + * (`hibernated-agent-identity-vault.ts`) for snapshots older than its + * stated retention window and delete them, returning how many were + * reaped for observability. Independent of `restoreWorkflowDeployments` + * -- an orphaned snapshot (its address hibernated, then permanently + * torn down without ever redeploying) has no relationship to that scan + * and this does not touch its concurrency. Intended to run once at + * boot, in either order relative to the restore scan. + */ + reapExpiredHibernationSnapshots(): Promise; /** * The workflow-substrate deployment addresses (`ins_dep_...`) this router * currently hosts a live supervisor for -- the set of addresses this @@ -811,6 +828,15 @@ export function createSidecarDeployRouter(deps: { }; const wired = createSidecarWorkflowSupervisor(wiredBaseConfig); + // Restore a hibernated identity directory, if this address has one + // vaulted, BEFORE the `loadOrGenerateKey` call below: it is a no-op + // for an address that was never hibernated (an ordinary fresh + // deploy), and otherwise puts the preserved keypair back on disk so + // that call loads it (`isNew: false`) instead of minting a new one. + const identityRestore = await (stepStateDataDir !== undefined + ? restoreAgentIdentity(stepStateDataDir, spec.agentAddress) + : Promise.resolve({ restored: false })); + // OUTBOUND half of mailbox ownership: register a signing key for // the deployment mail address on the host transport so the supervisor // signs the deployment's outbound mail. Every step -- single- or @@ -820,9 +846,25 @@ export function createSidecarDeployRouter(deps: { // `getTransportFor(senderAddress).send` throws "not registered". // Registration happens before `spawn()` so the address is live the // instant the first reply routes outbound. - const { keyPair } = await deps.keyStore.loadOrGenerateKey( - spec.agentAddress, - ); + const { keyPair, isNew: keyIsNew } = + await deps.keyStore.loadOrGenerateKey(spec.agentAddress); + // A restored vault entry that did not yield an existing on-disk key + // means the restore itself is broken (a corrupt or partial + // snapshot) -- the exact "wake silently rotates identity" failure + // this workaround exists to make impossible to ship unnoticed. + if (identityRestore.restored && keyIsNew) { + reportError( + new Error( + "restored a hibernated agent identity snapshot but " + + "loadOrGenerateKey still minted a fresh keypair; the " + + "restored snapshot was missing or corrupt key material", + ), + { + operation: "workflow-host-wiring.restoreAgentIdentity", + agentId: spec.agentAddress, + }, + ); + } deps.transport.register( spec.agentAddress, deps.createAgentCrypto(keyPair), @@ -1566,6 +1608,18 @@ export function createSidecarDeployRouter(deps: { agentAddress: string, opts: { reclaimDirs: boolean }, ): Promise { + // Snapshot the agent's identity directory (its reconnect-challenge + // keypair) BEFORE anything else, and before this function returns: + // the published `@intx/hub-agent` package's own undeploy handling + // destroys that directory unconditionally, but only AFTER this hook + // returns -- this is a compensating workaround for CL-6239 (still + // open: the real fix is a non-destructive upstream undeploy), not a + // substitute for it. Skipped for a reclaiming (non-hibernate) + // teardown, which is expected to destroy the identity along with + // everything else. + if (!opts.reclaimDirs && stepStateDataDir !== undefined) { + await snapshotAgentIdentity(stepStateDataDir, agentAddress); + } const deploymentId = deriveDeploymentId(agentAddress); deps.multistepMailRouter?.unregister(agentAddress); deps.multistepSignalRouter?.unregister(agentAddress); @@ -1745,6 +1799,14 @@ export function createSidecarDeployRouter(deps: { logger.warn`Skipped ${skippedQuarantinedCount} quarantined workflow deployment record(s) (permanent restore failures, already reported); undeploy an address to clear its record`; } }, + async reapExpiredHibernationSnapshots(): Promise { + if (stepStateDataDir === undefined) return 0; + const { reapedEntries } = await reapVaultSnapshots(stepStateDataDir); + if (reapedEntries.length > 0) { + logger.info`Reaped ${reapedEntries.length} expired hibernated-agent-identity snapshot(s)`; + } + return reapedEntries.length; + }, activeAddresses(): string[] { // `activeSupervisors` holds exactly the deployments with a live // supervisor -- the set this sidecar can currently route mail to.