From c6ac9481b9254ceda5488f590260a3c563bc6b96 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 22 Aug 2026 14:09:08 -0700 Subject: [PATCH 1/2] Vendor intx-types and intx-storage-isogit at upstream head Re-syncs vendor/intx-types and vendor/intx-storage-isogit from Interchange commit 55c4431e60cc97dae2f63bfd52de56166e42b13b (both were previously at ad0f99e7). vendor/intx-inference stays at ad0f99e7 for now (tracked separately); the shared PendingOperation shape did not change in this range so the two are still compatible. Both packages remain verbatim, no local patches. storage-isogit's index now requires a runtime-bound entry point (./node or ./browser) instead of a bare createIsogitStore export; the one first-party consumer switches to @intx/storage-isogit/node, which keeps the identical signature. Adds the package's two new upstream deps (@isomorphic-git/lightning-fs, buffer, both browser-only) and a 0.2.2-pinned @intx/crypto dev dependency, mirroring the existing @intx/mime pin on intx-inference. Excludes vendor/intx-storage-isogit/src/browser-bundle.test.ts via bunfig.toml: it bundles against an export condition that resolves @intx/log/@intx/mime to source files that only exist in upstream's own monorepo, not in our published-npm installs of those two packages. Full suite: 6042 pass, 1 pre-existing failure in vendor/intx-inference's reactor.test.ts unrelated to this sync (reproduces identically before these changes). Part of CL-5684; fixes CL-6826 https://linear.app/abklabs/issue/CL-6826 --- bun.lock | 17 + bunfig.toml | 8 + docs/VENDORING.md | 47 +- src/session/optimized-context-store.ts | 2 +- vendor/intx-storage-isogit/README.md | 68 ++- vendor/intx-storage-isogit/package.json | 13 + .../src/browser-bundle.test.ts | 25 ++ .../src/browser-path.test.ts | 34 ++ .../intx-storage-isogit/src/browser-path.ts | 55 +++ .../src/browser-runtime.ts | 39 ++ vendor/intx-storage-isogit/src/browser.ts | 23 + .../intx-storage-isogit/src/commit-helpers.ts | 92 ++++ .../src/gc-cache-transparency.test.ts | 4 +- vendor/intx-storage-isogit/src/gc.test.ts | 112 ++++- vendor/intx-storage-isogit/src/gc.ts | 237 +++++++--- vendor/intx-storage-isogit/src/history.ts | 37 +- vendor/intx-storage-isogit/src/index.ts | 139 +++--- vendor/intx-storage-isogit/src/init.test.ts | 2 +- vendor/intx-storage-isogit/src/init.ts | 52 ++- .../intx-storage-isogit/src/isogit-helpers.ts | 41 +- .../src/mail-store.test.ts | 230 +++++++++- vendor/intx-storage-isogit/src/mail-store.ts | 114 +++-- .../intx-storage-isogit/src/node-metrics.ts | 84 ++++ .../intx-storage-isogit/src/node-runtime.ts | 12 + vendor/intx-storage-isogit/src/node.ts | 79 ++++ vendor/intx-storage-isogit/src/object-walk.ts | 6 +- .../src/pack-receive.test.ts | 319 +++++++++++++- .../intx-storage-isogit/src/pack-receive.ts | 344 +++++++++------ .../intx-storage-isogit/src/pack-send.test.ts | 8 +- vendor/intx-storage-isogit/src/pack-send.ts | 28 +- .../src/peek-turns.test.ts | 2 +- vendor/intx-storage-isogit/src/repo-disk.ts | 109 +++-- vendor/intx-storage-isogit/src/repo-lock.ts | 5 +- .../intx-storage-isogit/src/runtime.test.ts | 192 +++++++++ vendor/intx-storage-isogit/src/runtime.ts | 282 ++++++++++++ vendor/intx-storage-isogit/src/store.test.ts | 238 ++++++++++- vendor/intx-storage-isogit/src/store.ts | 403 +++++++++++------- vendor/intx-types/README.md | 2 +- vendor/intx-types/package.json | 8 + vendor/intx-types/src/agent-address.test.ts | 52 +-- vendor/intx-types/src/agent-address.ts | 28 +- vendor/intx-types/src/approvals.ts | 4 +- vendor/intx-types/src/message-id.ts | 6 +- vendor/intx-types/src/package-json.test.ts | 83 ++++ vendor/intx-types/src/package-json.ts | 11 +- vendor/intx-types/src/sessions.ts | 6 +- vendor/intx-types/src/sidecar.test.ts | 12 +- vendor/intx-types/src/sidecar.ts | 240 ++++++----- .../src/wire-definition-hash.test.ts | 91 ++++ vendor/intx-types/src/wire-definition-hash.ts | 82 ++++ vendor/intx-types/src/wire-workflow.ts | 167 ++++++++ vendor/intx-types/src/workflow-run-id.test.ts | 33 ++ vendor/intx-types/src/workflow-run-id.ts | 38 +- .../intx-types/src/workflow-sources.test.ts | 66 +++ vendor/intx-types/src/workflow-sources.ts | 30 ++ 55 files changed, 3735 insertions(+), 726 deletions(-) create mode 100644 vendor/intx-storage-isogit/src/browser-bundle.test.ts create mode 100644 vendor/intx-storage-isogit/src/browser-path.test.ts create mode 100644 vendor/intx-storage-isogit/src/browser-path.ts create mode 100644 vendor/intx-storage-isogit/src/browser-runtime.ts create mode 100644 vendor/intx-storage-isogit/src/browser.ts create mode 100644 vendor/intx-storage-isogit/src/node-metrics.ts create mode 100644 vendor/intx-storage-isogit/src/node-runtime.ts create mode 100644 vendor/intx-storage-isogit/src/node.ts create mode 100644 vendor/intx-storage-isogit/src/runtime.test.ts create mode 100644 vendor/intx-storage-isogit/src/runtime.ts create mode 100644 vendor/intx-types/src/wire-definition-hash.test.ts create mode 100644 vendor/intx-types/src/wire-definition-hash.ts create mode 100644 vendor/intx-types/src/wire-workflow.ts create mode 100644 vendor/intx-types/src/workflow-run-id.test.ts create mode 100644 vendor/intx-types/src/workflow-sources.test.ts create mode 100644 vendor/intx-types/src/workflow-sources.ts diff --git a/bun.lock b/bun.lock index 321b4e47b..dbebe7ccc 100644 --- a/bun.lock +++ b/bun.lock @@ -70,9 +70,14 @@ "@intx/log": "0.2.2", "@intx/mime": "0.2.2", "@intx/types": "workspace:*", + "@isomorphic-git/lightning-fs": "4.7.0", "arktype": "catalog:", + "buffer": "^6.0.3", "isomorphic-git": "catalog:", }, + "devDependencies": { + "@intx/crypto": "0.2.2", + }, }, "vendor/intx-types": { "name": "@intx/types", @@ -223,6 +228,10 @@ "@intx/types": ["@intx/types@workspace:vendor/intx-types"], + "@isomorphic-git/idb-keyval": ["@isomorphic-git/idb-keyval@3.3.2", "", {}, "sha512-r8/AdpiS0/WJCNR/t/gsgL+M8NMVj/ek7s60uz3LmpCaTF2mEVlZJlB01ZzalgYzRLXwSPC92o+pdzjM7PN/pA=="], + + "@isomorphic-git/lightning-fs": ["@isomorphic-git/lightning-fs@4.7.0", "", { "dependencies": { "@isomorphic-git/idb-keyval": "3.3.2", "isomorphic-textencoder": "1.0.1", "just-debounce-it": "1.1.0", "just-once": "1.1.0" }, "bin": { "superblocktxt": "src/superblocktxt.js" } }, "sha512-eJg541itXKCOyj3DBAnd0KNY1mNgi29GCpy7FgKWsgatu/ulEKv+dMxxjhUNL68Jh3uPTaGNfeAjdJiHUYEGnw=="], + "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="], "@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="], @@ -453,6 +462,8 @@ "fast-levenshtein": ["fast-levenshtein@2.0.6", "", {}, "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw=="], + "fast-text-encoding": ["fast-text-encoding@1.0.6", "", {}, "sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w=="], + "fast-uri": ["fast-uri@3.1.2", "", {}, "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ=="], "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], @@ -547,6 +558,8 @@ "isomorphic-git": ["isomorphic-git@1.38.3", "", { "dependencies": { "async-lock": "^1.4.1", "clean-git-ref": "^2.0.1", "crc-32": "^1.2.0", "diff3": "0.0.3", "ignore": "^5.1.4", "minimisted": "^2.0.0", "pako": "^1.0.10", "pify": "^4.0.1", "readable-stream": "^4.0.0", "sha.js": "^2.4.12", "simple-get": "^4.0.1" }, "bin": { "isogit": "cli.cjs" } }, "sha512-rOpt0yIW9HD1o6mA4w2f4XP5Lj42QnccbFbhzkby6L+t9c2klQxf9seqyrw5x8JcWWnbuPuN7v7YJ7yvHj9OPA=="], + "isomorphic-textencoder": ["isomorphic-textencoder@1.0.1", "", { "dependencies": { "fast-text-encoding": "^1.0.0" } }, "sha512-676hESgHullDdHDsj469hr+7t3i/neBKU9J7q1T4RHaWwLAsaQnywC0D1dIUId0YZ+JtVrShzuBk1soo0+GVcQ=="], + "jose": ["jose@6.2.3", "", {}, "sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw=="], "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], @@ -565,6 +578,10 @@ "json5": ["json5@2.2.3", "", { "bin": { "json5": "lib/cli.js" } }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="], + "just-debounce-it": ["just-debounce-it@1.1.0", "", {}, "sha512-87Nnc0qZKgBZuhFZjYVjSraic0x7zwjhaTMrCKlj0QYKH6lh0KbFzVnfu6LHan03NO7J8ygjeBeD0epejn5Zcg=="], + + "just-once": ["just-once@1.1.0", "", {}, "sha512-+rZVpl+6VyTilK7vB/svlMPil4pxqIJZkbnN7DKZTOzyXfun6ZiFeq2Pk4EtCEHZ0VU4EkdFzG8ZK5F3PErcDw=="], + "keyv": ["keyv@4.5.4", "", { "dependencies": { "json-buffer": "3.0.1" } }, "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw=="], "levn": ["levn@0.4.1", "", { "dependencies": { "prelude-ls": "^1.2.1", "type-check": "~0.4.0" } }, "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ=="], diff --git a/bunfig.toml b/bunfig.toml index 81851a47a..191b0ade5 100644 --- a/bunfig.toml +++ b/bunfig.toml @@ -4,4 +4,12 @@ pathIgnorePatterns = [ "tests/fixtures/**", "eval/tasks/**", "tmp/**", + # Bundles vendor/intx-storage-isogit's browser entry point with Bun.build, + # under the "intx-src" export condition that resolves @intx/log and + # @intx/mime to ./src/*.ts. Upstream's own monorepo vendors those two + # packages too, so the condition finds real source; here they stay on + # published npm (dist-only, no src/), so the condition matches an export + # key whose target file doesn't exist. Not a vendored-code defect — see + # docs/VENDORING.md. + "vendor/intx-storage-isogit/src/browser-bundle.test.ts", ] diff --git a/docs/VENDORING.md b/docs/VENDORING.md index 7b2f028ed..021537d11 100644 --- a/docs/VENDORING.md +++ b/docs/VENDORING.md @@ -24,8 +24,21 @@ points straight at `./src/*.ts` files rather than a `dist/` build. | Package | Vendor path | License | Synced from upstream commit | Retrieved | Local patches | |---|---|---|---|---|---| | `@intx/inference` | `vendor/intx-inference/` | LGPL-2.1-only | `ad0f99e7977b3ad4f28d8cc8d446ac52a4a2d685` | 2026-08-10 | Yes — see `vendor/intx-inference/PATCHES.md` | -| `@intx/types` | `vendor/intx-types/` | LGPL-2.1-only | `ad0f99e7977b3ad4f28d8cc8d446ac52a4a2d685` | 2026-08-10 | None — verbatim | -| `@intx/storage-isogit` | `vendor/intx-storage-isogit/` | LGPL-2.1-only | `ad0f99e7977b3ad4f28d8cc8d446ac52a4a2d685` | 2026-08-10 | None — verbatim | +| `@intx/types` | `vendor/intx-types/` | LGPL-2.1-only | `55c4431e60cc97dae2f63bfd52de56166e42b13b` | 2026-08-22 | None — verbatim | +| `@intx/storage-isogit` | `vendor/intx-storage-isogit/` | LGPL-2.1-only | `55c4431e60cc97dae2f63bfd52de56166e42b13b` | 2026-08-22 | None — verbatim | + +`@intx/inference` is temporarily behind `@intx/types` and +`@intx/storage-isogit` as of the 2026-08-22 sync: this is stage 2 of a +staged re-vendor (the two lower-coupling packages first), with +`@intx/inference`'s re-sync tracked separately as stage 3. The +cross-package coupling described below did not regress in this gap — the +`PendingOperation` shape both packages already shared did not change +between `ad0f99e7` and `55c4431e` (`vendor/intx-types/src/runtime.ts` and +`src/index.ts` are byte-identical across that range), so `@intx/inference` +built against the older `@intx/types` still typechecks against the newer +one. A later re-sync of `@intx/inference` should still move all three +together per the coupling rule below, since that check only holds for +this specific gap, not in general. The license column records what each package declares in its own `package.json`; the corresponding `LICENSE` file travels with every vendored @@ -114,6 +127,36 @@ those packages now resolves to the single root instance. As of this sync, no modifications. A diff against any later upstream checkout at the same paths will show 100% upstream-authored lines. +## Notable upstream shape changes carried by the 2026-08-22 sync + +Upstream commit `5f798bea` ("Inject the runtime into iso-git storage") +split `@intx/storage-isogit`'s entry point: the package root (`.`) no +longer exports a bindable `createIsogitStore` free function, and callers +now pick a runtime-bound entry point instead — `./node` (backed by +`createNodeIsogitRuntime`) or `./browser`. `vendor/intx-storage-isogit/`'s +`package.json` `exports` map picks up both new subpaths verbatim from +upstream. The one first-party consumer, +`src/session/optimized-context-store.ts`, now imports `createIsogitStore` +from `@intx/storage-isogit/node` instead of the package root — the +`./node` re-export has the identical `(dir, signer?, gcPolicy?)` signature +the old root export had, so this is an import-path change, not a +behavioral one. The package also picked up two new dependencies +(`@isomorphic-git/lightning-fs`, `buffer`, both used only by the new +`./browser` runtime, which nothing here imports) and a new +`@intx/crypto` dev dependency for its own test suite, pinned to `0.2.2` +like the existing `@intx/mime` dev dependency on `@intx/inference` — the +same "stay on published npm for a package we don't vendor" pattern. + +One new upstream test, `browser-bundle.test.ts`, is excluded via +`bunfig.toml`'s `pathIgnorePatterns`. It bundles `browser.ts` with +`Bun.build` under the `intx-src` export condition, which resolves +`@intx/log` and `@intx/mime` to `./src/*.ts` — real files in upstream's +own monorepo, where those two are also vendored source. Here they remain +published npm installs (`dist/` only, no `src/`), so the condition +matches an export key whose target does not exist and the bundle fails +to resolve. This is an environment gap, not a defect in the vendored +code; re-check it whenever `@intx/log` or `@intx/mime` get vendored too. + `@intx/inference` carries local patches — real fixes not yet present upstream, not workarounds for something upstream has since fixed. Every patched location carries a one-line comment naming its site-specific entry diff --git a/src/session/optimized-context-store.ts b/src/session/optimized-context-store.ts index e9db1d7fa..a63a17391 100644 --- a/src/session/optimized-context-store.ts +++ b/src/session/optimized-context-store.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import { type } from "arktype"; -import { createIsogitStore } from "@intx/storage-isogit"; +import { createIsogitStore } from "@intx/storage-isogit/node"; import { ContentBlock, type ConnectorThreadState, diff --git a/vendor/intx-storage-isogit/README.md b/vendor/intx-storage-isogit/README.md index 04998ef47..29e94f854 100644 --- a/vendor/intx-storage-isogit/README.md +++ b/vendor/intx-storage-isogit/README.md @@ -1,7 +1,7 @@ # @intx/storage-isogit Isomorphic-git backed implementation of `ContextStore` and -`AuditStore`. Each agent gets its own git repository on disk; +`AuditStore`. Each agent gets its own git repository in the host filesystem; inference state lives on a working branch, the tool-authorization audit log lives on its own branch, and mail history lives in a dedicated audit store that commits each inbound and outbound @@ -12,7 +12,7 @@ Consumed by `@intx/agent` for in-process persistence, and by repositories that move between the hub and the sidecar as packs. ```ts -import { createIsogitStore } from "@intx/storage-isogit"; +import { createIsogitStore } from "@intx/storage-isogit/node"; const store = await createIsogitStore("./tmp/agent-repo", signer); @@ -20,9 +20,73 @@ const store = await createIsogitStore("./tmp/agent-repo", signer); // the inference and tool layers as appropriate. ``` +Node and Bun consumers use the `/node` entry point above. The package root is +runtime-neutral and exports `createIsogitStorage(runtime)` for hosts that +provide their own filesystem: + +```ts +import type { FsClient } from "isomorphic-git"; +import { createIsogitStorage } from "@intx/storage-isogit"; + +type IsogitRuntime = { + fs: FsClient; + rename(oldPath: string, newPath: string): Promise; + path: { + join(...parts: string[]): string; + relative(from: string, to: string): string; + resolve(filepath: string): string; + }; + flush?: () => Promise; +}; +``` + +The storage layer derives its other filesystem operations from `FsClient`, +including recursive directory creation and removal, existence checks, and +UTF-8 text reads. `rename` remains explicit because completed packs require an +atomic publish primitive. Persistent backends can implement `flush` to make +completed mutations durable before the API call resolves. + +Audit and error record paths are append-only. Repeating a byte-identical +record is an idempotent retry, including after an uncertain persistence +failure; reusing the same path for different content is rejected. Mail commits +reconcile an uncertain ref publication before the same store accepts another +message, so a committed message cannot have its ordinal reused. + +Browsers can use the included LightningFS IndexedDB adapter: + +```ts +import { createBrowserIsogitStorage } from "@intx/storage-isogit/browser"; + +const storage = createBrowserIsogitStorage("agent-storage"); +const store = await storage.createIsogitStore("/agents/example"); +``` + +The browser adapter supplies isomorphic-git's Buffer compatibility, atomic +rename, POSIX path operations, and IndexedDB-backed file storage. Each call to +`createBrowserIsogitStorage(name)` claims a fresh volume and clears any prior +contents for that name. Call it exactly once per name during a page or worker +lifetime, then pass the returned storage object to every consumer. Reusing the +same name from another factory call, bundle, tab, or worker is unsupported and +may clear active data. + +Browser storage is intentionally disposable: a page reload, tab close, or +worker restart ends the logical session, and the next owner starts from an +empty volume instead of attempting recovery. `flush()` writes the current +filesystem state to IndexedDB, but does not promise that state will survive a +new owner. IndexedDB is the storage medium, not a restart-persistence +guarantee. Use absolute repository paths; relative browser paths resolve from +`/`. Applications that require restart persistence must inject a durable +runtime through the package root and own that runtime's recovery guarantees. + The pack-send and pack-receive helpers (`createDeployPack`, `createNegotiatedPack`, `applyPack`, `receivePackObjects`) produce and consume the wire bytes that `@intx/pack-transport` chunks across the WebSocket. A `CommitSigner` is optional but required when the consumer needs every commit to carry a verifiable signature. + +Pack receivers flush an accepted pack before checkout or ref promotion. If a +later ref or persistence operation fails, the call rejects but retains the +pack because the ref update may already be observable. Callers must treat that +error as an uncertain promotion outcome rather than assuming the ref is +unchanged; whichever ref value is visible remains backed by durable objects. diff --git a/vendor/intx-storage-isogit/package.json b/vendor/intx-storage-isogit/package.json index 738200a11..38030fbc0 100644 --- a/vendor/intx-storage-isogit/package.json +++ b/vendor/intx-storage-isogit/package.json @@ -7,13 +7,26 @@ ".": { "types": "./src/index.ts", "default": "./src/index.ts" + }, + "./node": { + "types": "./src/node.ts", + "default": "./src/node.ts" + }, + "./browser": { + "types": "./src/browser.ts", + "default": "./src/browser.ts" } }, "dependencies": { "@intx/log": "0.2.2", "@intx/mime": "0.2.2", "@intx/types": "workspace:*", + "@isomorphic-git/lightning-fs": "4.7.0", "arktype": "catalog:", + "buffer": "^6.0.3", "isomorphic-git": "catalog:" + }, + "devDependencies": { + "@intx/crypto": "0.2.2" } } diff --git a/vendor/intx-storage-isogit/src/browser-bundle.test.ts b/vendor/intx-storage-isogit/src/browser-bundle.test.ts new file mode 100644 index 000000000..b98fb84be --- /dev/null +++ b/vendor/intx-storage-isogit/src/browser-bundle.test.ts @@ -0,0 +1,25 @@ +import { expect, test } from "bun:test"; +import path from "node:path"; + +test("browser entry bundles without Node filesystem or path modules", async () => { + const result = await Bun.build({ + entrypoints: [path.join(import.meta.dir, "browser.ts")], + target: "browser", + format: "esm", + conditions: ["intx-src"], + }); + if (!result.success) { + throw new Error( + result.logs + .map((log) => (log instanceof Error ? log.message : String(log))) + .join("\n"), + ); + } + const output = result.outputs[0]; + if (output === undefined) { + throw new Error("Browser storage bundle produced no output"); + } + const bundle = await output.text(); + expect(bundle).not.toContain('from "node:fs"'); + expect(bundle).not.toContain('from "node:path"'); +}); diff --git a/vendor/intx-storage-isogit/src/browser-path.test.ts b/vendor/intx-storage-isogit/src/browser-path.test.ts new file mode 100644 index 000000000..833b84c67 --- /dev/null +++ b/vendor/intx-storage-isogit/src/browser-path.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, test } from "bun:test"; +import path from "node:path"; + +import { browserPath } from "./browser-path"; + +describe("browserPath", () => { + test.each([ + [[], "."], + [["/repo", ".git", "objects"], "/repo/.git/objects"], + [["repo", "state", "..", "audit"], "repo/audit"], + [["/", "repo", "//state"], "/repo/state"], + ] as const)("join(%j)", (parts, expected) => { + expect(browserPath.join(...parts)).toBe(expected); + expect(browserPath.join(...parts)).toBe(path.posix.join(...parts)); + }); + + test.each([ + ["/repo", "/repo/.git/objects", ".git/objects"], + ["/repo/state", "/repo/audit", "../audit"], + ["/repo", "/repo", ""], + ["repo", "other/file", "../other/file"], + ])("relative(%s, %s)", (from, to, expected) => { + expect(browserPath.relative(from, to)).toBe(expected); + }); + + test.each([ + ["/repo/../state", "/state"], + ["repo/./state", "/repo/state"], + ["/../../repo", "/repo"], + ["", "/"], + ])("resolve(%s)", (filepath, expected) => { + expect(browserPath.resolve(filepath)).toBe(expected); + }); +}); diff --git a/vendor/intx-storage-isogit/src/browser-path.ts b/vendor/intx-storage-isogit/src/browser-path.ts new file mode 100644 index 000000000..404df24b8 --- /dev/null +++ b/vendor/intx-storage-isogit/src/browser-path.ts @@ -0,0 +1,55 @@ +import type { IsogitPath } from "./runtime"; + +function normalize(filepath: string, absolute: boolean): string { + const segments: string[] = []; + for (const segment of filepath.split("/")) { + if (segment === "" || segment === ".") { + continue; + } + if (segment === "..") { + if (segments.length > 0 && segments.at(-1) !== "..") { + segments.pop(); + } else if (!absolute) { + segments.push(segment); + } + continue; + } + segments.push(segment); + } + + if (absolute) { + return segments.length === 0 ? "/" : `/${segments.join("/")}`; + } + return segments.length === 0 ? "." : segments.join("/"); +} + +function resolve(filepath: string): string { + return normalize(filepath.startsWith("/") ? filepath : `/${filepath}`, true); +} + +function join(...parts: string[]): string { + if (parts.length === 0) { + return "."; + } + const combined = parts.join("/"); + return normalize(combined, combined.startsWith("/")); +} + +function relative(from: string, to: string): string { + const fromSegments = resolve(from).split("/").filter(Boolean); + const toSegments = resolve(to).split("/").filter(Boolean); + let shared = 0; + while ( + shared < fromSegments.length && + shared < toSegments.length && + fromSegments[shared] === toSegments[shared] + ) { + shared += 1; + } + return [ + ...fromSegments.slice(shared).map(() => ".."), + ...toSegments.slice(shared), + ].join("/"); +} + +export const browserPath: IsogitPath = { join, relative, resolve }; diff --git a/vendor/intx-storage-isogit/src/browser-runtime.ts b/vendor/intx-storage-isogit/src/browser-runtime.ts new file mode 100644 index 000000000..fea9fd803 --- /dev/null +++ b/vendor/intx-storage-isogit/src/browser-runtime.ts @@ -0,0 +1,39 @@ +import FS from "@isomorphic-git/lightning-fs"; +import { Buffer as BrowserBuffer } from "buffer"; + +import { browserPath } from "./browser-path"; +import type { IsogitRuntime } from "./runtime"; + +export type BrowserIsogitRuntime = { + runtime: IsogitRuntime & { flush(): Promise }; + fs: FS; +}; + +function ensureBuffer(): void { + if (globalThis.Buffer === undefined) { + globalThis.Buffer = BrowserBuffer; + } +} + +/** + * Claim a fresh, single-owner IndexedDB filesystem for this page. + * + * Construction clears any prior contents for `name`. Create one instance per + * name and pass it to every consumer; a second owner for the same name may + * clear the first owner's active data. + */ +export function createBrowserIsogitRuntime( + name = "interchange", +): BrowserIsogitRuntime { + ensureBuffer(); + const fs = new FS(name, { wipe: true }); + return { + runtime: { + fs, + path: browserPath, + rename: (oldPath, newPath) => fs.promises.rename(oldPath, newPath), + flush: () => fs.promises.flush(), + }, + fs, + }; +} diff --git a/vendor/intx-storage-isogit/src/browser.ts b/vendor/intx-storage-isogit/src/browser.ts new file mode 100644 index 000000000..86062d76d --- /dev/null +++ b/vendor/intx-storage-isogit/src/browser.ts @@ -0,0 +1,23 @@ +import { createIsogitStorage } from "./index"; +import { createBrowserIsogitRuntime } from "./browser-runtime"; + +export * from "./index"; +export { + createBrowserIsogitRuntime, + type BrowserIsogitRuntime, +} from "./browser-runtime"; + +/** + * Claim a fresh, single-owner LightningFS IndexedDB storage API. + * + * Construction clears prior contents for `name`. Create this once per name + * and pass the returned API to every consumer in the page or worker. + */ +export function createBrowserIsogitStorage(name = "interchange") { + const { runtime, fs } = createBrowserIsogitRuntime(name); + return { + ...createIsogitStorage(runtime), + runtime, + fs, + }; +} diff --git a/vendor/intx-storage-isogit/src/commit-helpers.ts b/vendor/intx-storage-isogit/src/commit-helpers.ts index b837c7465..44413b34f 100644 --- a/vendor/intx-storage-isogit/src/commit-helpers.ts +++ b/vendor/intx-storage-isogit/src/commit-helpers.ts @@ -1,4 +1,7 @@ +import git from "isomorphic-git"; + import type { CommitSigner } from "./signer"; +import { flushRuntime, type StorageRuntime } from "./runtime"; export type SigningArgs = { onSign?: (args: { payload: string }) => Promise<{ signature: string }>; @@ -14,3 +17,92 @@ export function buildSigningArgs( onSign: async ({ payload }) => ({ signature: await signer(payload) }), }; } + +type GitCommitArgs = Parameters[0]; + +type DurableCommitArgs = { + message: string; + author: NonNullable; +} & Pick; + +/** A ref write was attempted, but its publication outcome is uncertain. */ +export class UncertainRefPublicationError extends Error { + readonly oid: string; + readonly ref: string; + + constructor(ref: string, oid: string, cause: unknown) { + const detail = cause instanceof Error ? cause.message : String(cause); + super(`Ref publication for ${ref} at ${oid} is uncertain: ${detail}`, { + cause, + }); + this.name = "UncertainRefPublicationError"; + this.oid = oid; + this.ref = ref; + } +} + +/** Restore selected index entries to the currently visible HEAD. */ +export async function restoreIndexAfterFailedCommit( + runtime: StorageRuntime, + dir: string, + filepaths: readonly string[], +): Promise { + const failures: unknown[] = []; + for (const filepath of filepaths) { + try { + await git.resetIndex({ fs: runtime.fs.git, dir, filepath }); + } catch (cause) { + failures.push(cause); + } + } + try { + await flushRuntime(runtime); + } catch (cause) { + failures.push(cause); + } + if (failures.length > 0) { + throw new AggregateError(failures, "Could not restore the Git index"); + } +} + +/** + * Create a commit without exposing its ref until the new objects are durable. + * + * LightningFS persists existing file bodies before its directory namespace. + * Updating a live ref in the same phase that creates loose-object paths can + * therefore leave a durable ref pointing at an absent object after a reload. + * The first flush publishes the object namespace; only then is the ref moved. + */ +export async function commitDurably( + runtime: StorageRuntime, + dir: string, + args: DurableCommitArgs, +): Promise { + const branch = await git.currentBranch({ + fs: runtime.fs.git, + dir, + fullname: true, + }); + const oid = await git.commit({ + fs: runtime.fs.git, + dir, + ...args, + noUpdateBranch: true, + }); + + await flushRuntime(runtime); + const ref = branch ?? "HEAD"; + try { + await git.writeRef({ + fs: runtime.fs.git, + dir, + ref, + value: oid, + force: true, + }); + await flushRuntime(runtime); + } catch (cause) { + throw new UncertainRefPublicationError(ref, oid, cause); + } + return oid; +} diff --git a/vendor/intx-storage-isogit/src/gc-cache-transparency.test.ts b/vendor/intx-storage-isogit/src/gc-cache-transparency.test.ts index 321f26733..d1c9c5f63 100644 --- a/vendor/intx-storage-isogit/src/gc-cache-transparency.test.ts +++ b/vendor/intx-storage-isogit/src/gc-cache-transparency.test.ts @@ -13,8 +13,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import git from "isomorphic-git"; -import { initAgentRepo } from "./init"; -import { runGC } from "./gc"; +import { initAgentRepo } from "./node"; +import { runGC } from "./node"; const author = { name: "Test", email: "test@test.dev" }; const tempDirs: string[] = []; diff --git a/vendor/intx-storage-isogit/src/gc.test.ts b/vendor/intx-storage-isogit/src/gc.test.ts index 80ec992a0..b9011474a 100644 --- a/vendor/intx-storage-isogit/src/gc.test.ts +++ b/vendor/intx-storage-isogit/src/gc.test.ts @@ -3,17 +3,22 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import git from "isomorphic-git"; -import { initAgentRepo } from "./init"; +import { initAgentRepo } from "./node"; import { applyPack, receivePackObjects, type CommitVerifier, type TreeValidator, -} from "./pack-receive"; -import { collectReachableObjects } from "./object-walk"; -import { repoDiskUsage } from "./repo-disk"; -import { runGC } from "./gc"; -import { IsogitStore } from "./store"; +} from "./node"; +import { collectReachableObjects } from "./node"; +import { + createIsogitStorage, + createNodeIsogitRuntime, + type IsogitRuntime, +} from "./node"; +import { repoDiskUsage } from "./node"; +import { runGC } from "./node"; +import { IsogitStore } from "./node"; const author = { name: "Test", email: "test@test.dev" }; @@ -91,6 +96,23 @@ async function deployOnto( return source.tip; } +function runtimeFailingOnFlush(failOn: number): IsogitRuntime { + const runtime = createNodeIsogitRuntime(); + let flushCount = 0; + return { + ...runtime, + flush: () => { + flushCount += 1; + if (flushCount === failOn) { + return Promise.reject( + new Error(`injected flush ${flushCount.toString()} failure`), + ); + } + return Promise.resolve(); + }, + }; +} + describe("runGC", () => { test("reclaims disk from superseded packs and loose objects across both refs", async () => { const dir = await tempDir(); @@ -301,4 +323,82 @@ describe("runGC", () => { expect(after[0]?.hash).toBe(before[0]?.hash); await store.readManifestHistory(10); }); + + test("keeps superseded object bodies quarantined when the commit flush fails", async () => { + const dir = await tempDir(); + await initAgentRepo(dir); + const mainTip = await commitOnMain(dir, "main-state.txt"); + const deployTip = await deployOnto( + dir, + { "deploy/prompt.txt": "deploy content" }, + "deploy-1", + ); + const before = repoDiskUsage(dir); + expect(before.packCount).toBeGreaterThan(0); + expect(before.looseObjectCount).toBeGreaterThan(0); + + const failingStorage = createIsogitStorage(runtimeFailingOnFlush(1)); + await expect( + failingStorage.runGC(dir, { retention: "keep-history" }), + ).rejects.toThrow("injected flush 1 failure"); + + const trashRoot = path.join(dir, ".git", "objects", "gc-trash"); + const trashRuns = await fs.promises.readdir(trashRoot); + expect(trashRuns).toHaveLength(1); + const trashRun = trashRuns[0]; + if (trashRun === undefined) throw new Error("missing GC quarantine"); + const retiredPacks = await fs.promises.readdir( + path.join(trashRoot, trashRun, "pack"), + ); + const retiredLooseDirectories = await fs.promises.readdir( + path.join(trashRoot, trashRun, "loose"), + ); + expect(retiredPacks.some((name) => name.endsWith(".pack"))).toBe(true); + expect(retiredPacks.some((name) => name.endsWith(".idx"))).toBe(true); + expect(retiredLooseDirectories.length).toBeGreaterThan(0); + + expect((await git.readCommit({ fs, dir, oid: mainTip })).oid).toBe(mainTip); + expect((await git.readCommit({ fs, dir, oid: deployTip })).oid).toBe( + deployTip, + ); + + // A later GC first removes the abandoned quarantine, durably flushes + // that cleanup, and then performs its own consolidation. + await runGC(dir, { retention: "keep-history" }); + expect(await fs.promises.readdir(trashRoot)).toEqual([]); + expect((await git.readCommit({ fs, dir, oid: mainTip })).oid).toBe(mainTip); + expect((await git.readCommit({ fs, dir, oid: deployTip })).oid).toBe( + deployTip, + ); + }); + + test("keeps refs readable when the quarantine cleanup flush fails", async () => { + const dir = await tempDir(); + await initAgentRepo(dir); + const mainTip = await commitOnMain(dir, "main-state.txt"); + const deployTip = await deployOnto( + dir, + { "deploy/prompt.txt": "deploy content" }, + "deploy-1", + ); + + const failingStorage = createIsogitStorage(runtimeFailingOnFlush(2)); + await expect( + failingStorage.runGC(dir, { retention: "keep-history" }), + ).rejects.toThrow("injected flush 2 failure"); + + expect((await git.readCommit({ fs, dir, oid: mainTip })).oid).toBe(mainTip); + expect((await git.readCommit({ fs, dir, oid: deployTip })).oid).toBe( + deployTip, + ); + expect( + await fs.promises.readdir(path.join(dir, ".git", "objects", "gc-trash")), + ).toEqual([]); + + await runGC(dir, { retention: "keep-history" }); + expect((await git.readCommit({ fs, dir, oid: mainTip })).oid).toBe(mainTip); + expect((await git.readCommit({ fs, dir, oid: deployTip })).oid).toBe( + deployTip, + ); + }); }); diff --git a/vendor/intx-storage-isogit/src/gc.ts b/vendor/intx-storage-isogit/src/gc.ts index e5d3d4d91..b8ce7f67e 100644 --- a/vendor/intx-storage-isogit/src/gc.ts +++ b/vendor/intx-storage-isogit/src/gc.ts @@ -1,7 +1,6 @@ -import fs from "node:fs"; -import path from "node:path"; import git from "isomorphic-git"; import { getLogger } from "@intx/log"; +import { hasCode } from "@intx/types"; import { collectReachableObjects } from "./object-walk"; import { publishPackAtomically } from "./pack-receive"; import { @@ -12,8 +11,15 @@ import { type RepoDiskUsage, } from "./repo-disk"; import { withRepoDirLock } from "./repo-lock"; +import { flushRuntime, type StorageRuntime } from "./runtime"; const logger = getLogger(["interchange", "storage-isogit", "gc"]); +const GC_TRASH_DIR = "gc-trash"; + +type ObjectStoreEntry = { + name: string; + filepath: string; +}; /** * How much commit history a GC pass preserves. @@ -67,6 +73,7 @@ export type GCPolicy = { * would then delete it. */ async function collectHistoryObjects( + runtime: StorageRuntime, dir: string, tipOid: string, ): Promise> { @@ -82,7 +89,11 @@ async function collectHistoryObjects( let parents: string[]; try { - const { commit } = await git.readCommit({ fs, dir, oid: commitOid }); + const { commit } = await git.readCommit({ + fs: runtime.fs.git, + dir, + oid: commitOid, + }); parents = commit.parent; } catch (err) { if (err instanceof Error && "code" in err && err.code === "NotFoundError") @@ -90,7 +101,7 @@ async function collectHistoryObjects( throw err; } - for (const oid of await collectReachableObjects(dir, commitOid)) { + for (const oid of await collectReachableObjects(runtime, dir, commitOid)) { objects.add(oid); } for (const parent of parents) { @@ -102,59 +113,130 @@ async function collectHistoryObjects( } /** - * Absolute paths of every `.pack` and `.idx` file currently in the repo's - * pack directory. Snapshotted before the consolidated pack is published so - * the freshly published pair is never in the removal set. + * Every `.pack` and `.idx` file currently in the repo's pack directory. + * Snapshotted before the consolidated pack is published so the freshly + * published pair is never in the retirement set. */ -function listPackFiles(dir: string): string[] { - const packDir = path.join(dir, ".git", "objects", "pack"); +async function listPackFiles( + runtime: StorageRuntime, + dir: string, +): Promise { + const packDir = runtime.path.join(dir, ".git", "objects", "pack"); let entries: string[]; try { - entries = fs.readdirSync(packDir); + entries = await runtime.fs.readdir(packDir); } catch (cause) { - if ( - cause instanceof Error && - (cause as NodeJS.ErrnoException).code === "ENOENT" - ) { + if (hasCode(cause) && cause.code === "ENOENT") { return []; } throw cause; } return entries .filter((name) => name.endsWith(".pack") || name.endsWith(".idx")) - .map((name) => path.join(packDir, name)); + .map((name) => ({ + name, + filepath: runtime.path.join(packDir, name), + })); } /** - * Remove every loose object fan-out directory under `.git/objects/`. - * - * Safe only after the consolidated pack containing the entire keep set is - * published: every kept object that was loose is then also packed, and every - * loose object outside the keep set is garbage. The `pack` and `info` - * children are left untouched. + * Snapshot every loose-object fan-out directory under `.git/objects/`. + * Non-object children such as `pack`, `info`, and `gc-trash` are excluded by + * the two-lowercase-hex directory-name constraint. */ -async function removeLooseObjects(dir: string): Promise { - const objectsDir = path.join(dir, ".git", "objects"); +async function listLooseObjectDirectories( + runtime: StorageRuntime, + dir: string, +): Promise { + const objectsDir = runtime.path.join(dir, ".git", "objects"); let entries: string[]; try { - entries = fs.readdirSync(objectsDir); + entries = await runtime.fs.readdir(objectsDir); } catch (cause) { - if ( - cause instanceof Error && - (cause as NodeJS.ErrnoException).code === "ENOENT" - ) { - return; + if (hasCode(cause) && cause.code === "ENOENT") { + return []; } throw cause; } - for (const name of entries) { - if (name === "pack" || name === "info") continue; - if (!/^[0-9a-f]{2}$/.test(name)) continue; - await fs.promises.rm(path.join(objectsDir, name), { - recursive: true, - force: true, - }); + return entries + .filter((name) => /^[0-9a-f]{2}$/.test(name)) + .map((name) => ({ + name, + filepath: runtime.path.join(objectsDir, name), + })); +} + +function gcTrashRoot(runtime: StorageRuntime, dir: string): string { + return runtime.path.join(dir, ".git", "objects", GC_TRASH_DIR); +} + +/** + * Delete quarantine left by a prior GC that reached its durability commit + * point but did not finish cleanup. Quarantine is outside every path Git + * scans for packs or loose objects. If it is present in a durable namespace, + * the replacement pack was part of that same flushed namespace; a crash + * before the flush restores the prior namespace without quarantine instead. + */ +async function removeStaleGCTrash( + runtime: StorageRuntime, + dir: string, +): Promise { + const trashRoot = gcTrashRoot(runtime, dir); + let entries: string[]; + try { + entries = await runtime.fs.readdir(trashRoot); + } catch (cause) { + if (hasCode(cause) && cause.code === "ENOENT") return false; + throw cause; } + if (entries.length === 0) return false; + + await runtime.fs.remove(trashRoot, { recursive: true, force: true }); + return true; +} + +/** + * Retire the old object-store namespace without deleting object bodies. + * LightningFS persists file bodies eagerly but its directory namespace only + * at flush time. Renaming into an unscanned quarantine therefore leaves both + * crash outcomes valid: a pre-flush reopen sees the old namespace and a + * post-flush reopen sees the consolidated pack plus quarantined old bodies. + */ +async function quarantineSupersededObjects( + runtime: StorageRuntime, + dir: string, + transferId: string, + supersededPacks: readonly ObjectStoreEntry[], + looseObjectDirectories: readonly ObjectStoreEntry[], +): Promise { + const trashDir = runtime.path.join(gcTrashRoot(runtime, dir), transferId); + const packTrashDir = runtime.path.join(trashDir, "pack"); + const looseTrashDir = runtime.path.join(trashDir, "loose"); + await runtime.fs.mkdir(packTrashDir, { recursive: true }); + await runtime.fs.mkdir(looseTrashDir, { recursive: true }); + + const orderedPacks = [...supersededPacks].sort((left, right) => { + const leftRank = left.name.endsWith(".idx") ? 0 : 1; + const rightRank = right.name.endsWith(".idx") ? 0 : 1; + if (leftRank !== rightRank) return leftRank - rightRank; + if (left.name < right.name) return -1; + if (left.name > right.name) return 1; + return 0; + }); + for (const entry of orderedPacks) { + await runtime.fs.rename( + entry.filepath, + runtime.path.join(packTrashDir, entry.name), + ); + } + for (const entry of looseObjectDirectories) { + await runtime.fs.rename( + entry.filepath, + runtime.path.join(looseTrashDir, entry.name), + ); + } + + return trashDir; } /** @@ -168,8 +250,10 @@ async function removeLooseObjects(dir: string): Promise { * other's live objects). Pack the keep set into one self-contained pack via * `git.packObjects` and publish it through the same atomic staging dance * receives use, so a concurrent unlocked reader never observes a torn pack. - * Only then remove the packs that predated this pass and every loose object; - * each kept object is by then present in the consolidated pack. + * Only then rename the packs that predated this pass and every loose-object + * fan-out directory into an unscanned quarantine. A flush makes the new pack + * and retired namespace durable together; object bodies are deleted only + * after that commit point, followed by a second cleanup flush. * * # Concurrency * @@ -178,23 +262,28 @@ async function removeLooseObjects(dir: string): Promise { * inline after a commit/apply while still holding that lock, and external * callers not already under it use {@link runGC}, which acquires it. The * lock excludes concurrent writers, so the keep set computed from the refs - * cannot be invalidated by a commit landing mid-pass. Removal of a + * cannot be invalidated by a commit landing mid-pass. Retirement of a * superseded pack or loose object races only with unlocked readers, the same - * POSIX window `unpublishPack` already accepts — and strictly safer here, - * since every removed-but-reachable object is also in the freshly published - * consolidated pack. + * window `unpublishPack` already accepts. Indexes are retired before their + * packs, and every retired-but-reachable object is also in the freshly + * published consolidated pack. * * Returns the disk usage before and after plus the reclaimed byte delta. A - * repo with no resolvable refs is left untouched. Exported for use within - * the storage package only — it is intentionally absent from the package's - * public barrel. + * repo with no resolvable refs is not repacked, though stale quarantine is + * still reclaimed. Exported for use within the storage package only — it is + * intentionally absent from the package's public barrel. */ export async function gcUnderLock( + runtime: StorageRuntime, dir: string, opts: { retention: RetentionPolicy }, ): Promise { - const before = repoDiskUsage(dir); - const refs = await listRepoRefs(dir); + if (await removeStaleGCTrash(runtime, dir)) { + await flushRuntime(runtime); + } + + const before = await repoDiskUsage(runtime, dir); + const refs = await listRepoRefs(runtime, dir); if (refs.length === 0) { return { before, after: before, reclaimedBytes: 0, keptObjects: 0 }; @@ -204,15 +293,16 @@ export async function gcUnderLock( for (const { oid } of refs) { const reachable = opts.retention === "tip-only" - ? await collectReachableObjects(dir, oid) - : await collectHistoryObjects(dir, oid); + ? await collectReachableObjects(runtime, dir, oid) + : await collectHistoryObjects(runtime, dir, oid); for (const objectOid of reachable) keep.add(objectOid); } - const supersededPacks = listPackFiles(dir); + const supersededPacks = await listPackFiles(runtime, dir); + const looseObjectDirectories = await listLooseObjectDirectories(runtime, dir); const result = await git.packObjects({ - fs, + fs: runtime.fs.git, dir, oids: [...keep], write: false, @@ -223,14 +313,23 @@ export async function gcUnderLock( ); } const transferId = `gc-${crypto.randomUUID().replace(/-/g, "")}`; - await publishPackAtomically(dir, result.packfile, transferId); + await publishPackAtomically(runtime, dir, result.packfile, transferId); - for (const packFile of supersededPacks) { - await fs.promises.rm(packFile, { force: true }); - } - await removeLooseObjects(dir); + const trashDir = await quarantineSupersededObjects( + runtime, + dir, + transferId, + supersededPacks, + looseObjectDirectories, + ); - const after = repoDiskUsage(dir); + // Durability commit point: deletion cannot begin until the replacement + // pack is published and every retired body is reachable through quarantine. + await flushRuntime(runtime); + await runtime.fs.remove(trashDir, { recursive: true, force: true }); + await flushRuntime(runtime); + + const after = await repoDiskUsage(runtime, dir); return { before, after, @@ -247,10 +346,11 @@ export async function gcUnderLock( * while already under the lock call {@link gcUnderLock} directly. */ export async function runGC( + runtime: StorageRuntime, dir: string, opts: { retention: RetentionPolicy }, ): Promise { - return withRepoDirLock(dir, () => gcUnderLock(dir, opts)); + return withRepoDirLock(runtime, dir, () => gcUnderLock(runtime, dir, opts)); } function warnIfOverBudget(dir: string, bytes: number, warnBytes: number): void { @@ -279,10 +379,11 @@ function warnIfOverBudget(dir: string, bytes: number, warnBytes: number): void { * — the case where accumulation is most likely runaway. */ export async function maybeGCUnderLock( + runtime: StorageRuntime, dir: string, policy: GCPolicy, ): Promise { - const counts = repoObjectCounts(dir); + const counts = await repoObjectCounts(runtime, dir); if ( counts.packCount < policy.packThreshold && counts.looseObjectCount < policy.looseThreshold @@ -290,12 +391,14 @@ export async function maybeGCUnderLock( return; } try { - const result = await gcUnderLock(dir, { retention: policy.retention }); + const result = await gcUnderLock(runtime, dir, { + retention: policy.retention, + }); logger.info`reclaimed ${String(result.reclaimedBytes)} bytes from ${dir}: packs ${String(result.before.packCount)} to ${String(result.after.packCount)}, loose ${String(result.before.looseObjectCount)} to ${String(result.after.looseObjectCount)}`; warnIfOverBudget(dir, result.after.gitBytes, policy.warnBytes); } catch (err) { - logger.warn`GC of ${dir} failed; the repo is unchanged but unreclaimed — ${err instanceof Error ? err.message : String(err)}`; - warnIfOverBudget(dir, gitBytes(dir), policy.warnBytes); + logger.warn`GC of ${dir} failed; active refs remain intact, but garbage may remain quarantined — ${err instanceof Error ? err.message : String(err)}`; + warnIfOverBudget(dir, await gitBytes(runtime, dir), policy.warnBytes); } } @@ -305,6 +408,12 @@ export async function maybeGCUnderLock( * substrate uses this from inside its own higher-level lock; sidecar writers * that already hold the per-directory lock call `maybeGCUnderLock` directly. */ -export async function maybeGC(dir: string, policy: GCPolicy): Promise { - return withRepoDirLock(dir, () => maybeGCUnderLock(dir, policy)); +export async function maybeGC( + runtime: StorageRuntime, + dir: string, + policy: GCPolicy, +): Promise { + return withRepoDirLock(runtime, dir, () => + maybeGCUnderLock(runtime, dir, policy), + ); } diff --git a/vendor/intx-storage-isogit/src/history.ts b/vendor/intx-storage-isogit/src/history.ts index 0530ebdba..f6e6612ff 100644 --- a/vendor/intx-storage-isogit/src/history.ts +++ b/vendor/intx-storage-isogit/src/history.ts @@ -1,31 +1,44 @@ -import fs from "node:fs"; import git from "isomorphic-git"; import type { ContextCommit } from "@intx/types/runtime"; import { AUTHOR } from "./init"; +import { flushRuntime, type StorageRuntime } from "./runtime"; /** * Switch the working tree to the named branch. The branch must already exist. */ -export async function switchBranch(dir: string, ref: string): Promise { - await git.checkout({ fs, dir, ref }); +export async function switchBranch( + runtime: StorageRuntime, + dir: string, + ref: string, +): Promise { + await git.checkout({ fs: runtime.fs.git, dir, ref }); + await flushRuntime(runtime); } /** * Create a new branch at HEAD and immediately switch to it. */ export async function createAndSwitchBranch( + runtime: StorageRuntime, dir: string, name: string, ): Promise { - await git.branch({ fs, dir, ref: name }); - await git.checkout({ fs, dir, ref: name }); + await git.branch({ fs: runtime.fs.git, dir, ref: name }); + // Persist the new branch before checkout overwrites the already-durable + // HEAD body. A reload must never leave HEAD naming an absent branch. + await flushRuntime(runtime); + await git.checkout({ fs: runtime.fs.git, dir, ref: name }); + await flushRuntime(runtime); } /** * Return the name of the currently checked-out branch. */ -export async function currentBranch(dir: string): Promise { - const branch = await git.currentBranch({ fs, dir }); +export async function currentBranch( + runtime: StorageRuntime, + dir: string, +): Promise { + const branch = await git.currentBranch({ fs: runtime.fs.git, dir }); if (branch === null || branch === undefined) { throw new Error("Repository is in detached HEAD state"); } @@ -35,18 +48,22 @@ export async function currentBranch(dir: string): Promise { /** * List all local branches. */ -export async function listBranches(dir: string): Promise { - return git.listBranches({ fs, dir }); +export async function listBranches( + runtime: StorageRuntime, + dir: string, +): Promise { + return git.listBranches({ fs: runtime.fs.git, dir }); } /** * Return recent commits as ContextCommit entries. */ export async function logHistory( + runtime: StorageRuntime, dir: string, limit = 10, ): Promise { - const entries = await git.log({ fs, dir, depth: limit }); + const entries = await git.log({ fs: runtime.fs.git, dir, depth: limit }); return entries.map((e) => { const base = { hash: e.oid, diff --git a/vendor/intx-storage-isogit/src/index.ts b/vendor/intx-storage-isogit/src/index.ts index 653797f11..280b2a729 100644 --- a/vendor/intx-storage-isogit/src/index.ts +++ b/vendor/intx-storage-isogit/src/index.ts @@ -1,8 +1,31 @@ import type { ContextStore, AuditStore } from "@intx/types/runtime"; -import { initAgentRepo } from "./init"; +import { + createAndSwitchBranch, + currentBranch, + listBranches, + logHistory, + switchBranch, +} from "./history"; +import { initAgentRepo, initRepo } from "./init"; +import { applyPack, receivePackObjects } from "./pack-receive"; +import { createDeployPack, createNegotiatedPack } from "./pack-send"; +import { collectReachableObjects } from "./object-walk"; +import { + countLooseObjects, + countPackFiles, + gitBytes, + listRepoRefs, + repoDiskUsage, +} from "./repo-disk"; import { IsogitStore, type DurableMirrorReads } from "./store"; import type { CommitSigner } from "./signer"; -import type { GCPolicy } from "./gc"; +import { maybeGC, runGC, type GCPolicy } from "./gc"; +import { createMailAuditStore, listMail } from "./mail-store"; +import { + normalizeRuntime, + type IsogitRuntime, + type StorageRuntime, +} from "./runtime"; export type { ContextStore, AuditStore, CommitSigner }; export type { @@ -10,61 +33,65 @@ export type { TreeValidator, TreeValidatorResult, } from "./pack-receive"; -export { IsogitStore }; +export type { IncludeShaPredicate } from "./pack-send"; export type { DurableMirrorReads }; -export { - switchBranch, - createAndSwitchBranch, - currentBranch, - listBranches, - logHistory, -} from "./history"; -export { initRepo, initAgentRepo, type InitRepoOpts } from "./init"; -export { applyPack, receivePackObjects } from "./pack-receive"; -export { - createDeployPack, - createNegotiatedPack, - type IncludeShaPredicate, -} from "./pack-send"; -export { collectReachableObjects } from "./object-walk"; -export { - repoDiskUsage, - listRepoRefs, - gitBytes, - countLooseObjects, - countPackFiles, - type RepoDiskUsage, -} from "./repo-disk"; -export { - runGC, - maybeGC, - type RetentionPolicy, - type GCResult, - type GCPolicy, -} from "./gc"; -export { - createMailAuditStore, - listMail, - type MailAuditStore, - type MailCommitOptions, - type MailDirection, - type MailCommitResult, - type MailEntry, +export type { InitRepoOpts } from "./init"; +export type { RetentionPolicy, GCResult, GCPolicy } from "./gc"; +export type { RepoDiskUsage } from "./repo-disk"; +export type { + MailAuditStore, + MailCommitOptions, + MailCommitResult, + MailDirection, + MailEntry, } from "./mail-store"; +export type { IsogitPath, IsogitRuntime } from "./runtime"; + +function bindRepoDir( + runtime: StorageRuntime, + operation: (runtime: StorageRuntime, dir: string, ...args: TArgs) => TResult, +): (dir: string, ...args: TArgs) => TResult { + return (dir, ...args) => + operation(runtime, runtime.path.resolve(dir), ...args); +} -/** - * Initialize an agent repository at `dir` and return a store backed by that - * repository. The returned object implements both ContextStore (inference - * state) and AuditStore (tool authorization records). - * - * When `gcPolicy` is supplied, each commit reclaims the repo on the write - * path once it crosses the policy's thresholds. - */ -export async function createIsogitStore( - dir: string, - signer?: CommitSigner, - gcPolicy?: GCPolicy, -): Promise { - await initAgentRepo(dir); - return new IsogitStore(dir, signer, gcPolicy); +/** Bind the complete storage API to one filesystem runtime. */ +export function createIsogitStorage(runtime: IsogitRuntime) { + const storageRuntime = normalizeRuntime(runtime); + return { + storageRuntime, + createIsogitStore: async ( + dir: string, + signer?: CommitSigner, + gcPolicy?: GCPolicy, + ): Promise => { + const resolvedDir = storageRuntime.path.resolve(dir); + await initAgentRepo(storageRuntime, resolvedDir); + return new IsogitStore(storageRuntime, resolvedDir, signer, gcPolicy); + }, + initRepo: bindRepoDir(storageRuntime, initRepo), + initAgentRepo: bindRepoDir(storageRuntime, initAgentRepo), + switchBranch: bindRepoDir(storageRuntime, switchBranch), + createAndSwitchBranch: bindRepoDir(storageRuntime, createAndSwitchBranch), + currentBranch: bindRepoDir(storageRuntime, currentBranch), + listBranches: bindRepoDir(storageRuntime, listBranches), + logHistory: bindRepoDir(storageRuntime, logHistory), + applyPack: bindRepoDir(storageRuntime, applyPack), + receivePackObjects: bindRepoDir(storageRuntime, receivePackObjects), + createDeployPack: bindRepoDir(storageRuntime, createDeployPack), + createNegotiatedPack: bindRepoDir(storageRuntime, createNegotiatedPack), + collectReachableObjects: bindRepoDir( + storageRuntime, + collectReachableObjects, + ), + repoDiskUsage: bindRepoDir(storageRuntime, repoDiskUsage), + countLooseObjects: bindRepoDir(storageRuntime, countLooseObjects), + countPackFiles: bindRepoDir(storageRuntime, countPackFiles), + gitBytes: bindRepoDir(storageRuntime, gitBytes), + listRepoRefs: bindRepoDir(storageRuntime, listRepoRefs), + runGC: bindRepoDir(storageRuntime, runGC), + maybeGC: bindRepoDir(storageRuntime, maybeGC), + createMailAuditStore: bindRepoDir(storageRuntime, createMailAuditStore), + listMail: bindRepoDir(storageRuntime, listMail), + }; } diff --git a/vendor/intx-storage-isogit/src/init.test.ts b/vendor/intx-storage-isogit/src/init.test.ts index 3d4c8255d..2661554d5 100644 --- a/vendor/intx-storage-isogit/src/init.test.ts +++ b/vendor/intx-storage-isogit/src/init.test.ts @@ -8,7 +8,7 @@ import { createSSHSignature, verifySSHSignature, } from "@intx/crypto"; -import { initRepo } from "./init"; +import { initRepo } from "./node"; import type { CommitSigner } from "./signer"; const tempDirs: string[] = []; diff --git a/vendor/intx-storage-isogit/src/init.ts b/vendor/intx-storage-isogit/src/init.ts index 04e597b36..bc657cf87 100644 --- a/vendor/intx-storage-isogit/src/init.ts +++ b/vendor/intx-storage-isogit/src/init.ts @@ -1,8 +1,7 @@ -import fs from "node:fs"; -import path from "node:path"; import git from "isomorphic-git"; import type { CommitSigner } from "./signer"; import { buildSigningArgs } from "./commit-helpers"; +import { flushRuntime, type StorageRuntime } from "./runtime"; const AUTHOR = { name: "interchange-harness", @@ -16,9 +15,12 @@ const HUB_AUTHOR = { const DEFAULT_GITIGNORE = "keys/\n"; -async function isGitRepo(dir: string): Promise { - return fs.promises - .stat(path.join(dir, ".git")) +async function isGitRepo( + runtime: StorageRuntime, + dir: string, +): Promise { + return runtime.fs + .stat(runtime.path.join(dir, ".git")) .then(() => true) .catch(() => false); } @@ -49,27 +51,32 @@ export type InitRepoOpts = { * work, so the initial commit is always created. */ export async function initRepo( + runtime: StorageRuntime, dir: string, opts: InitRepoOpts = {}, ): Promise { - await fs.promises.mkdir(dir, { recursive: true }); + await runtime.fs.mkdir(dir, { recursive: true }); - if (await isGitRepo(dir)) return; + if (await isGitRepo(runtime, dir)) return; - await git.init({ fs, dir, defaultBranch: "main" }); + await git.init({ fs: runtime.fs.git, dir, defaultBranch: "main" }); const gitignoreBody = opts.gitignore ?? DEFAULT_GITIGNORE; - await fs.promises.writeFile(path.join(dir, ".gitignore"), gitignoreBody); - await git.add({ fs, dir, filepath: ".gitignore" }); + await runtime.fs.writeFile( + runtime.path.join(dir, ".gitignore"), + gitignoreBody, + ); + await git.add({ fs: runtime.fs.git, dir, filepath: ".gitignore" }); const author = opts.signer === undefined ? AUTHOR : HUB_AUTHOR; await git.commit({ - fs, + fs: runtime.fs.git, dir, message: "Initialize repository", author, ...buildSigningArgs(opts.signer), }); + await flushRuntime(runtime); } /** @@ -81,23 +88,30 @@ export async function initRepo( * * Idempotent: safe to call on a directory that already contains a git repo. */ -export async function initAgentRepo(dir: string): Promise { - await fs.promises.mkdir(dir, { recursive: true }); - await fs.promises.mkdir(path.join(dir, "state"), { recursive: true }); +export async function initAgentRepo( + runtime: StorageRuntime, + dir: string, +): Promise { + await runtime.fs.mkdir(dir, { recursive: true }); + await runtime.fs.mkdir(runtime.path.join(dir, "state"), { recursive: true }); - if (await isGitRepo(dir)) return; + if (await isGitRepo(runtime, dir)) { + await flushRuntime(runtime); + return; + } - await git.init({ fs, dir, defaultBranch: "main" }); + await git.init({ fs: runtime.fs.git, dir, defaultBranch: "main" }); - await fs.promises.writeFile(path.join(dir, ".gitignore"), "keys/\n"); - await git.add({ fs, dir, filepath: ".gitignore" }); + await runtime.fs.writeFile(runtime.path.join(dir, ".gitignore"), "keys/\n"); + await git.add({ fs: runtime.fs.git, dir, filepath: ".gitignore" }); await git.commit({ - fs, + fs: runtime.fs.git, dir, message: "Initialize agent repository", author: AUTHOR, }); + await flushRuntime(runtime); } export { AUTHOR, HUB_AUTHOR }; diff --git a/vendor/intx-storage-isogit/src/isogit-helpers.ts b/vendor/intx-storage-isogit/src/isogit-helpers.ts index 13e206ff2..e120a7af4 100644 --- a/vendor/intx-storage-isogit/src/isogit-helpers.ts +++ b/vendor/intx-storage-isogit/src/isogit-helpers.ts @@ -1,22 +1,49 @@ -import fs from "node:fs"; import git from "isomorphic-git"; import { type } from "arktype"; +import type { StorageRuntime } from "./runtime"; const CommitObject = type({ tree: "string" }); const TreeEntry = type({ oid: "string", type: "string" }); const RawObject = type({ object: type.instanceOf(Uint8Array) }); -export async function readCommitObject(dir: string, oid: string) { - const { object } = await git.readObject({ fs, dir, oid, format: "parsed" }); +export async function readCommitObject( + runtime: StorageRuntime, + dir: string, + oid: string, +) { + const { object } = await git.readObject({ + fs: runtime.fs.git, + dir, + oid, + format: "parsed", + }); return CommitObject.assert(object); } -export async function readTreeEntries(dir: string, oid: string) { - const { object } = await git.readObject({ fs, dir, oid, format: "parsed" }); +export async function readTreeEntries( + runtime: StorageRuntime, + dir: string, + oid: string, +) { + const { object } = await git.readObject({ + fs: runtime.fs.git, + dir, + oid, + format: "parsed", + }); return TreeEntry.array().assert(object); } -export async function readRawObject(dir: string, oid: string) { - const { object } = await git.readObject({ fs, dir, oid, format: "content" }); +export async function readRawObject( + runtime: StorageRuntime, + dir: string, + oid: string, +) { + const { object } = await git.readObject({ + fs: runtime.fs.git, + dir, + oid, + format: "content", + }); return RawObject.assert({ object }); } diff --git a/vendor/intx-storage-isogit/src/mail-store.test.ts b/vendor/intx-storage-isogit/src/mail-store.test.ts index c8c09924a..93cab1493 100644 --- a/vendor/intx-storage-isogit/src/mail-store.test.ts +++ b/vendor/intx-storage-isogit/src/mail-store.test.ts @@ -2,8 +2,53 @@ import { describe, test, expect, beforeEach } from "bun:test"; import fs from "node:fs"; import path from "node:path"; import os from "node:os"; -import { initAgentRepo } from "./init"; -import { createMailAuditStore, listMail } from "./mail-store"; +import git, { type PromiseFsClient } from "isomorphic-git"; +import { initAgentRepo } from "./node"; +import { createMailAuditStore, listMail } from "./node"; +import { createIsogitStorage } from "./index"; +import { createNodeIsogitRuntime } from "./node-runtime"; + +function createRefWriteFailureRuntime( + refPath: string, + effect: "before" | "after", +) { + const runtime = createNodeIsogitRuntime(); + const nodePromises = fs.promises; + let armed = false; + const promiseFs: PromiseFsClient = { + promises: { + readFile: nodePromises.readFile.bind(nodePromises), + writeFile: async (...args: unknown[]) => { + const filepath = args[0]; + if (!armed || filepath !== refPath) { + return Reflect.apply(nodePromises.writeFile, nodePromises, args); + } + if (effect === "after") { + await Reflect.apply(nodePromises.writeFile, nodePromises, args); + } + throw new Error(`injected ${effect}-effect ref write failure`); + }, + unlink: nodePromises.unlink.bind(nodePromises), + readdir: nodePromises.readdir.bind(nodePromises), + mkdir: nodePromises.mkdir.bind(nodePromises), + rmdir: nodePromises.rmdir.bind(nodePromises), + stat: nodePromises.stat.bind(nodePromises), + lstat: nodePromises.lstat.bind(nodePromises), + readlink: nodePromises.readlink.bind(nodePromises), + symlink: nodePromises.symlink.bind(nodePromises), + chmod: nodePromises.chmod.bind(nodePromises), + }, + }; + return { + runtime: { ...runtime, fs: promiseFs }, + arm: () => { + armed = true; + }, + disarm: () => { + armed = false; + }, + }; +} function buildRawMessage(opts: { messageId: string; @@ -263,4 +308,185 @@ describe("listMail", () => { if (entry === undefined) throw new Error("expected entry"); expect(new TextDecoder().decode(entry.raw)).toContain("hello from alice"); }); + + test("does not reuse an ordinal after uncertain ref publication", async () => { + let flushes = 0; + const storage = createIsogitStorage({ + ...createNodeIsogitRuntime(), + flush: () => { + flushes += 1; + if (flushes === 4) { + return Promise.reject(new Error("injected final flush failure")); + } + return Promise.resolve(); + }, + }); + const store = await storage.createMailAuditStore(testDir); + + const first = await store.commitMail( + buildRawMessage({ messageId: "" }), + "in", + ); + if (first === null) throw new Error("expected first mail commit"); + + await expect( + store.commitMail( + buildRawMessage({ + messageId: "", + inReplyTo: "", + }), + "in", + ), + ).rejects.toThrow("injected final flush failure"); + + const third = await store.commitMail( + buildRawMessage({ + messageId: "", + inReplyTo: "", + }), + "in", + ); + if (third === null) throw new Error("expected third mail commit"); + + expect(third.filepath).toBe(`state/mail/${first.threadId}/0003-in.eml`); + expect( + (await storage.listMail(testDir)).map((entry) => entry.messageId), + ).toEqual([ + "", + "", + "", + ]); + }); + + test("reconciles a ref write that takes effect before rejecting", async () => { + const refPath = path.join(testDir, ".git", "refs", "heads", "main"); + const failure = createRefWriteFailureRuntime(refPath, "after"); + const storage = createIsogitStorage(failure.runtime); + const store = await storage.createMailAuditStore(testDir); + + const first = await store.commitMail( + buildRawMessage({ messageId: "" }), + "in", + ); + if (first === null) throw new Error("expected first mail commit"); + + failure.arm(); + await expect( + store.commitMail( + buildRawMessage({ + messageId: "", + inReplyTo: "", + }), + "out", + ), + ).rejects.toThrow("Ref publication"); + failure.disarm(); + + const third = await store.commitMail( + buildRawMessage({ + messageId: "", + inReplyTo: "", + }), + "in", + ); + if (third === null) throw new Error("expected third mail commit"); + + expect(third.threadId).toBe(first.threadId); + expect(third.filepath).toBe(`state/mail/${first.threadId}/0003-in.eml`); + expect( + (await storage.listMail(testDir)).map((entry) => entry.messageId), + ).toEqual([ + "", + "", + "", + ]); + }); + + test("does not advance mail state when a ref write has no effect", async () => { + const refPath = path.join(testDir, ".git", "refs", "heads", "main"); + const failure = createRefWriteFailureRuntime(refPath, "before"); + const storage = createIsogitStorage(failure.runtime); + const store = await storage.createMailAuditStore(testDir); + + const first = await store.commitMail( + buildRawMessage({ messageId: "" }), + "in", + ); + if (first === null) throw new Error("expected first mail commit"); + const previousOid = await git.resolveRef({ fs, dir: testDir, ref: "HEAD" }); + + failure.arm(); + await expect( + store.commitMail( + buildRawMessage({ + messageId: "", + inReplyTo: "", + }), + "out", + ), + ).rejects.toThrow("Ref publication"); + failure.disarm(); + expect(await git.resolveRef({ fs, dir: testDir, ref: "HEAD" })).toBe( + previousOid, + ); + + const third = await store.commitMail( + buildRawMessage({ + messageId: "", + inReplyTo: "", + }), + "out", + ); + if (third === null) throw new Error("expected third mail commit"); + + expect(third.threadId).toBe(first.threadId); + expect(third.filepath).toBe(`state/mail/${first.threadId}/0002-out.eml`); + expect( + (await storage.listMail(testDir)).map((entry) => entry.messageId), + ).toEqual(["", ""]); + }); + + test("a failed mail commit cannot leak into a cycle commit", async () => { + let failed = false; + const storage = createIsogitStorage({ + ...createNodeIsogitRuntime(), + flush: () => { + if (!failed) { + failed = true; + return Promise.reject(new Error("injected object flush failure")); + } + return Promise.resolve(); + }, + }); + const mailStore = await storage.createMailAuditStore(testDir); + + await expect( + mailStore.commitMail( + buildRawMessage({ messageId: "" }), + "in", + ), + ).rejects.toThrow("injected object flush failure"); + + const [threadId] = await fs.promises.readdir( + path.join(testDir, "state", "mail"), + ); + if (threadId === undefined) throw new Error("expected mail thread"); + const [filename] = await fs.promises.readdir( + path.join(testDir, "state", "mail", threadId), + ); + if (filename === undefined) throw new Error("expected mail file"); + + const contextStore = await storage.createIsogitStore(testDir); + await contextStore.writeTurns([]); + const cycle = await contextStore.commit({ message: "Cycle: inference" }); + + await expect( + git.readBlob({ + fs, + dir: testDir, + oid: cycle.hash, + filepath: `state/mail/${threadId}/${filename}`, + }), + ).rejects.toMatchObject({ code: "NotFoundError" }); + }); }); diff --git a/vendor/intx-storage-isogit/src/mail-store.ts b/vendor/intx-storage-isogit/src/mail-store.ts index 7d2c794ee..168d1e08d 100644 --- a/vendor/intx-storage-isogit/src/mail-store.ts +++ b/vendor/intx-storage-isogit/src/mail-store.ts @@ -1,12 +1,16 @@ -import fs from "node:fs"; -import path from "node:path"; import git from "isomorphic-git"; import { parseHeaderSection } from "@intx/mime"; -import { hexEncode } from "@intx/types"; +import { hasCode, hexEncode } from "@intx/types"; import { AUTHOR } from "./init"; import type { CommitSigner } from "./signer"; -import { buildSigningArgs } from "./commit-helpers"; +import { + buildSigningArgs, + commitDurably, + restoreIndexAfterFailedCommit, + UncertainRefPublicationError, +} from "./commit-helpers"; import { withRepoDirLock } from "./repo-lock"; +import type { StorageRuntime } from "./runtime"; const MAIL_DIR = "state/mail"; @@ -68,6 +72,7 @@ function parseThreadingHeaders(raw: Uint8Array): { } export async function createMailAuditStore( + runtime: StorageRuntime, dir: string, signer?: CommitSigner, ): Promise { @@ -77,8 +82,9 @@ export async function createMailAuditStore( const messageIndex = new Map(); // thread-id -> thread state const threads = new Map(); + let reconciliationFailure: Error | undefined; - await rebuildIndex(dir, messageIndex, threads); + await rebuildIndex(runtime, dir, messageIndex, threads); function resolveThread( inReplyTo: string | undefined, @@ -119,7 +125,14 @@ export async function createMailAuditStore( // The whole body runs under the per-directory lock: the ordinal peek, // the commit, and the in-memory index/ordinal advance must be atomic // against a concurrent reactor commit or GC pass sharing this repo. - return withRepoDirLock(dir, async () => { + return withRepoDirLock(runtime, dir, async () => { + if (reconciliationFailure !== undefined) { + throw new Error( + "Mail audit store cannot accept writes after failed ref reconciliation", + { cause: reconciliationFailure }, + ); + } + const { messageId, inReplyTo, references } = parseThreadingHeaders(rawMessage); @@ -133,14 +146,13 @@ export async function createMailAuditStore( const threadId = resolveThread(inReplyTo, references); const ordinal = peekNextOrdinal(threadId); const filename = `${formatOrdinal(ordinal)}-${direction}.eml`; - const filepath = path.join(MAIL_DIR, threadId, filename); + const filepath = runtime.path.join(MAIL_DIR, threadId, filename); - const fullDir = path.join(dir, MAIL_DIR, threadId); - await fs.promises.mkdir(fullDir, { recursive: true }); + const fullDir = runtime.path.join(dir, MAIL_DIR, threadId); + await runtime.fs.mkdir(fullDir, { recursive: true }); - const fullPath = path.join(dir, filepath); - await fs.promises.writeFile(fullPath, rawMessage); - await git.add({ fs, dir, filepath }); + const fullPath = runtime.path.join(dir, filepath); + await runtime.fs.writeFile(fullPath, rawMessage); const label = direction === "in" ? "inbound" : "outbound"; const subject = `Record ${label} mail ${messageId}`; @@ -148,13 +160,46 @@ export async function createMailAuditStore( options?.checkpointHash !== undefined ? `${subject}\n\nCheckpoint: ${options.checkpointHash}` : subject; - await git.commit({ - fs, - dir, - message, - author: AUTHOR, - ...signingArgs, - }); + try { + await git.add({ fs: runtime.fs.git, dir, filepath }); + await commitDurably(runtime, dir, { + message, + author: AUTHOR, + ...signingArgs, + }); + } catch (cause) { + try { + await restoreIndexAfterFailedCommit(runtime, dir, [filepath]); + } catch (reconciliationCause) { + reconciliationFailure = new Error( + "Could not restore the mail index after commit failure", + { + cause: new AggregateError([cause, reconciliationCause]), + }, + ); + throw reconciliationFailure; + } + + if (!(cause instanceof UncertainRefPublicationError)) throw cause; + + try { + const actualOid = await git.resolveRef({ + fs: runtime.fs.git, + dir, + ref: cause.ref, + }); + if (actualOid === cause.oid) { + advanceOrdinal(threadId); + messageIndex.set(messageId, threadId); + } + } catch (reconciliationCause) { + reconciliationFailure = new Error( + `Could not reconcile uncertain mail commit ${cause.oid}`, + { cause: reconciliationCause }, + ); + } + throw cause; + } advanceOrdinal(threadId); messageIndex.set(messageId, threadId); @@ -191,14 +236,17 @@ function parseFilename(filename: string): { return { ordinal, direction: directionStr }; } -async function scanMail(dir: string): Promise { - const mailDir = path.join(dir, MAIL_DIR); +async function scanMail( + runtime: StorageRuntime, + dir: string, +): Promise { + const mailDir = runtime.path.join(dir, MAIL_DIR); let threadDirs: string[]; try { - threadDirs = await fs.promises.readdir(mailDir); + threadDirs = await runtime.fs.readdir(mailDir); } catch (e: unknown) { - if (e instanceof Error && "code" in e && e.code === "ENOENT") { + if (hasCode(e) && e.code === "ENOENT") { return []; } throw e; @@ -209,17 +257,17 @@ async function scanMail(dir: string): Promise { const entries: MailEntry[] = []; for (const threadId of threadDirs) { - const threadPath = path.join(mailDir, threadId); - const stat = await fs.promises.stat(threadPath); + const threadPath = runtime.path.join(mailDir, threadId); + const stat = await runtime.fs.stat(threadPath); if (!stat.isDirectory()) continue; - const files = await fs.promises.readdir(threadPath); + const files = await runtime.fs.readdir(threadPath); const emlFiles = files.filter((f) => f.endsWith(".eml")).sort(); for (const file of emlFiles) { const { ordinal, direction } = parseFilename(file); - const fullPath = path.join(threadPath, file); - const raw = await fs.promises.readFile(fullPath); + const fullPath = runtime.path.join(threadPath, file); + const raw = await runtime.fs.readFile(fullPath); const { messageId } = parseThreadingHeaders(raw); entries.push({ threadId, ordinal, direction, messageId, raw }); @@ -229,16 +277,20 @@ async function scanMail(dir: string): Promise { return entries; } -export async function listMail(dir: string): Promise { - return scanMail(dir); +export async function listMail( + runtime: StorageRuntime, + dir: string, +): Promise { + return scanMail(runtime, dir); } async function rebuildIndex( + runtime: StorageRuntime, dir: string, messageIndex: Map, threads: Map, ): Promise { - const entries = await scanMail(dir); + const entries = await scanMail(runtime, dir); for (const entry of entries) { messageIndex.set(entry.messageId, entry.threadId); diff --git a/vendor/intx-storage-isogit/src/node-metrics.ts b/vendor/intx-storage-isogit/src/node-metrics.ts new file mode 100644 index 000000000..189b5b75d --- /dev/null +++ b/vendor/intx-storage-isogit/src/node-metrics.ts @@ -0,0 +1,84 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { hasCode } from "@intx/types"; +import type { RepoDiskUsage, RepoObjectCounts } from "./repo-disk"; + +// Synchronous Node counterparts to the runtime-backed helpers in repo-disk.ts. +// Keep their counting and absence semantics aligned. + +function countDirEntries(dir: string): number { + try { + return fs.readdirSync(dir).length; + } catch (cause) { + if (hasCode(cause) && cause.code === "ENOENT") return 0; + throw cause; + } +} + +export function countLooseObjects(repoDir: string): number { + const objectsDir = path.join(repoDir, ".git", "objects"); + let fanoutDirs: string[]; + try { + fanoutDirs = fs.readdirSync(objectsDir); + } catch (cause) { + if (hasCode(cause) && cause.code === "ENOENT") return 0; + throw cause; + } + let total = 0; + for (const name of fanoutDirs) { + if (name === "pack" || name === "info") continue; + if (!/^[0-9a-f]{2}$/.test(name)) continue; + total += countDirEntries(path.join(objectsDir, name)); + } + return total; +} + +export function countPackFiles(repoDir: string): number { + const packDir = path.join(repoDir, ".git", "objects", "pack"); + try { + return fs.readdirSync(packDir).filter((name) => name.endsWith(".pack")) + .length; + } catch (cause) { + if (hasCode(cause) && cause.code === "ENOENT") return 0; + throw cause; + } +} + +export function gitBytes(repoDir: string): number { + let total = 0; + const stack = [path.join(repoDir, ".git")]; + while (stack.length > 0) { + const current = stack.pop(); + if (current === undefined) break; + let stat: fs.Stats; + try { + stat = fs.lstatSync(current); + } catch (cause) { + if (hasCode(cause) && cause.code === "ENOENT") continue; + throw cause; + } + if (stat.isDirectory()) { + for (const child of fs.readdirSync(current)) { + stack.push(path.join(current, child)); + } + } else if (stat.isFile()) { + total += stat.size; + } + } + return total; +} + +export function repoObjectCounts(dir: string): RepoObjectCounts { + return { + packCount: countPackFiles(dir), + looseObjectCount: countLooseObjects(dir), + }; +} + +export function repoDiskUsage(dir: string): RepoDiskUsage { + return { + gitBytes: gitBytes(dir), + ...repoObjectCounts(dir), + }; +} diff --git a/vendor/intx-storage-isogit/src/node-runtime.ts b/vendor/intx-storage-isogit/src/node-runtime.ts new file mode 100644 index 000000000..8f522e11a --- /dev/null +++ b/vendor/intx-storage-isogit/src/node-runtime.ts @@ -0,0 +1,12 @@ +import fs from "node:fs"; +import path from "node:path"; + +import type { IsogitRuntime } from "./runtime"; + +export function createNodeIsogitRuntime(): IsogitRuntime { + return { + fs, + path, + rename: (oldPath, newPath) => fs.promises.rename(oldPath, newPath), + }; +} diff --git a/vendor/intx-storage-isogit/src/node.ts b/vendor/intx-storage-isogit/src/node.ts new file mode 100644 index 000000000..885dc8b9e --- /dev/null +++ b/vendor/intx-storage-isogit/src/node.ts @@ -0,0 +1,79 @@ +import type { AuditStore, ContextStore } from "@intx/types/runtime"; + +import { + createIsogitStorage, + type CommitSigner, + type DurableMirrorReads, + type GCPolicy, +} from "./index"; +import { createNodeIsogitRuntime } from "./node-runtime"; +import { IsogitStore as RuntimeIsogitStore } from "./store"; +export { createNodeIsogitRuntime } from "./node-runtime"; +export { + countLooseObjects, + countPackFiles, + gitBytes, + repoDiskUsage, +} from "./node-metrics"; + +export { + createIsogitStorage, + type CommitSigner, + type DurableMirrorReads, + type GCPolicy, + type ContextStore, + type AuditStore, + type CommitVerifier, + type TreeValidator, + type TreeValidatorResult, + type IncludeShaPredicate, + type InitRepoOpts, + type RetentionPolicy, + type GCResult, + type RepoDiskUsage, + type MailAuditStore, + type MailCommitOptions, + type MailCommitResult, + type MailDirection, + type MailEntry, + type IsogitPath, + type IsogitRuntime, +} from "./index"; + +export const runtime = createNodeIsogitRuntime(); +const storage = createIsogitStorage(runtime); +const { storageRuntime } = storage; + +export const { + applyPack, + collectReachableObjects, + createAndSwitchBranch, + createDeployPack, + createMailAuditStore, + createNegotiatedPack, + currentBranch, + initAgentRepo, + initRepo, + listBranches, + listMail, + listRepoRefs, + logHistory, + maybeGC, + receivePackObjects, + runGC, + switchBranch, +} = storage; + +export class IsogitStore extends RuntimeIsogitStore { + constructor(dir: string, signer?: CommitSigner, gcPolicy?: GCPolicy) { + super(storageRuntime, storageRuntime.path.resolve(dir), signer, gcPolicy); + } +} + +export async function createIsogitStore( + dir: string, + signer?: CommitSigner, + gcPolicy?: GCPolicy, +): Promise { + return storage.createIsogitStore(dir, signer, gcPolicy); +} diff --git a/vendor/intx-storage-isogit/src/object-walk.ts b/vendor/intx-storage-isogit/src/object-walk.ts index d6071ee36..1e46d2f90 100644 --- a/vendor/intx-storage-isogit/src/object-walk.ts +++ b/vendor/intx-storage-isogit/src/object-walk.ts @@ -1,21 +1,23 @@ import { readCommitObject, readTreeEntries } from "./isogit-helpers"; +import type { StorageRuntime } from "./runtime"; /** * Collect all unique object OIDs reachable from a commit: the commit itself, * its tree, and all blobs and subtrees recursively. */ export async function collectReachableObjects( + runtime: StorageRuntime, dir: string, commitOid: string, ): Promise { const seen = new Set(); seen.add(commitOid); - const commit = await readCommitObject(dir, commitOid); + const commit = await readCommitObject(runtime, dir, commitOid); seen.add(commit.tree); async function walkTree(treeOid: string): Promise { - const entries = await readTreeEntries(dir, treeOid); + const entries = await readTreeEntries(runtime, dir, treeOid); for (const entry of entries) { if (seen.has(entry.oid)) continue; seen.add(entry.oid); diff --git a/vendor/intx-storage-isogit/src/pack-receive.test.ts b/vendor/intx-storage-isogit/src/pack-receive.test.ts index 0e2ad2e38..50176c164 100644 --- a/vendor/intx-storage-isogit/src/pack-receive.test.ts +++ b/vendor/intx-storage-isogit/src/pack-receive.test.ts @@ -1,4 +1,4 @@ -import { describe, test, expect, afterEach } from "bun:test"; +import { afterEach, describe, expect, spyOn, test } from "bun:test"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -8,14 +8,28 @@ import { createSSHSignature, verifySSHSignature, } from "@intx/crypto"; -import { initAgentRepo } from "./init"; +import { initAgentRepo } from "./node"; +import { createIsogitStorage } from "./index"; import { applyPack, receivePackObjects, type CommitVerifier, type TreeValidator, -} from "./pack-receive"; -import { collectReachableObjects } from "./object-walk"; +} from "./node"; +import { collectReachableObjects } from "./node"; +import { createNodeIsogitRuntime } from "./node-runtime"; +import { publishPackAtomically as publishPackAtomicallyImpl } from "./pack-receive"; +import { normalizeRuntime } from "./runtime"; + +const publishRuntime = normalizeRuntime(createNodeIsogitRuntime()); + +function publishPackAtomically( + dir: string, + pack: Uint8Array, + transferId: string, +): Promise { + return publishPackAtomicallyImpl(publishRuntime, dir, pack, transferId); +} const tempDirs: string[] = []; @@ -408,6 +422,181 @@ describe("applyPack", () => { }); }); +describe("publishPackAtomically", () => { + test("renames the completed pack before publishing its index", async () => { + const source = await makeSourceRepo(); + const pack = await createPackFromRepo(source.dir, source.oids); + const targetDir = await tempDir(); + await initAgentRepo(targetDir); + + const originalRename = fs.promises.rename.bind(fs.promises); + const renames: { from: string; to: string }[] = []; + const renameSpy = spyOn(fs.promises, "rename").mockImplementation( + async (from, to) => { + await originalRename(from, to); + renames.push({ from: from.toString(), to: to.toString() }); + }, + ); + + try { + await publishPackAtomically(targetDir, pack, "rename-order"); + } finally { + renameSpy.mockRestore(); + } + + const finalPack = path.join( + targetDir, + ".git", + "objects", + "pack", + "pack-recv-rename-order.pack", + ); + const finalIdx = finalPack.replace(/\.pack$/, ".idx"); + expect(renames).toEqual([ + { + from: path.join( + targetDir, + ".git", + "objects", + "pack-staging", + "rename-order", + "pack.pack", + ), + to: finalPack, + }, + { + from: path.join( + targetDir, + ".git", + "objects", + "pack-staging", + "rename-order", + "pack.idx", + ), + to: finalIdx, + }, + ]); + await fs.promises.access(finalPack); + await fs.promises.access(finalIdx); + await expect( + fs.promises.access( + path.join(targetDir, ".git", "objects", "pack-staging", "rename-order"), + ), + ).rejects.toThrow(); + }); + + test.each(["pack", "idx"] as const)( + "does not overwrite an existing final %s", + async (extension) => { + const source = await makeSourceRepo(); + const pack = await createPackFromRepo(source.dir, source.oids); + const targetDir = await tempDir(); + await initAgentRepo(targetDir); + + const packDir = path.join(targetDir, ".git", "objects", "pack"); + const existingPath = path.join( + packDir, + `pack-recv-duplicate.${extension}`, + ); + await fs.promises.mkdir(packDir, { recursive: true }); + await fs.promises.writeFile(existingPath, "existing"); + + await expect( + publishPackAtomically(targetDir, pack, "duplicate"), + ).rejects.toThrow(/already published/); + expect(await fs.promises.readFile(existingPath, "utf8")).toBe("existing"); + }, + ); + + test("rejects concurrent publishers using the same transfer ID", async () => { + const source = await makeSourceRepo(); + const pack = await createPackFromRepo(source.dir, source.oids); + const targetDir = await tempDir(); + await initAgentRepo(targetDir); + + const originalIndexPack = git.indexPack.bind(git); + let signalIndexingStarted: () => void = () => undefined; + const indexingStarted = new Promise((resolve) => { + signalIndexingStarted = resolve; + }); + let releaseIndexing: () => void = () => undefined; + const indexingReleased = new Promise((resolve) => { + releaseIndexing = resolve; + }); + let blockNextIndex = true; + const indexPackSpy = spyOn(git, "indexPack").mockImplementation( + async (args) => { + if (blockNextIndex) { + blockNextIndex = false; + signalIndexingStarted(); + await indexingReleased; + } + return originalIndexPack(args); + }, + ); + + let firstPublish: Promise | undefined; + try { + firstPublish = publishPackAtomically(targetDir, pack, "concurrent"); + await indexingStarted; + + await expect( + publishPackAtomically(targetDir, pack, "concurrent"), + ).rejects.toThrow(/already published or in progress/); + expect(indexPackSpy).toHaveBeenCalledTimes(1); + + releaseIndexing(); + await firstPublish; + } finally { + releaseIndexing(); + await firstPublish?.catch(() => undefined); + indexPackSpy.mockRestore(); + } + }); + + test("cleans up when index publication fails after the pack rename", async () => { + const source = await makeSourceRepo(); + const pack = await createPackFromRepo(source.dir, source.oids); + const targetDir = await tempDir(); + await initAgentRepo(targetDir); + + const originalRename = fs.promises.rename.bind(fs.promises); + let renameCount = 0; + const renameSpy = spyOn(fs.promises, "rename").mockImplementation( + async (from, to) => { + renameCount += 1; + if (renameCount === 2) throw new Error("index publish failed"); + await originalRename(from, to); + }, + ); + + try { + await expect( + publishPackAtomically(targetDir, pack, "partial"), + ).rejects.toThrow("index publish failed"); + } finally { + renameSpy.mockRestore(); + } + + const finalPack = path.join( + targetDir, + ".git", + "objects", + "pack", + "pack-recv-partial.pack", + ); + await expect(fs.promises.access(finalPack)).rejects.toThrow(); + await expect( + fs.promises.access(finalPack.replace(/\.pack$/, ".idx")), + ).rejects.toThrow(); + await expect( + fs.promises.access( + path.join(targetDir, ".git", "objects", "pack-staging", "partial"), + ), + ).rejects.toThrow(); + }); +}); + async function makeSignedSourceRepo(keyPair: { privateKey: Uint8Array; publicKey: Uint8Array; @@ -935,6 +1124,102 @@ async function fileExists(p: string): Promise { } } +function createStorageFailingPromotionFlush( + targetDir: string, + ref: string, + expectedSha: string, +) { + let failureInjected = false; + return createIsogitStorage({ + ...createNodeIsogitRuntime(), + flush: async () => { + const currentSha = await git + .resolveRef({ fs, dir: targetDir, ref }) + .catch(() => null); + if (!failureInjected && currentSha === expectedSha) { + failureInjected = true; + throw new Error("injected post-promotion flush failure"); + } + }, + }); +} + +describe("post-promotion flush failures", () => { + test("receive retains the durable pack for the promoted ref", async () => { + const source = await makeRepoWithPaths([ + { filepath: "state/v1.jsonl", content: "v1" }, + ]); + const pack = await createPackFromRepo(source.dir, source.oids); + const targetDir = await tempDir(); + const ref = "refs/heads/state"; + const transferId = "receive-flush-failure"; + const storage = createStorageFailingPromotionFlush( + targetDir, + ref, + source.commitSha, + ); + await storage.initAgentRepo(targetDir); + + await expect( + storage.receivePackObjects( + targetDir, + pack, + ref, + source.commitSha, + transferId, + null, + ), + ).rejects.toThrow("injected post-promotion flush failure"); + + expect(await fileExists(publishedPackFile(targetDir, transferId))).toBe( + true, + ); + expect(await fileExists(publishedIdxFile(targetDir, transferId))).toBe( + true, + ); + expect(await git.resolveRef({ fs, dir: targetDir, ref })).toBe( + source.commitSha, + ); + expect( + (await git.readCommit({ fs, dir: targetDir, oid: source.commitSha })).oid, + ).toBe(source.commitSha); + }); + + test("apply retains the durable pack and materialized tree for the promoted ref", async () => { + const source = await makeSourceRepo(); + const pack = await createPackFromRepo(source.dir, source.oids); + const targetDir = await tempDir(); + const ref = "refs/heads/deploy"; + const transferId = "apply-flush-failure"; + const storage = createStorageFailingPromotionFlush( + targetDir, + ref, + source.commitSha, + ); + await storage.initAgentRepo(targetDir); + + await expect( + storage.applyPack(targetDir, pack, ref, source.commitSha, transferId), + ).rejects.toThrow("injected post-promotion flush failure"); + + expect(await fileExists(publishedPackFile(targetDir, transferId))).toBe( + true, + ); + expect(await fileExists(publishedIdxFile(targetDir, transferId))).toBe( + true, + ); + expect(await git.resolveRef({ fs, dir: targetDir, ref })).toBe( + source.commitSha, + ); + expect( + await fs.promises.readFile( + path.join(targetDir, "deploy", "prompt.txt"), + "utf8", + ), + ).toBe("You are a helpful agent."); + }); +}); + // These tests assert that post-publish validation rejections in // receivePackObjects and applyPack remove the published .pack + .idx // pair from objects/pack/. Without this, rejected packs would @@ -999,11 +1284,19 @@ describe("receivePackObjects unpublishes on post-publish rejection", () => { const pack = await createPackFromRepo(source.dir, source.oids); const targetDir = await tempDir(); - await initAgentRepo(targetDir); + let flushes = 0; + const storage = createIsogitStorage({ + ...createNodeIsogitRuntime(), + flush: () => { + flushes += 1; + return Promise.resolve(); + }, + }); + await storage.initAgentRepo(targetDir); const bogusExpected = "0".repeat(40); await expect( - receivePackObjects( + storage.receivePackObjects( targetDir, pack, "refs/heads/state", @@ -1019,6 +1312,7 @@ describe("receivePackObjects unpublishes on post-publish rejection", () => { expect(await fileExists(publishedIdxFile(targetDir, "rejected-sha"))).toBe( false, ); + expect(flushes).toBe(2); }); test("path_violation removes .pack and .idx", async () => { @@ -1060,11 +1354,19 @@ describe("applyPack unpublishes on post-publish rejection", () => { const pack = await createPackFromRepo(source.dir, source.oids); const targetDir = await tempDir(); - await initAgentRepo(targetDir); + let flushes = 0; + const storage = createIsogitStorage({ + ...createNodeIsogitRuntime(), + flush: () => { + flushes += 1; + return Promise.resolve(); + }, + }); + await storage.initAgentRepo(targetDir); const bogusExpected = "0".repeat(40); await expect( - applyPack( + storage.applyPack( targetDir, pack, "refs/heads/deploy", @@ -1079,5 +1381,6 @@ describe("applyPack unpublishes on post-publish rejection", () => { expect( await fileExists(publishedIdxFile(targetDir, "apply-rejected-sha")), ).toBe(false); + expect(flushes).toBe(2); }); }); diff --git a/vendor/intx-storage-isogit/src/pack-receive.ts b/vendor/intx-storage-isogit/src/pack-receive.ts index a0a14158a..dadd7ccdf 100644 --- a/vendor/intx-storage-isogit/src/pack-receive.ts +++ b/vendor/intx-storage-isogit/src/pack-receive.ts @@ -1,8 +1,8 @@ -import fs from "node:fs"; -import path from "node:path"; import git from "isomorphic-git"; import { readRawObject } from "./isogit-helpers"; import { withRepoDirLock } from "./repo-lock"; +import { decodeUTF8, flushRuntime, type StorageRuntime } from "./runtime"; +import { hasCode } from "@intx/types"; /** * Verifies the signature embedded in a git commit object. @@ -75,8 +75,8 @@ function stripGpgsig(raw: string): string { const SAFE_PATH_SEGMENT = /^[a-zA-Z0-9_-]+$/; -// Sibling of objects/pack/. Lives on the same filesystem so fs.link and -// fs.rename between staging and pack/ are atomic metadata operations. +// Sibling of objects/pack/. Both directories share the storage namespace in +// which the runtime guarantees atomic rename operations. const STAGING_DIR_NAME = "pack-staging"; /** @@ -86,39 +86,63 @@ const STAGING_DIR_NAME = "pack-staging"; * to unpublish without re-deriving the naming convention. */ function publishedPackPaths( + runtime: StorageRuntime, dir: string, transferId: string, ): { finalPackPath: string; finalIdxPath: string } { - const packDir = path.join(dir, ".git", "objects", "pack"); - const finalPackPath = path.join(packDir, `pack-recv-${transferId}.pack`); + const packDir = runtime.path.join(dir, ".git", "objects", "pack"); + const finalPackPath = runtime.path.join( + packDir, + `pack-recv-${transferId}.pack`, + ); const finalIdxPath = finalPackPath.replace(/\.pack$/, ".idx"); return { finalPackPath, finalIdxPath }; } +async function pathExists( + runtime: StorageRuntime, + filepath: string, +): Promise { + try { + await runtime.fs.access(filepath); + return true; + } catch (cause) { + if (hasCode(cause) && cause.code === "ENOENT") return false; + throw cause; + } +} + /** * Remove a previously-published `.pack` + `.idx` pair from objects/pack/. * * Callers reject a published pack by calling this after their post-publish * validation fails (signature, tree-validator, sha mismatch, CAS - * non-fast-forward, etc.). POSIX semantics mean unlink does not affect - * file descriptors a concurrent reader has already opened, so a reader - * mid-call against the rejected pack finishes its read against the - * already-loaded bytes; subsequent calls will not see the pack at all. - * Iso-git does not hold open file descriptors between calls (verified - * by reading `readObjectPacked` at `index.cjs:3394-3398`), so the - * unlink is safe against the standard pack-discovery path. - * - * Wrapped in `.catch(() => undefined)` per rm so a secondary failure + * non-fast-forward, etc.). A reader that already loaded the rejected pack + * can finish against those bytes; subsequent calls will not discover the + * removed pair. Iso-git does not retain pack resources between calls + * (verified by reading `readObjectPacked` at `index.cjs:3394-3398`). + * + * Wrapped in `.catch(() => undefined)` per removal so a secondary failure * (permissions, I/O) does not mask whatever rejection the caller is - * about to throw. A failed unlink leaks the rejected pack on disk + * about to throw. A failed removal leaves the rejected pack in storage * until external cleanup; that is acceptable because the caller is * already in an error path and the alternative is hiding the real * cause behind a cleanup throw. */ -async function unpublishPack(dir: string, transferId: string): Promise { - const { finalPackPath, finalIdxPath } = publishedPackPaths(dir, transferId); - await fs.promises.rm(finalPackPath, { force: true }).catch(() => undefined); - await fs.promises.rm(finalIdxPath, { force: true }).catch(() => undefined); +async function unpublishPack( + runtime: StorageRuntime, + dir: string, + transferId: string, +): Promise { + const { finalPackPath, finalIdxPath } = publishedPackPaths( + runtime, + dir, + transferId, + ); + await runtime.fs + .remove(finalPackPath, { force: true }) + .catch(() => undefined); + await runtime.fs.remove(finalIdxPath, { force: true }).catch(() => undefined); } type TreeEntry = { @@ -128,8 +152,12 @@ type TreeEntry = { oid: string; }; -async function topLevelNames(dir: string, oid: string): Promise> { - const { tree } = await git.readTree({ fs, dir, oid }); +async function topLevelNames( + runtime: StorageRuntime, + dir: string, + oid: string, +): Promise> { + const { tree } = await git.readTree({ fs: runtime.fs.git, dir, oid }); return new Set(tree.map((e) => e.path)); } @@ -148,54 +176,74 @@ async function topLevelNames(dir: string, oid: string): Promise> { * working tree will be stale until the next successful applyPack. */ async function checkoutTree( + runtime: StorageRuntime, dir: string, commitSha: string, ref: string, ): Promise { - const { commit } = await git.readCommit({ fs, dir, oid: commitSha }); - const { tree } = await git.readTree({ fs, dir, oid: commit.tree }); + const { commit } = await git.readCommit({ + fs: runtime.fs.git, + dir, + oid: commitSha, + }); + const { tree } = await git.readTree({ + fs: runtime.fs.git, + dir, + oid: commit.tree, + }); // Collect top-level names managed by deploy trees (new + previous). const managed = new Set(tree.map((e) => e.path)); - const prevSha = await git.resolveRef({ fs, dir, ref }).catch(() => null); + const prevSha = await git + .resolveRef({ fs: runtime.fs.git, dir, ref }) + .catch(() => null); if (prevSha !== null) { - const { commit: prev } = await git.readCommit({ fs, dir, oid: prevSha }); - for (const name of await topLevelNames(dir, prev.tree)) { + const { commit: prev } = await git.readCommit({ + fs: runtime.fs.git, + dir, + oid: prevSha, + }); + for (const name of await topLevelNames(runtime, dir, prev.tree)) { managed.add(name); } } - const existing = await fs.promises.readdir(dir); + const existing = await runtime.fs.readdir(dir); for (const name of existing) { if (!managed.has(name)) continue; - await fs.promises.rm(path.join(dir, name), { + await runtime.fs.remove(runtime.path.join(dir, name), { recursive: true, force: true, }); } - await writeTreeEntries(dir, dir, tree); + await writeTreeEntries(runtime, dir, dir, tree); } async function writeTreeEntries( + runtime: StorageRuntime, repoDir: string, targetDir: string, entries: TreeEntry[], ): Promise { for (const entry of entries) { - const entryPath = path.join(targetDir, entry.path); + const entryPath = runtime.path.join(targetDir, entry.path); if (entry.type === "tree") { - await fs.promises.mkdir(entryPath, { recursive: true }); - const { tree } = await git.readTree({ fs, dir: repoDir, oid: entry.oid }); - await writeTreeEntries(repoDir, entryPath, tree); + await runtime.fs.mkdir(entryPath, { recursive: true }); + const { tree } = await git.readTree({ + fs: runtime.fs.git, + dir: repoDir, + oid: entry.oid, + }); + await writeTreeEntries(runtime, repoDir, entryPath, tree); } else if (entry.type === "blob") { const { blob } = await git.readBlob({ - fs, + fs: runtime.fs.git, dir: repoDir, oid: entry.oid, }); - await fs.promises.writeFile(entryPath, blob, { + await runtime.fs.writeFile(entryPath, blob, { mode: entry.mode === "100755" ? 0o755 : 0o644, }); } @@ -263,10 +311,8 @@ async function writeTreeEntries( * * We stage in `objects/pack-staging//`. The per-transfer * subdirectory keeps concurrent receives' temp pairs isolated from each - * other; the kernel guarantees that the staging directory and the - * final `objects/pack/` are on the same filesystem (they share a - * parent), so `fs.link` and `fs.rename` between them are atomic - * metadata operations. + * other. It shares a storage namespace with the final `objects/pack/` + * directory, so the runtime's atomic rename contract applies between them. * * # Sequence (numbered for cross-reference with cleanup contract) * @@ -284,13 +330,13 @@ async function writeTreeEntries( * 3. Atomic publish (transitions readers from "no new pack" to "new * pack visible"): * - * a. `fs.link` staging `.pack` -> final `.pack`. The new + * a. Atomically rename staging `.pack` -> final `.pack`. The new * `.pack` now exists in `objects/pack/`. Readers scanning * for `.idx` files still do not see the new pack (no `.idx` * for it in `objects/pack/` yet). The staging `.pack` - * directory entry is still present. + * directory entry vanishes. * - * b. `fs.rename` staging `.idx` -> final `.idx`. The new `.idx` + * b. Atomically rename staging `.idx` -> final `.idx`. The new `.idx` * appears atomically in `objects/pack/`; readers' next * directory scan finds it and resolves the `.pack` written * in 3a. The staging directory's `.idx` entry vanishes; @@ -298,21 +344,19 @@ async function writeTreeEntries( * the first place, this transition is observable only to * this function. * - * c. Remove the staging directory recursively (`unlink` of the - * remaining `.pack` plus `rmdir`). The final `.pack` inode - * persists via the link created in 3a. + * c. Remove the now-empty staging directory. * * 4. On any throw before publish completes, recursively remove the - * staging directory. `fs.rm({recursive: true, force: true})` - * tolerates partial states (e.g. throw mid-write before `.idx` - * exists, or throw inside `indexPack`). + * staging directory. Forced recursive removal tolerates partial states + * (e.g. throw mid-write before `.idx` exists, or throw inside + * `indexPack`). * * # Cleanup contract * * On successful return, no staging files remain. On throw before * publish (steps 1-2), the staging directory and any files inside it * are removed. A throw partway through publish (between 3a and 3b) - * leaves a linked-but-unindexed `.pack` in `objects/pack/`; this is + * leaves an unindexed `.pack` in `objects/pack/`; this is * harmless (iso-git ignores `.pack` files with no matching `.idx`, * verified at `index.cjs:3394-3398`) and the recovery path on the next * call would re-write the same content. The cleanup-on-throw path @@ -330,11 +374,9 @@ async function writeTreeEntries( * Callers that reject the published pack (signature failure, tree * validator rejection, sha mismatch, CAS non-fast-forward) call * `unpublishPack` to remove the published `.pack` + `.idx` pair from - * `objects/pack/`. POSIX semantics let a concurrent reader that - * already opened the rejected files finish its read against the - * cached bytes, and iso-git does not hold open descriptors between - * calls, so the unlink does not introduce the concurrent-read race - * that the staging strategy exists to eliminate. + * `objects/pack/`. A concurrent reader that already loaded the rejected + * pack can finish against those bytes, while subsequent calls no longer + * discover the pair. * * # Scope * @@ -357,6 +399,7 @@ async function writeTreeEntries( * next reader deciding whether the dance is still needed. */ export async function publishPackAtomically( + runtime: StorageRuntime, dir: string, pack: Uint8Array, transferId: string, @@ -367,30 +410,58 @@ export async function publishPackAtomically( ); } - const packDir = path.join(dir, ".git", "objects", "pack"); - const stagingRoot = path.join(dir, ".git", "objects", STAGING_DIR_NAME); - const stagingDir = path.join(stagingRoot, transferId); - - await fs.promises.mkdir(packDir, { recursive: true }); - await fs.promises.mkdir(stagingDir, { recursive: true }); + const packDir = runtime.path.join(dir, ".git", "objects", "pack"); + const stagingRoot = runtime.path.join( + dir, + ".git", + "objects", + STAGING_DIR_NAME, + ); + const stagingDir = runtime.path.join(stagingRoot, transferId); - const stagingPackPath = path.join(stagingDir, "pack.pack"); + const stagingPackPath = runtime.path.join(stagingDir, "pack.pack"); const stagingIdxPath = stagingPackPath.replace(/\.pack$/, ".idx"); - const { finalPackPath, finalIdxPath } = publishedPackPaths(dir, transferId); + const { finalPackPath, finalIdxPath } = publishedPackPaths( + runtime, + dir, + transferId, + ); + + await runtime.fs.mkdir(packDir, { recursive: true }); + await runtime.fs.mkdir(stagingRoot, { recursive: true }); + if ( + (await pathExists(runtime, finalPackPath)) || + (await pathExists(runtime, finalIdxPath)) + ) { + throw new Error( + `transferId "${transferId}" already published in ${dir}; callers must guarantee transferId uniqueness across all historical receives`, + ); + } + try { + await runtime.fs.mkdir(stagingDir); + } catch (cause) { + if (hasCode(cause) && cause.code === "EEXIST") { + throw new Error( + `transferId "${transferId}" already published or in progress in ${dir}; callers must guarantee transferId uniqueness across all historical receives`, + { cause }, + ); + } + throw cause; + } // iso-git's indexPack takes a repo-root-relative filepath; derive it // from the absolute path rather than rebuilding the segment list so // the two stay coupled. - const stagingFilepath = path.relative(dir, stagingPackPath); + const stagingFilepath = runtime.path.relative(dir, stagingPackPath); let oids: string[]; try { // Step 1: write the pack bytes to the staging directory. - await fs.promises.writeFile(stagingPackPath, pack); + await runtime.fs.writeFile(stagingPackPath, pack); // Step 2: index the pack. iso-git writes the .idx next to the // .pack — both stay inside the staging directory, invisible to // any reader scanning objects/pack/. const result = await git.indexPack({ - fs, + fs: runtime.fs.git, dir, filepath: stagingFilepath, }); @@ -400,56 +471,35 @@ export async function publishPackAtomically( // partial state (write succeeded but indexPack threw, etc.). The // rm is wrapped so a secondary cleanup failure (permissions, I/O) // does not mask the original error. - await fs.promises - .rm(stagingDir, { recursive: true, force: true }) + await runtime.fs + .remove(stagingDir, { recursive: true, force: true }) .catch(() => undefined); throw err; } - // Step 3: atomic publish. The .pack link (3a) and the .idx rename + // Step 3: atomic publish. The .pack rename (3a) and the .idx rename // (3b) are the two operations that determine externally-observable // state; a failure inside this block means the publish is partial // or absent, and the recovery rms below clear the half-published // pack from objects/pack/. Staging-directory cleanup is NOT part of // this block — see below. try { - await fs.promises.link(stagingPackPath, finalPackPath); // 3a - await fs.promises.rename(stagingIdxPath, finalIdxPath); // 3b + await runtime.fs.rename(stagingPackPath, finalPackPath); // 3a + await runtime.fs.rename(stagingIdxPath, finalIdxPath); // 3b } catch (err) { - // EEXIST on the link means `finalPackPath` already exists, which - // can only happen if a prior publishPackAtomically call on the - // same `dir` used the same `transferId`. The contract is that - // transferId is unique across all historical receives on `dir`; - // both production callers honour this (hub uses crypto.randomUUID, - // sidecar uses a per-process monotonic counter). Treat the EEXIST - // as a programmer error and surface it cleanly without running - // the recovery rms — those would destroy the earlier call's - // published pack and break any reader holding it. - if ( - err !== null && - typeof err === "object" && - "code" in err && - err.code === "EEXIST" - ) { - await fs.promises - .rm(stagingDir, { recursive: true, force: true }) - .catch(() => undefined); - throw new Error( - `transferId "${transferId}" already published in ${dir}; callers must guarantee transferId uniqueness across all historical receives`, - { cause: err }, - ); - } // Partial-publish recovery: if 3a succeeded and 3b failed we - // leave a linked-but-unindexed .pack in objects/pack/. The rm of + // leave an unindexed .pack in objects/pack/. The rm of // finalPackPath clears it; finalIdxPath cannot exist here (3b // never completed), so no rm is needed for it. Each recovery rm // is wrapped so a secondary failure (permissions, I/O) does not // mask the original publish error — the caller needs to see the // publish failure, not whatever the cleanup tripped on. - await fs.promises - .rm(stagingDir, { recursive: true, force: true }) + await runtime.fs + .remove(stagingDir, { recursive: true, force: true }) + .catch(() => undefined); + await runtime.fs + .remove(finalPackPath, { force: true }) .catch(() => undefined); - await fs.promises.rm(finalPackPath, { force: true }).catch(() => undefined); throw err; } @@ -464,14 +514,15 @@ export async function publishPackAtomically( // directory on rm failure leaks disk until external cleanup; that // is strictly preferable to retroactively destroying a successful // publish. - await fs.promises - .rm(stagingDir, { recursive: true, force: true }) + await runtime.fs + .remove(stagingDir, { recursive: true, force: true }) .catch(() => undefined); return oids; } export async function receivePackObjects( + runtime: StorageRuntime, dir: string, pack: Uint8Array, ref: string, @@ -480,7 +531,8 @@ export async function receivePackObjects( expectedOldSha: string | null, validateTree?: TreeValidator, ): Promise { - const oids = await publishPackAtomically(dir, pack, transferId); + const oids = await publishPackAtomically(runtime, dir, pack, transferId); + let currentOldSha: string | null = null; // Post-publish validation runs inside a try so any rejection path // (sha mismatch, CAS non-fast-forward, tree validator) unpublishes @@ -496,8 +548,8 @@ export async function receivePackObjects( ); } - const currentOldSha = await git - .resolveRef({ fs, dir, ref }) + currentOldSha = await git + .resolveRef({ fs: runtime.fs.git, dir, ref }) .catch(() => null); if (currentOldSha !== expectedOldSha) { @@ -510,12 +562,12 @@ export async function receivePackObjects( if (validateTree !== undefined) { const { commit } = await git.readCommit({ - fs, + fs: runtime.fs.git, dir, oid: expectedSha, }); const { tree } = await git.readTree({ - fs, + fs: runtime.fs.git, dir, oid: commit.tree, }); @@ -540,7 +592,11 @@ export async function receivePackObjects( `readBlob: path ${relPath} not found in commit ${expectedSha} tree`, ); } - const next = await git.readTree({ fs, dir, oid: entry.oid }); + const next = await git.readTree({ + fs: runtime.fs.git, + dir, + oid: entry.oid, + }); currentTree = next.tree; } const last = segments[segments.length - 1]; @@ -550,7 +606,11 @@ export async function receivePackObjects( `readBlob: path ${relPath} not found in commit ${expectedSha} tree`, ); } - const { blob } = await git.readBlob({ fs, dir, oid: blobEntry.oid }); + const { blob } = await git.readBlob({ + fs: runtime.fs.git, + dir, + oid: blobEntry.oid, + }); return blob; }; const listDir = async (relPath: string): Promise => { @@ -565,7 +625,11 @@ export async function receivePackObjects( `listDir: path ${relPath} is not a directory in commit ${expectedSha} tree`, ); } - const next = await git.readTree({ fs, dir, oid: entry.oid }); + const next = await git.readTree({ + fs: runtime.fs.git, + dir, + oid: entry.oid, + }); currentTree = next.tree; } return currentTree.map((e) => e.path); @@ -580,15 +644,25 @@ export async function receivePackObjects( } } - // Ref write happens after publish and validation so the ref never - // references a commit whose pack is unpublished or whose tree was - // rejected. - await git.writeRef({ fs, dir, ref, value: expectedSha, force: true }); - return currentOldSha; + // The pack must be durable before ref promotion begins. Once writeRef is + // attempted, an I/O error cannot prove whether the ref changed, so later + // failures must retain the pack rather than risk creating a dangling ref. + await flushRuntime(runtime); } catch (err) { - await unpublishPack(dir, transferId); + await unpublishPack(runtime, dir, transferId); + await flushRuntime(runtime); throw err; } + + await git.writeRef({ + fs: runtime.fs.git, + dir, + ref, + value: expectedSha, + force: true, + }); + await flushRuntime(runtime); + return currentOldSha; } /** @@ -612,6 +686,7 @@ export async function receivePackObjects( * The caller is responsible for ensuring `dir` is an initialized git repo. */ export async function applyPack( + runtime: StorageRuntime, dir: string, pack: Uint8Array, ref: string, @@ -623,8 +698,8 @@ export async function applyPack( // reactor's context commits, the mail-audit commits, and GC. Hold the // per-directory lock across the publish, validation, checkout, and ref // write so none of them interleave with this apply. - await withRepoDirLock(dir, async () => { - const oids = await publishPackAtomically(dir, pack, transferId); + await withRepoDirLock(runtime, dir, async () => { + const oids = await publishPackAtomically(runtime, dir, pack, transferId); // Post-publish validation runs inside a try so any rejection path // (sha mismatch, missing signature, signature failure) unpublishes @@ -642,7 +717,7 @@ export async function applyPack( if (verifyCommit !== undefined) { const { commit } = await git.readCommit({ - fs, + fs: runtime.fs.git, dir, oid: expectedSha, }); @@ -655,8 +730,12 @@ export async function applyPack( // Reconstruct the signing payload from the raw object bytes. // readCommit().payload is unreliable for SSH signatures because // isogit's withoutSignature() only handles PGP armor markers. - const { object: rawBytes } = await readRawObject(dir, expectedSha); - const payload = stripGpgsig(new TextDecoder().decode(rawBytes)); + const { object: rawBytes } = await readRawObject( + runtime, + dir, + expectedSha, + ); + const payload = stripGpgsig(decodeUTF8(rawBytes)); if (!(await verifyCommit(payload, commit.gpgsig))) { throw new Error( @@ -665,14 +744,29 @@ export async function applyPack( } } - // Checkout reads from the now-published pack; ref is written last - // so it never references a commit whose working tree is not - // materialized. - await checkoutTree(dir, expectedSha, ref); - await git.writeRef({ fs, dir, ref, value: expectedSha, force: true }); + // Make the accepted pack durable before checkout and ref promotion. + // A later failure may leave the promotion outcome uncertain, but every + // possible ref value will resolve to a durable object store. + await flushRuntime(runtime); + + // Ref is written last so it never references a commit whose working + // tree has not been materialized. + await checkoutTree(runtime, dir, expectedSha, ref); } catch (err) { - await unpublishPack(dir, transferId); + await unpublishPack(runtime, dir, transferId); + await flushRuntime(runtime); throw err; } + + // Do not unpublish after promotion begins: writeRef or the final flush can + // fail after making the new ref observable. + await git.writeRef({ + fs: runtime.fs.git, + dir, + ref, + value: expectedSha, + force: true, + }); + await flushRuntime(runtime); }); } diff --git a/vendor/intx-storage-isogit/src/pack-send.test.ts b/vendor/intx-storage-isogit/src/pack-send.test.ts index 06287d4ee..4d01b00c2 100644 --- a/vendor/intx-storage-isogit/src/pack-send.test.ts +++ b/vendor/intx-storage-isogit/src/pack-send.test.ts @@ -3,10 +3,10 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import git from "isomorphic-git"; -import { createDeployPack, createNegotiatedPack } from "./pack-send"; -import { collectReachableObjects } from "./object-walk"; -import { applyPack } from "./pack-receive"; -import { initAgentRepo } from "./init"; +import { createDeployPack, createNegotiatedPack } from "./node"; +import { collectReachableObjects } from "./node"; +import { applyPack } from "./node"; +import { initAgentRepo } from "./node"; const tempDirs: string[] = []; diff --git a/vendor/intx-storage-isogit/src/pack-send.ts b/vendor/intx-storage-isogit/src/pack-send.ts index a38970f35..20a75e3b3 100644 --- a/vendor/intx-storage-isogit/src/pack-send.ts +++ b/vendor/intx-storage-isogit/src/pack-send.ts @@ -1,7 +1,7 @@ -import fs from "node:fs"; import git from "isomorphic-git"; import { collectReachableObjects } from "./object-walk"; import { withRepoDirLock } from "./repo-lock"; +import type { StorageRuntime } from "./runtime"; /** * Create a git packfile containing all objects reachable from a ref. @@ -11,17 +11,18 @@ import { withRepoDirLock } from "./repo-lock"; * as chunked repo.pack.push frames. */ export async function createDeployPack( + runtime: StorageRuntime, dir: string, ref: string, ): Promise<{ pack: Uint8Array; commitSha: string }> { // Read under the per-directory lock so a concurrent GC pass cannot prune // a loose object out from under the reachability walk or the pack write. - return withRepoDirLock(dir, async () => { - const commitSha = await git.resolveRef({ fs, dir, ref }); - const oids = await collectReachableObjects(dir, commitSha); + return withRepoDirLock(runtime, dir, async () => { + const commitSha = await git.resolveRef({ fs: runtime.fs.git, dir, ref }); + const oids = await collectReachableObjects(runtime, dir, commitSha); const result = await git.packObjects({ - fs, + fs: runtime.fs.git, dir, oids, write: false, @@ -50,6 +51,7 @@ export async function createDeployPack( export type IncludeShaPredicate = (sha: string) => boolean | Promise; async function collectCommitChain( + runtime: StorageRuntime, dir: string, start: string, ): Promise { @@ -60,7 +62,7 @@ async function collectCommitChain( if (oid === undefined) break; if (seen.has(oid)) continue; seen.add(oid); - const { commit } = await git.readCommit({ fs, dir, oid }); + const { commit } = await git.readCommit({ fs: runtime.fs.git, dir, oid }); for (const parent of commit.parent) { if (!seen.has(parent)) queue.push(parent); } @@ -69,15 +71,16 @@ async function collectCommitChain( } async function reachableFromCommits( + runtime: StorageRuntime, dir: string, commits: readonly string[], ): Promise> { const reachable = new Set(); for (const commitOid of commits) { - const chain = await collectCommitChain(dir, commitOid); + const chain = await collectCommitChain(runtime, dir, commitOid); for (const ancestor of chain) { if (reachable.has(ancestor)) continue; - const objects = await collectReachableObjects(dir, ancestor); + const objects = await collectReachableObjects(runtime, dir, ancestor); for (const oid of objects) { reachable.add(oid); } @@ -125,6 +128,7 @@ export type CreateNegotiatedPackOptions = { }; export async function createNegotiatedPack( + runtime: StorageRuntime, dir: string, wants: readonly string[], haves: readonly string[], @@ -136,12 +140,12 @@ export async function createNegotiatedPack( } const wantedObjects = - options?.wantedObjects ?? (await reachableFromCommits(dir, wants)); + options?.wantedObjects ?? (await reachableFromCommits(runtime, dir, wants)); const knownHaves: string[] = []; for (const have of haves) { try { - await git.readCommit({ fs, dir, oid: have }); + await git.readCommit({ fs: runtime.fs.git, dir, oid: have }); knownHaves.push(have); } catch { // Unknown have — the client's advertised state is not present @@ -149,7 +153,7 @@ export async function createNegotiatedPack( } } - const haveObjects = await reachableFromCommits(dir, knownHaves); + const haveObjects = await reachableFromCommits(runtime, dir, knownHaves); const candidates: string[] = []; for (const oid of wantedObjects) { @@ -170,7 +174,7 @@ export async function createNegotiatedPack( if (oids.length === 0) return null; const result = await git.packObjects({ - fs, + fs: runtime.fs.git, dir, oids, write: false, diff --git a/vendor/intx-storage-isogit/src/peek-turns.test.ts b/vendor/intx-storage-isogit/src/peek-turns.test.ts index 23c6cfcaa..3bb3ae376 100644 --- a/vendor/intx-storage-isogit/src/peek-turns.test.ts +++ b/vendor/intx-storage-isogit/src/peek-turns.test.ts @@ -2,7 +2,7 @@ import { describe, test, expect, afterEach } from "bun:test"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { createIsogitStore } from "./index"; +import { createIsogitStore } from "./node"; import type { ConversationTurn, TokenUsage } from "@intx/types/runtime"; const ZERO_USAGE: TokenUsage = { diff --git a/vendor/intx-storage-isogit/src/repo-disk.ts b/vendor/intx-storage-isogit/src/repo-disk.ts index 77e0575d4..61f5d7b4d 100644 --- a/vendor/intx-storage-isogit/src/repo-disk.ts +++ b/vendor/intx-storage-isogit/src/repo-disk.ts @@ -1,6 +1,6 @@ -import fs from "node:fs"; -import path from "node:path"; import git from "isomorphic-git"; +import { hasCode } from "@intx/types"; +import type { StorageRuntime } from "./runtime"; /** * Disk-occupancy snapshot of an agent repo's `.git` directory. Drives the @@ -18,20 +18,18 @@ export type RepoDiskUsage = { * does not exist (a repo subtree that has not been created yet) -- absence * is a real zero, not an error to surface. */ -function countDirEntries(dir: string): number { - let entries: string[]; +async function countDirEntries( + runtime: StorageRuntime, + dir: string, +): Promise { try { - entries = fs.readdirSync(dir); + return (await runtime.fs.readdir(dir)).length; } catch (cause) { - if ( - cause instanceof Error && - (cause as NodeJS.ErrnoException).code === "ENOENT" - ) { + if (hasCode(cause) && cause.code === "ENOENT") { return 0; } throw cause; } - return entries.length; } /** @@ -39,18 +37,19 @@ function countDirEntries(dir: string): number { * un-packed per-commit objects isomorphic-git writes on each commit; their * count rising and collapsing after a repack is the pack-growth signature a * GC pass reclaims. The two-hex-char fan-out dirs plus `pack`/`info` are the - * only children of `objects/`; the latter two are skipped. + * only children of `objects/`; the latter two are skipped. Keep this filter in + * step with the synchronous Node counterpart in `node-metrics.ts`. */ -export function countLooseObjects(repoDir: string): number { - const objectsDir = path.join(repoDir, ".git", "objects"); +export async function countLooseObjects( + runtime: StorageRuntime, + repoDir: string, +): Promise { + const objectsDir = runtime.path.join(repoDir, ".git", "objects"); let fanoutDirs: string[]; try { - fanoutDirs = fs.readdirSync(objectsDir); + fanoutDirs = await runtime.fs.readdir(objectsDir); } catch (cause) { - if ( - cause instanceof Error && - (cause as NodeJS.ErrnoException).code === "ENOENT" - ) { + if (hasCode(cause) && cause.code === "ENOENT") { return 0; } throw cause; @@ -60,7 +59,10 @@ export function countLooseObjects(repoDir: string): number { if (name === "pack" || name === "info") continue; // Loose-object fan-out dirs are exactly two lowercase hex chars. if (!/^[0-9a-f]{2}$/.test(name)) continue; - total += countDirEntries(path.join(objectsDir, name)); + total += await countDirEntries( + runtime, + runtime.path.join(objectsDir, name), + ); } return total; } @@ -71,16 +73,16 @@ export function countLooseObjects(repoDir: string): number { * GC pass, so the pack count is the monotonic accumulation a write-path GC * trigger watches. */ -export function countPackFiles(repoDir: string): number { - const packDir = path.join(repoDir, ".git", "objects", "pack"); +export async function countPackFiles( + runtime: StorageRuntime, + repoDir: string, +): Promise { + const packDir = runtime.path.join(repoDir, ".git", "objects", "pack"); let entries: string[]; try { - entries = fs.readdirSync(packDir); + entries = await runtime.fs.readdir(packDir); } catch (cause) { - if ( - cause instanceof Error && - (cause as NodeJS.ErrnoException).code === "ENOENT" - ) { + if (hasCode(cause) && cause.code === "ENOENT") { return 0; } throw cause; @@ -91,31 +93,31 @@ export function countPackFiles(repoDir: string): number { /** * Total byte size of the repo's `.git` directory (loose + pack + refs + * logs). A coarse repo-size proxy for the disk-pressure warning. Walks the - * tree with `fs.lstatSync`; bounded by the repo size, which is the thing - * being measured. + * tree through the configured filesystem; bounded by the repo size, which is + * the thing being measured. */ -export function gitBytes(repoDir: string): number { - const gitDir = path.join(repoDir, ".git"); +export async function gitBytes( + runtime: StorageRuntime, + repoDir: string, +): Promise { + const gitDir = runtime.path.join(repoDir, ".git"); let total = 0; const stack: string[] = [gitDir]; while (stack.length > 0) { const current = stack.pop(); if (current === undefined) break; - let stat: fs.Stats; + let stat; try { - stat = fs.lstatSync(current); + stat = await runtime.fs.lstat(current); } catch (cause) { - if ( - cause instanceof Error && - (cause as NodeJS.ErrnoException).code === "ENOENT" - ) { + if (hasCode(cause) && cause.code === "ENOENT") { continue; } throw cause; } if (stat.isDirectory()) { - for (const child of fs.readdirSync(current)) { - stack.push(path.join(current, child)); + for (const child of await runtime.fs.readdir(current)) { + stack.push(runtime.path.join(current, child)); } } else if (stat.isFile()) { total += stat.size; @@ -135,10 +137,17 @@ export type RepoObjectCounts = { looseObjectCount: number; }; -export function repoObjectCounts(dir: string): RepoObjectCounts { +export async function repoObjectCounts( + runtime: StorageRuntime, + dir: string, +): Promise { + const [packCount, looseObjectCount] = await Promise.all([ + countPackFiles(runtime, dir), + countLooseObjects(runtime, dir), + ]); return { - packCount: countPackFiles(dir), - looseObjectCount: countLooseObjects(dir), + packCount, + looseObjectCount, }; } @@ -146,10 +155,17 @@ export function repoObjectCounts(dir: string): RepoObjectCounts { * Snapshot the GC-relevant disk counters for the agent repo at `dir`, * including the full `.git` byte walk. */ -export function repoDiskUsage(dir: string): RepoDiskUsage { +export async function repoDiskUsage( + runtime: StorageRuntime, + dir: string, +): Promise { + const [bytes, counts] = await Promise.all([ + gitBytes(runtime, dir), + repoObjectCounts(runtime, dir), + ]); return { - gitBytes: gitBytes(dir), - ...repoObjectCounts(dir), + gitBytes: bytes, + ...counts, }; } @@ -163,13 +179,14 @@ export function repoDiskUsage(dir: string): RepoDiskUsage { * complete. */ export async function listRepoRefs( + runtime: StorageRuntime, dir: string, ): Promise<{ ref: string; oid: string }[]> { - const branches = await git.listBranches({ fs, dir }); + const branches = await git.listBranches({ fs: runtime.fs.git, dir }); const refs: { ref: string; oid: string }[] = []; for (const branch of branches) { const ref = `refs/heads/${branch}`; - const oid = await git.resolveRef({ fs, dir, ref }); + const oid = await git.resolveRef({ fs: runtime.fs.git, dir, ref }); refs.push({ ref, oid }); } return refs; diff --git a/vendor/intx-storage-isogit/src/repo-lock.ts b/vendor/intx-storage-isogit/src/repo-lock.ts index 859ce62a4..ae0e71b4b 100644 --- a/vendor/intx-storage-isogit/src/repo-lock.ts +++ b/vendor/intx-storage-isogit/src/repo-lock.ts @@ -1,4 +1,4 @@ -import { resolve } from "node:path"; +import type { StorageRuntime } from "./runtime"; /** * Process-wide per-directory serialization for agent-repo object-store @@ -30,10 +30,11 @@ import { resolve } from "node:path"; const locks = new Map>(); export async function withRepoDirLock( + runtime: StorageRuntime, dir: string, fn: () => Promise, ): Promise { - const key = resolve(dir); + const key = runtime.path.resolve(dir); const previous = locks.get(key) ?? Promise.resolve(); let releaseFn: () => void = () => undefined; const tail = new Promise((res) => { diff --git a/vendor/intx-storage-isogit/src/runtime.test.ts b/vendor/intx-storage-isogit/src/runtime.test.ts new file mode 100644 index 000000000..5c4319d29 --- /dev/null +++ b/vendor/intx-storage-isogit/src/runtime.test.ts @@ -0,0 +1,192 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import type { CallbackFsClient, FsClient } from "isomorphic-git"; +import { createIsogitStorage } from "./index"; +import { createNodeIsogitRuntime } from "./node-runtime"; +import { normalizeRuntime, type IsogitRuntime } from "./runtime"; + +const tempDirs: string[] = []; + +async function tempDir(): Promise { + const dir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), "isogit-runtime-"), + ); + tempDirs.push(dir); + return dir; +} + +afterEach(async () => { + const dirs = tempDirs.splice(0); + await Promise.all( + dirs.map((dir) => fs.promises.rm(dir, { recursive: true, force: true })), + ); +}); + +function callbackFs(): CallbackFsClient { + return { + readFile: fs.readFile.bind(fs), + writeFile: fs.writeFile.bind(fs), + unlink: fs.unlink.bind(fs), + readdir: fs.readdir.bind(fs), + mkdir: fs.mkdir.bind(fs), + rmdir: fs.rmdir.bind(fs), + stat: fs.stat.bind(fs), + lstat: fs.lstat.bind(fs), + readlink: fs.readlink.bind(fs), + symlink: fs.symlink.bind(fs), + chmod: fs.chmod.bind(fs), + }; +} + +function runtimeWith(fsClient: FsClient): IsogitRuntime { + return { + fs: fsClient, + path, + rename: (oldPath, newPath) => fs.promises.rename(oldPath, newPath), + }; +} + +describe("normalizeRuntime", () => { + test("normalizes promise filesystem operations", async () => { + const dir = await tempDir(); + const runtime = normalizeRuntime(createNodeIsogitRuntime()); + const nested = path.join(dir, "one", "two"); + const filepath = path.join(nested, "value.txt"); + + await runtime.fs.mkdir(nested, { recursive: true }); + await runtime.fs.writeFile(filepath, "promise client"); + + expect(await runtime.fs.readTextFile(filepath)).toBe("promise client"); + await runtime.fs.remove(path.join(dir, "one"), { + force: true, + recursive: true, + }); + await expect(runtime.fs.stat(filepath)).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + test("normalizes callback filesystem operations", async () => { + const dir = await tempDir(); + const runtime = normalizeRuntime(runtimeWith(callbackFs())); + const filepath = path.join(dir, "callback.txt"); + + await runtime.fs.writeFile(filepath, "callback client"); + + expect(await runtime.fs.readTextFile(filepath)).toBe("callback client"); + }); + + test("preserves errors returned by the filesystem", async () => { + const expected = Object.assign(new Error("permission denied"), { + code: "EACCES", + }); + const fsClient = callbackFs(); + fsClient.readFile = ( + _filepath: unknown, + callback: (cause: unknown) => void, + ) => { + callback(expected); + }; + const runtime = normalizeRuntime(runtimeWith(fsClient)); + + await expect(runtime.fs.readFile("unreadable")).rejects.toBe(expected); + }); + + test("routes atomic rename through the host runtime", async () => { + const calls: [string, string][] = []; + const host = createNodeIsogitRuntime(); + const runtime = normalizeRuntime({ + ...host, + rename: (oldPath, newPath) => { + calls.push([oldPath, newPath]); + return Promise.resolve(); + }, + }); + + await runtime.fs.rename("staging.pack", "final.pack"); + + expect(calls).toEqual([["staging.pack", "final.pack"]]); + }); +}); + +test("storage flushes after completed mutation boundaries", async () => { + const dir = await tempDir(); + let flushes = 0; + const storage = createIsogitStorage({ + ...createNodeIsogitRuntime(), + flush: () => { + flushes += 1; + return Promise.resolve(); + }, + }); + + await storage.initAgentRepo(dir); + await storage.createAndSwitchBranch(dir, "flush-test"); + + expect(flushes).toBe(3); + + await fs.promises.rm(path.join(dir, "state"), { recursive: true }); + await storage.initAgentRepo(dir); + + expect(flushes).toBe(4); + await expect(fs.promises.stat(path.join(dir, "state"))).resolves.toBeTruthy(); +}); + +test("flushes commit objects and the published branch ref exactly once", async () => { + const dir = await tempDir(); + const refPath = path.join(dir, ".git", "refs", "heads", "main"); + const observedRefs: (string | null)[] = []; + const storage = createIsogitStorage({ + ...createNodeIsogitRuntime(), + flush: async () => { + const ref = await fs.promises.readFile(refPath, "utf8").catch(() => null); + observedRefs.push(ref?.trim() ?? null); + }, + }); + const store = await storage.createIsogitStore(dir); + const initialRef = (await fs.promises.readFile(refPath, "utf8")).trim(); + observedRefs.length = 0; + + await store.writeTurns([ + { + role: "user", + content: [{ type: "text", text: "durable before publication" }], + timestamp: 1, + }, + ]); + const committed = await store.commit({ message: "durable commit" }); + + expect(observedRefs).toEqual([initialRef, committed.hash]); +}); + +test("flushes a new branch before publishing it through HEAD", async () => { + const dir = await tempDir(); + const headPath = path.join(dir, ".git", "HEAD"); + const branchPath = path.join(dir, ".git", "refs", "heads", "durable-branch"); + const observations: { branchExists: boolean; head: string }[] = []; + const storage = createIsogitStorage({ + ...createNodeIsogitRuntime(), + flush: async () => { + const [head, branchExists] = await Promise.all([ + fs.promises.readFile(headPath, "utf8"), + fs.promises + .stat(branchPath) + .then(() => true) + .catch(() => false), + ]); + observations.push({ branchExists, head: head.trim() }); + }, + }); + + await storage.initAgentRepo(dir); + observations.length = 0; + await storage.createAndSwitchBranch(dir, "durable-branch"); + + expect(observations).toEqual([ + { branchExists: true, head: "ref: refs/heads/main" }, + { branchExists: true, head: "ref: refs/heads/durable-branch" }, + ]); +}); diff --git a/vendor/intx-storage-isogit/src/runtime.ts b/vendor/intx-storage-isogit/src/runtime.ts new file mode 100644 index 000000000..814194670 --- /dev/null +++ b/vendor/intx-storage-isogit/src/runtime.ts @@ -0,0 +1,282 @@ +import type { FsClient } from "isomorphic-git"; +import { hasCode } from "@intx/types"; + +const textDecoder = new TextDecoder(); + +export function decodeUTF8(bytes: Uint8Array): string { + return textDecoder.decode(bytes); +} + +export interface IsogitPath { + join(...parts: string[]): string; + relative(from: string, to: string): string; + resolve(filepath: string): string; +} + +/** Minimal host capabilities required by the storage implementation. */ +export interface IsogitRuntime { + readonly fs: FsClient; + readonly path: IsogitPath; + /** Atomically move one namespace entry from `oldPath` to `newPath`. */ + rename(oldPath: string, newPath: string): Promise; + /** Flush pending backend writes at a completed mutation boundary. */ + flush?(): Promise; +} + +export interface FileSystemStat { + readonly size: number; + isDirectory(): boolean; + isFile(): boolean; +} + +type WriteFileOptions = { + mode?: number; +}; + +type RemoveOptions = { + force?: boolean; + recursive?: boolean; +}; + +type FsMethod = + | "lstat" + | "mkdir" + | "readFile" + | "readdir" + | "rmdir" + | "stat" + | "unlink" + | "writeFile"; + +type UnknownOperation = (...args: unknown[]) => unknown; + +function operation( + value: unknown, + method: FsMethod, +): UnknownOperation | undefined { + if (typeof value !== "object" || value === null) { + return undefined; + } + const candidate: unknown = Reflect.get(value, method); + if (typeof candidate !== "function") { + return undefined; + } + return (...args) => { + const result: unknown = Reflect.apply(candidate, value, args); + return result; + }; +} + +/** + * Invoke either shape accepted by isomorphic-git without changing errors. + * Promise clients expose methods under `promises`; callback clients expose + * the same methods on the client itself. + */ +function callFs( + fs: FsClient, + method: FsMethod, + args: readonly unknown[], +): Promise { + const promises = + typeof fs === "object" && fs !== null && "promises" in fs + ? Reflect.get(fs, "promises") + : undefined; + const promiseOperation = operation(promises, method); + if (promiseOperation !== undefined) { + return Promise.resolve(promiseOperation(...args)); + } + + const callbackOperation = operation(fs, method); + if (callbackOperation === undefined) { + return Promise.reject( + new TypeError(`Filesystem client does not implement ${method}`), + ); + } + return new Promise((resolve, reject) => { + callbackOperation(...args, (cause: unknown, result: unknown) => { + if (cause !== null && cause !== undefined) { + reject(cause); + } else { + resolve(result); + } + }); + }); +} + +function fileStat(value: unknown, method: FsMethod): FileSystemStat { + if (typeof value !== "object" || value === null) { + throw new TypeError(`Filesystem ${method} returned an invalid stat`); + } + const size: unknown = Reflect.get(value, "size"); + const isDirectory: unknown = Reflect.get(value, "isDirectory"); + const isFile: unknown = Reflect.get(value, "isFile"); + if ( + typeof size !== "number" || + typeof isDirectory !== "function" || + typeof isFile !== "function" + ) { + throw new TypeError(`Filesystem ${method} returned an invalid stat`); + } + return { + size, + isDirectory: () => Boolean(Reflect.apply(isDirectory, value, [])), + isFile: () => Boolean(Reflect.apply(isFile, value, [])), + }; +} + +function directoryEntries(value: unknown): string[] { + if (!Array.isArray(value)) { + throw new TypeError("Filesystem readdir returned a non-array value"); + } + return value.map((entry) => { + if (typeof entry !== "string") { + throw new TypeError("Filesystem readdir returned a non-string entry"); + } + return entry; + }); +} + +export interface StorageFileSystem { + readonly git: FsClient; + access(filepath: string): Promise; + readFile(filepath: string): Promise; + readTextFile(filepath: string): Promise; + writeFile( + filepath: string, + data: string | Uint8Array, + options?: WriteFileOptions, + ): Promise; + readdir(filepath: string): Promise; + mkdir(filepath: string, options?: { recursive?: boolean }): Promise; + stat(filepath: string): Promise; + lstat(filepath: string): Promise; + remove(filepath: string, options?: RemoveOptions): Promise; + rename(oldPath: string, newPath: string): Promise; +} + +/** Normalized internal view derived from the public runtime contract. */ +export interface StorageRuntime { + readonly fs: StorageFileSystem; + readonly path: IsogitPath; + flush?(): Promise; +} + +function createStorageFileSystem(runtime: IsogitRuntime): StorageFileSystem { + const stat = async (filepath: string) => + fileStat(await callFs(runtime.fs, "stat", [filepath]), "stat"); + const lstat = async (filepath: string) => + fileStat(await callFs(runtime.fs, "lstat", [filepath]), "lstat"); + + async function mkdir( + filepath: string, + options?: { recursive?: boolean }, + ): Promise { + if (options?.recursive !== true) { + await callFs(runtime.fs, "mkdir", [filepath]); + return; + } + + try { + await callFs(runtime.fs, "mkdir", [filepath]); + return; + } catch (cause) { + if (hasCode(cause) && cause.code === "EEXIST") { + if ((await stat(filepath)).isDirectory()) return; + throw cause; + } + if (!hasCode(cause) || cause.code !== "ENOENT") throw cause; + } + + const resolved = runtime.path.resolve(filepath); + const parent = runtime.path.resolve(runtime.path.join(resolved, "..")); + if (parent === resolved) { + throw new Error(`Cannot create parent directory for ${filepath}`); + } + await mkdir(parent, { recursive: true }); + try { + await callFs(runtime.fs, "mkdir", [filepath]); + } catch (cause) { + if (!hasCode(cause) || cause.code !== "EEXIST") throw cause; + if (!(await stat(filepath)).isDirectory()) throw cause; + } + } + + async function remove( + filepath: string, + options: RemoveOptions = {}, + ): Promise { + let entry: FileSystemStat; + try { + entry = await lstat(filepath); + } catch (cause) { + if (options.force === true && hasCode(cause) && cause.code === "ENOENT") { + return; + } + throw cause; + } + + if (!entry.isDirectory()) { + await callFs(runtime.fs, "unlink", [filepath]); + return; + } + + if (options.recursive === true) { + const entries = directoryEntries( + await callFs(runtime.fs, "readdir", [filepath]), + ); + for (const name of entries) { + await remove(runtime.path.join(filepath, name), { + recursive: true, + ...(options.force === undefined ? {} : { force: options.force }), + }); + } + } + await callFs(runtime.fs, "rmdir", [filepath]); + } + + const readFile = async (filepath: string): Promise => { + const value = await callFs(runtime.fs, "readFile", [filepath]); + if (value instanceof Uint8Array) { + return value; + } + if (value instanceof ArrayBuffer) { + return new Uint8Array(value); + } + throw new TypeError("Filesystem readFile returned a non-binary value"); + }; + + return { + git: runtime.fs, + access: async (filepath) => { + await lstat(filepath); + }, + readFile, + readTextFile: async (filepath) => decodeUTF8(await readFile(filepath)), + writeFile: async (filepath, data, options) => { + const args = + options === undefined ? [filepath, data] : [filepath, data, options]; + await callFs(runtime.fs, "writeFile", args); + }, + readdir: async (filepath) => + directoryEntries(await callFs(runtime.fs, "readdir", [filepath])), + mkdir, + stat, + lstat, + remove, + rename: (oldPath, newPath) => runtime.rename(oldPath, newPath), + }; +} + +export function normalizeRuntime(runtime: IsogitRuntime): StorageRuntime { + return { + fs: createStorageFileSystem(runtime), + path: runtime.path, + ...(runtime.flush === undefined + ? {} + : { flush: () => runtime.flush?.() ?? Promise.resolve() }), + }; +} + +export async function flushRuntime(runtime: StorageRuntime): Promise { + await runtime.flush?.(); +} diff --git a/vendor/intx-storage-isogit/src/store.test.ts b/vendor/intx-storage-isogit/src/store.test.ts index 6f1e90393..112e17809 100644 --- a/vendor/intx-storage-isogit/src/store.test.ts +++ b/vendor/intx-storage-isogit/src/store.test.ts @@ -4,16 +4,18 @@ import os from "node:os"; import path from "node:path"; import git from "isomorphic-git"; import { + createIsogitStorage, createIsogitStore, currentBranch, createAndSwitchBranch, switchBranch, listBranches, logHistory, -} from "./index"; +} from "./node"; +import { createNodeIsogitRuntime } from "./node-runtime"; import { base64Encode } from "@intx/types"; -import { IsogitStore } from "./store"; -import { initAgentRepo } from "./init"; +import { IsogitStore } from "./node"; +import { initAgentRepo } from "./node"; import type { AssistantTurn, ConversationTurn, @@ -288,6 +290,26 @@ describe("readAt", () => { const atFirst = await store.readAt(first.hash); expect(atFirst).toEqual(v1); }); + + test("surfaces corrupt object errors", async () => { + const dir = await tempDir(); + const store = await createIsogitStore(dir); + + await store.writeTurns([]); + const commit = await store.commit({ message: "checkpoint" }); + await fs.promises.writeFile( + path.join( + dir, + ".git", + "objects", + commit.hash.slice(0, 2), + commit.hash.slice(2), + ), + "corrupt object", + ); + + await expect(store.readAt(commit.hash)).rejects.toThrow(); + }); }); function makeAuditRecord(overrides: Partial = {}): AuditRecord { @@ -594,7 +616,13 @@ describe("error store", () => { await store.commitErrors([record]); let thrown: Error | undefined; try { - await store.commitErrors([record]); + await store.commitErrors([ + makeErrorRecord({ + seq: 1, + category: "credential_failure", + message: "Different failure", + }), + ]); } catch (cause) { thrown = cause instanceof Error ? cause : new Error(String(cause)); } @@ -610,7 +638,11 @@ describe("error store", () => { // Second batch contains a duplicate of the first record and a new one. // The duplicate should be caught in pre-flight before any writes. - const dup = makeErrorRecord({ seq: 1, category: "first" }); + const dup = makeErrorRecord({ + seq: 1, + category: "first", + message: "Conflicting duplicate", + }); const extra = makeErrorRecord({ seq: 2, category: "second" }); let thrown: Error | undefined; try { @@ -644,6 +676,202 @@ describe("error store", () => { } expect(thrown?.message).toContain("unsafe characters"); }); + + test("commitErrors accepts an exact retry without another commit", async () => { + const dir = await tempDir(); + const store = await createAuditStore(dir); + const record = makeErrorRecord(); + + await store.commitErrors([record]); + await store.commitErrors([record]); + + const entries = await git.log({ fs, dir }); + expect( + entries.filter( + (entry) => entry.commit.message.trimEnd() === "Record 1 error record", + ), + ).toHaveLength(1); + }); +}); + +describe("audit and error durability retries", () => { + function withFailingFlush( + fail: (flushCount: number) => Promise | void, + ): ReturnType { + const runtime = createNodeIsogitRuntime(); + let flushCount = 0; + return createIsogitStorage({ + ...runtime, + async flush() { + flushCount += 1; + await fail(flushCount); + }, + }); + } + + test("commitAudit retries when the failed ref flush left HEAD advanced", async () => { + const dir = await tempDir(); + await initAgentRepo(dir); + let failed = false; + const storage = withFailingFlush((flushCount) => { + if (!failed && flushCount === 3) { + failed = true; + throw new Error("injected post-ref flush failure"); + } + }); + const store = await storage.createIsogitStore(dir); + const record = makeAuditRecord(); + + await expect(store.commitAudit([record])).rejects.toThrow( + "Ref publication", + ); + await store.commitAudit([record]); + + expect(await store.loadAudit("session-1")).toEqual([record]); + const entries = await git.log({ fs, dir }); + expect( + entries.filter((entry) => + entry.commit.message.startsWith("Record 1 tool audit record"), + ), + ).toHaveLength(1); + }); + + test("commitAudit retries when the failed ref flush left HEAD unchanged", async () => { + const dir = await tempDir(); + await initAgentRepo(dir); + const initialOid = await git.resolveRef({ fs, dir, ref: "HEAD" }); + let failed = false; + const storage = withFailingFlush(async (flushCount) => { + if (!failed && flushCount === 3) { + failed = true; + await fs.promises.writeFile( + path.join(dir, ".git", "refs", "heads", "main"), + `${initialOid}\n`, + ); + throw new Error("injected rolled-back ref flush failure"); + } + }); + const store = await storage.createIsogitStore(dir); + const record = makeAuditRecord(); + + await expect(store.commitAudit([record])).rejects.toThrow( + "Ref publication", + ); + expect(await git.resolveRef({ fs, dir, ref: "HEAD" })).toBe(initialOid); + + await store.commitAudit([record]); + + expect(await git.resolveRef({ fs, dir, ref: "HEAD" })).not.toBe(initialOid); + expect(await store.loadAudit("session-1")).toEqual([record]); + }); + + test("commitErrors retries an advanced record with accumulated errors", async () => { + const dir = await tempDir(); + await initAgentRepo(dir); + let failed = false; + const storage = withFailingFlush((flushCount) => { + if (!failed && flushCount === 3) { + failed = true; + throw new Error("injected post-ref flush failure"); + } + }); + const store = await storage.createIsogitStore(dir); + const first = makeErrorRecord({ seq: 1, category: "first" }); + const second = makeErrorRecord({ seq: 2, category: "second" }); + + await expect(store.commitErrors([first])).rejects.toThrow( + "Ref publication", + ); + await store.commitErrors([first, second]); + + const [entry] = await git.log({ fs, dir, depth: 1 }); + expect(entry?.commit.message.trimEnd()).toBe("Record 1 error record"); + expect( + JSON.parse( + await fs.promises.readFile( + path.join( + dir, + "state", + "errors", + "session-1", + "00000002-second.json", + ), + "utf8", + ), + ), + ).toEqual(second); + }); + + test("a failed record commit cannot leak into the next cycle commit", async () => { + const dir = await tempDir(); + await initAgentRepo(dir); + let failed = false; + const storage = withFailingFlush((flushCount) => { + if (!failed && flushCount === 2) { + failed = true; + throw new Error("injected object flush failure"); + } + }); + const store = await storage.createIsogitStore(dir); + const record = makeAuditRecord(); + + await expect(store.commitAudit([record])).rejects.toThrow( + "injected object flush failure", + ); + await store.writeTurns([]); + const cycle = await store.commit({ message: "Cycle: inference" }); + + await expect( + git.readBlob({ + fs, + dir, + oid: cycle.hash, + filepath: "state/audit/session-1/call-1.json", + }), + ).rejects.toMatchObject({ code: "NotFoundError" }); + + await store.commitAudit([record]); + const [entry] = await git.log({ fs, dir, depth: 1 }); + expect(entry?.commit.message.trimEnd()).toBe("Record 1 tool audit record"); + }); + + test("a failed cycle commit cannot leak into the next error commit", async () => { + const dir = await tempDir(); + await initAgentRepo(dir); + let failed = false; + const storage = withFailingFlush((flushCount) => { + if (!failed && flushCount === 2) { + failed = true; + throw new Error("injected cycle object flush failure"); + } + }); + const store = await storage.createIsogitStore(dir); + + await store.writeTurns([ + { + role: "user", + content: [{ type: "text", text: "uncommitted cycle" }], + timestamp: 1, + }, + ]); + await expect(store.commit({ message: "Cycle: inference" })).rejects.toThrow( + "injected cycle object flush failure", + ); + + await store.commitErrors([makeErrorRecord()]); + const head = await git.resolveRef({ fs, dir, ref: "HEAD" }); + await expect( + git.readBlob({ fs, dir, oid: head, filepath: "turns.jsonl" }), + ).rejects.toMatchObject({ code: "NotFoundError" }); + await expect( + git.readBlob({ + fs, + dir, + oid: head, + filepath: "state/errors/session-1/00000001-credential_failure.json", + }), + ).resolves.toBeTruthy(); + }); }); describe("load reads from the working tree", () => { diff --git a/vendor/intx-storage-isogit/src/store.ts b/vendor/intx-storage-isogit/src/store.ts index 38aaa8e9b..3cbb096e5 100644 --- a/vendor/intx-storage-isogit/src/store.ts +++ b/vendor/intx-storage-isogit/src/store.ts @@ -1,5 +1,3 @@ -import fs from "node:fs"; -import path from "node:path"; import git from "isomorphic-git"; import { ApprovalSnapshot, @@ -24,9 +22,15 @@ import { } from "@intx/types/audit"; import { AUTHOR } from "./init"; import type { CommitSigner } from "./signer"; -import { buildSigningArgs } from "./commit-helpers"; +import { + buildSigningArgs, + commitDurably, + restoreIndexAfterFailedCommit, +} from "./commit-helpers"; import { withRepoDirLock } from "./repo-lock"; import { maybeGCUnderLock, type GCPolicy } from "./gc"; +import { decodeUTF8, flushRuntime, type StorageRuntime } from "./runtime"; +import { hasCode } from "@intx/types"; const TURNS_FILE = "turns.jsonl"; const PROMPT_FILE = "prompt.jsonl"; @@ -139,20 +143,21 @@ function parseMetadata(raw: unknown): MetadataData { * tool). Any non-absence read error still surfaces. */ async function tolerantLog( + runtime: StorageRuntime, dir: string, limit: number, ): Promise>[]> { const out: Awaited>[] = []; let oid: string | undefined; try { - oid = await git.resolveRef({ fs, dir, ref: "HEAD" }); + oid = await git.resolveRef({ fs: runtime.fs.git, dir, ref: "HEAD" }); } catch { return out; } while (oid !== undefined && out.length < limit) { let entry: Awaited>; try { - entry = await git.readCommit({ fs, dir, oid }); + entry = await git.readCommit({ fs: runtime.fs.git, dir, oid }); } catch (err) { if (err instanceof Error && "code" in err && err.code === "NotFoundError") break; @@ -165,10 +170,11 @@ async function tolerantLog( } async function readCommitLog( + runtime: StorageRuntime, dir: string, limit: number, ): Promise { - const entries = await tolerantLog(dir, limit); + const entries = await tolerantLog(runtime, dir, limit); return entries.map((e) => { const base = { hash: e.oid, @@ -183,6 +189,13 @@ async function readCommitLog( const AUDIT_DIR = "state/audit"; const ERRORS_DIR = "state/errors"; +type DurableRecordFile = { + filepath: string; + directory: string; + contents: string; + duplicateMessage: string; +}; + const SAFE_PATH_SEGMENT = /^[a-zA-Z0-9_-]+$/; const UNSAFE_FILENAME_CHARS = /[^a-zA-Z0-9_-]/g; @@ -208,12 +221,15 @@ function sanitizeCallId(callId: string): string { return callId.replace(UNSAFE_FILENAME_CHARS, "_"); } -async function pathExists(fullPath: string): Promise { +async function pathExists( + runtime: StorageRuntime, + fullPath: string, +): Promise { try { - await fs.promises.access(fullPath); + await runtime.fs.access(fullPath); return true; } catch (cause) { - if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") { + if (hasCode(cause) && cause.code === "ENOENT") { return false; } throw cause; @@ -233,12 +249,18 @@ function decodeJsonlLines(text: string): unknown[] { } async function readBlobAtCommit( + runtime: StorageRuntime, dir: string, oid: string, filepath: string, ): Promise { try { - const { blob } = await git.readBlob({ fs, dir, oid, filepath }); + const { blob } = await git.readBlob({ + fs: runtime.fs.git, + dir, + oid, + filepath, + }); return blob; } catch (cause) { if ( @@ -248,7 +270,7 @@ async function readBlobAtCommit( ) { return null; } - return null; + throw cause; } } @@ -307,13 +329,20 @@ export interface DurableMirrorReads { export class IsogitStore implements ContextStore, AuditStore, DurableMirrorReads { + private readonly runtime: StorageRuntime; private readonly dir: string; private readonly signer: CommitSigner | undefined; private readonly gcPolicy: GCPolicy | undefined; private pendingConnectorState: ConnectorThreadState | null = null; private lastTurns: ConversationTurn[] = []; - constructor(dir: string, signer?: CommitSigner, gcPolicy?: GCPolicy) { + constructor( + runtime: StorageRuntime, + dir: string, + signer?: CommitSigner, + gcPolicy?: GCPolicy, + ) { + this.runtime = runtime; this.dir = dir; this.signer = signer; this.gcPolicy = gcPolicy; @@ -327,7 +356,100 @@ export class IsogitStore // the configured policy. A no-op when no policy was supplied. private async maybeGC(): Promise { if (this.gcPolicy === undefined) return; - await maybeGCUnderLock(this.dir, this.gcPolicy); + await maybeGCUnderLock(this.runtime, this.dir, this.gcPolicy); + } + + /** Persist append-only record files with retry-safe publication semantics. */ + private async commitRecordFiles( + files: readonly DurableRecordFile[], + commitMessage: (count: number) => string, + ): Promise { + const prepared: { + file: DurableRecordFile; + exists: boolean; + committed: boolean; + }[] = []; + const headOid = await git.resolveRef({ + fs: this.runtime.fs.git, + dir: this.dir, + ref: "HEAD", + }); + + for (const file of files) { + const committedBlob = await readBlobAtCommit( + this.runtime, + this.dir, + headOid, + file.filepath, + ); + const committed = committedBlob !== null; + if ( + committedBlob !== null && + decodeUTF8(committedBlob) !== file.contents + ) { + throw new Error(file.duplicateMessage); + } + + const fullPath = this.runtime.path.join(this.dir, file.filepath); + try { + const existing = await this.runtime.fs.readTextFile(fullPath); + if (existing !== file.contents) { + throw new Error(file.duplicateMessage); + } + prepared.push({ file, exists: true, committed }); + } catch (cause) { + if (hasCode(cause) && cause.code === "ENOENT") { + prepared.push({ file, exists: false, committed }); + } else { + throw cause; + } + } + } + + try { + for (const { file, exists } of prepared) { + if (!exists) { + await this.runtime.fs.mkdir(file.directory, { recursive: true }); + await this.runtime.fs.writeFile( + this.runtime.path.join(this.dir, file.filepath), + file.contents, + ); + } + await git.add({ + fs: this.runtime.fs.git, + dir: this.dir, + filepath: file.filepath, + }); + } + + const changedCount = prepared.filter( + ({ committed }) => !committed, + ).length; + if (changedCount === 0) { + await flushRuntime(this.runtime); + } else { + await commitDurably(this.runtime, this.dir, { + message: commitMessage(changedCount), + author: AUTHOR, + ...this.signingArgs(), + }); + } + } catch (cause) { + try { + await restoreIndexAfterFailedCommit( + this.runtime, + this.dir, + prepared.map(({ file }) => file.filepath), + ); + } catch (resetCause) { + throw new Error("Could not restore the record index after failure", { + cause: new AggregateError([cause, resetCause]), + }); + } + throw cause; + } + await this.maybeGC(); + await flushRuntime(this.runtime); } setConnectorState(state: ConnectorThreadState | null): void { @@ -340,11 +462,11 @@ export class IsogitStore tokenUsage: TokenUsage; connectorState: ConnectorThreadState | null; }> { - const turnsPath = path.join(this.dir, TURNS_FILE); + const turnsPath = this.runtime.path.join(this.dir, TURNS_FILE); let turns: ConversationTurn[] = []; - if (await pathExists(turnsPath)) { - const text = await fs.promises.readFile(turnsPath, "utf-8"); + if (await pathExists(this.runtime, turnsPath)) { + const text = await this.runtime.fs.readTextFile(turnsPath); turns = parseTurns(text); } @@ -357,15 +479,15 @@ export class IsogitStore tokenUsage: TokenUsage; connectorState: ConnectorThreadState | null; }> { - const metadataPath = path.join(this.dir, METADATA_FILE); - if (!(await pathExists(metadataPath))) { + const metadataPath = this.runtime.path.join(this.dir, METADATA_FILE); + if (!(await pathExists(this.runtime, metadataPath))) { return { pendingOperations: [], tokenUsage: { ...EMPTY_USAGE }, connectorState: null, }; } - const text = await fs.promises.readFile(metadataPath, "utf-8"); + const text = await this.runtime.fs.readTextFile(metadataPath); const parsed: unknown = JSON.parse(text); const data = parseMetadata(parsed); return { @@ -379,7 +501,7 @@ export class IsogitStore options: { message: string }, _signal?: AbortSignal, ): Promise { - return withRepoDirLock(this.dir, async () => { + return withRepoDirLock(this.runtime, this.dir, async () => { const tracked = [ TURNS_FILE, PROMPT_FILE, @@ -387,32 +509,51 @@ export class IsogitStore MANIFEST_FILE, METADATA_FILE, ]; + const stagedFilepaths: string[] = []; for (const filepath of tracked) { - const fullPath = path.join(this.dir, filepath); - if (await pathExists(fullPath)) { - await git.add({ fs, dir: this.dir, filepath }); + const fullPath = this.runtime.path.join(this.dir, filepath); + if (await pathExists(this.runtime, fullPath)) { + stagedFilepaths.push(filepath); } } - const blobsDir = path.join(this.dir, TOOL_OUTPUT_DIR); - if (await pathExists(blobsDir)) { - const entries = await fs.promises.readdir(blobsDir); + const blobsDir = this.runtime.path.join(this.dir, TOOL_OUTPUT_DIR); + if (await pathExists(this.runtime, blobsDir)) { + const entries = await this.runtime.fs.readdir(blobsDir); for (const entry of entries) { + stagedFilepaths.push(`${TOOL_OUTPUT_DIR}/${entry}`); + } + } + + let oid: string; + try { + for (const filepath of stagedFilepaths) { await git.add({ - fs, + fs: this.runtime.fs.git, dir: this.dir, - filepath: `${TOOL_OUTPUT_DIR}/${entry}`, + filepath, }); } - } - const oid = await git.commit({ - fs, - dir: this.dir, - message: options.message, - author: AUTHOR, - ...this.signingArgs(), - }); + oid = await commitDurably(this.runtime, this.dir, { + message: options.message, + author: AUTHOR, + ...this.signingArgs(), + }); + } catch (cause) { + try { + await restoreIndexAfterFailedCommit( + this.runtime, + this.dir, + stagedFilepaths, + ); + } catch (resetCause) { + throw new Error("Could not restore the context index after failure", { + cause: new AggregateError([cause, resetCause]), + }); + } + throw cause; + } const described = await this.describeHead(oid, options.message); await this.maybeGC(); @@ -424,7 +565,11 @@ export class IsogitStore expectedOid: string, message: string, ): Promise { - const entries = await git.log({ fs, dir: this.dir, depth: 2 }); + const entries = await git.log({ + fs: this.runtime.fs.git, + dir: this.dir, + depth: 2, + }); const entry = entries[0]; if (entry === undefined || entry.oid !== expectedOid) { throw new Error( @@ -441,20 +586,26 @@ export class IsogitStore } async branch(name: string, _signal?: AbortSignal): Promise { - await git.branch({ fs, dir: this.dir, ref: name }); + await git.branch({ fs: this.runtime.fs.git, dir: this.dir, ref: name }); + await flushRuntime(this.runtime); } async log(limit?: number, _signal?: AbortSignal): Promise { - return readCommitLog(this.dir, limit ?? 10); + return readCommitLog(this.runtime, this.dir, limit ?? 10); } async readAt( hash: string, _signal?: AbortSignal, ): Promise { - const blob = await readBlobAtCommit(this.dir, hash, TURNS_FILE); + const blob = await readBlobAtCommit( + this.runtime, + this.dir, + hash, + TURNS_FILE, + ); if (blob === null) return []; - const text = new TextDecoder().decode(blob); + const text = decodeUTF8(blob); return parseTurns(text); } @@ -466,23 +617,22 @@ export class IsogitStore ): Promise { const safeKey = sanitizeCallId(key); const filename = `${safeKey}${blobExtensionFor(contentType)}`; - const dirPath = path.join(this.dir, TOOL_OUTPUT_DIR); - await fs.promises.mkdir(dirPath, { recursive: true }); - await fs.promises.writeFile(path.join(dirPath, filename), bytes); + const dirPath = this.runtime.path.join(this.dir, TOOL_OUTPUT_DIR); + await this.runtime.fs.mkdir(dirPath, { recursive: true }); + await this.runtime.fs.writeFile( + this.runtime.path.join(dirPath, filename), + bytes, + ); } async readBlob(key: string, _signal?: AbortSignal): Promise { const safeKey = sanitizeCallId(key); - const dirPath = path.join(this.dir, TOOL_OUTPUT_DIR); + const dirPath = this.runtime.path.join(this.dir, TOOL_OUTPUT_DIR); let entries: string[]; try { - entries = await fs.promises.readdir(dirPath); + entries = await this.runtime.fs.readdir(dirPath); } catch (cause) { - if ( - cause instanceof Error && - "code" in cause && - cause.code === "ENOENT" - ) { + if (hasCode(cause) && cause.code === "ENOENT") { throw new Error(`Blob not found for key: ${JSON.stringify(key)}`); } throw cause; @@ -494,16 +644,15 @@ export class IsogitStore if (match === undefined) { throw new Error(`Blob not found for key: ${JSON.stringify(key)}`); } - const buf = await fs.promises.readFile(path.join(dirPath, match)); - return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength); + return this.runtime.fs.readFile(this.runtime.path.join(dirPath, match)); } async writePrompt( turns: ConversationTurn[], _signal?: AbortSignal, ): Promise { - await fs.promises.writeFile( - path.join(this.dir, PROMPT_FILE), + await this.runtime.fs.writeFile( + this.runtime.path.join(this.dir, PROMPT_FILE), encodeJsonlLines(turns), ); } @@ -512,8 +661,8 @@ export class IsogitStore turn: AssistantTurn, _signal?: AbortSignal, ): Promise { - await fs.promises.writeFile( - path.join(this.dir, RESPONSE_FILE), + await this.runtime.fs.writeFile( + this.runtime.path.join(this.dir, RESPONSE_FILE), encodeJsonlLines([turn]), ); } @@ -522,8 +671,8 @@ export class IsogitStore records: TransformRecordType[], _signal?: AbortSignal, ): Promise { - await fs.promises.writeFile( - path.join(this.dir, MANIFEST_FILE), + await this.runtime.fs.writeFile( + this.runtime.path.join(this.dir, MANIFEST_FILE), encodeJsonlLines(records), ); } @@ -532,8 +681,8 @@ export class IsogitStore turns: ConversationTurn[], _signal?: AbortSignal, ): Promise { - await fs.promises.writeFile( - path.join(this.dir, TURNS_FILE), + await this.runtime.fs.writeFile( + this.runtime.path.join(this.dir, TURNS_FILE), encodeJsonlLines(turns), ); // Advance the in-memory marker only after the durable write succeeds. @@ -564,8 +713,8 @@ export class IsogitStore tokenUsage: metadata.tokenUsage, connectorState: this.pendingConnectorState, }; - await fs.promises.writeFile( - path.join(this.dir, METADATA_FILE), + await this.runtime.fs.writeFile( + this.runtime.path.join(this.dir, METADATA_FILE), JSON.stringify(payload, null, 2), ); } @@ -575,13 +724,13 @@ export class IsogitStore _signal?: AbortSignal, ): Promise { if (limit <= 0) return []; - const entries = await tolerantLog(this.dir, limit); + const entries = await tolerantLog(this.runtime, this.dir, limit); const collected: TransformRecordType[] = []; for (const entry of entries) { let blob: Uint8Array; try { ({ blob } = await git.readBlob({ - fs, + fs: this.runtime.fs.git, dir: this.dir, oid: entry.oid, filepath: MANIFEST_FILE, @@ -589,7 +738,7 @@ export class IsogitStore } catch { continue; } - const text = new TextDecoder().decode(blob); + const text = decodeUTF8(blob); const parsedLines = decodeJsonlLines(text); for (const raw of parsedLines) { const result = TransformRecord(raw); @@ -609,57 +758,32 @@ export class IsogitStore _signal?: AbortSignal, ): Promise { if (records.length === 0) return; - await withRepoDirLock(this.dir, async () => { - // Pre-flight: validate all records and check for duplicates before - // writing anything to disk. This avoids orphaned files if a - // duplicate is detected partway through the batch. - const planned: { record: AuditRecordType; filepath: string }[] = []; + await withRepoDirLock(this.runtime, this.dir, async () => { + const planned: DurableRecordFile[] = []; for (const record of records) { assertSafeSegment(record.sessionId, "sessionId"); const safeCallId = sanitizeCallId(record.callId); - const filepath = path.join( + const filepath = this.runtime.path.join( AUDIT_DIR, record.sessionId, `${safeCallId}.json`, ); - const fullPath = path.join(this.dir, filepath); - - try { - await fs.promises.access(fullPath); - throw new Error( - `Duplicate audit record: ${record.sessionId}/${record.callId}`, - ); - } catch (e) { - if (e instanceof Error && "code" in e && e.code === "ENOENT") { - // Expected: file does not exist yet. - } else { - throw e; - } - } - - planned.push({ record, filepath }); + planned.push({ + filepath, + directory: this.runtime.path.join( + this.dir, + AUDIT_DIR, + record.sessionId, + ), + contents: JSON.stringify(record, null, 2), + duplicateMessage: `Duplicate audit record: ${record.sessionId}/${record.callId}`, + }); } - - // Write phase: all validation passed, safe to write files. - for (const { record, filepath } of planned) { - const sessionDir = path.join(this.dir, AUDIT_DIR, record.sessionId); - await fs.promises.mkdir(sessionDir, { recursive: true }); - const fullPath = path.join(this.dir, filepath); - await fs.promises.writeFile(fullPath, JSON.stringify(record, null, 2)); - await git.add({ fs, dir: this.dir, filepath }); - } - - const count = records.length; - const noun = count === 1 ? "record" : "records"; - await git.commit({ - fs, - dir: this.dir, - message: `Record ${count} tool audit ${noun}`, - author: AUTHOR, - ...this.signingArgs(), + await this.commitRecordFiles(planned, (count) => { + const noun = count === 1 ? "record" : "records"; + return `Record ${count} tool audit ${noun}`; }); - await this.maybeGC(); }); } @@ -668,11 +792,8 @@ export class IsogitStore _signal?: AbortSignal, ): Promise { if (records.length === 0) return; - await withRepoDirLock(this.dir, async () => { - // Pre-flight: validate all records and check for duplicates before - // writing anything to disk. This avoids orphaned files if a - // duplicate is detected partway through the batch. - const planned: { record: ErrorRecord; filepath: string }[] = []; + await withRepoDirLock(this.runtime, this.dir, async () => { + const planned: DurableRecordFile[] = []; for (const record of records) { assertSafeSegment(record.sessionId, "sessionId"); @@ -681,48 +802,26 @@ export class IsogitStore "_", ); const seq = String(record.seq).padStart(8, "0"); - const filepath = path.join( + const filepath = this.runtime.path.join( ERRORS_DIR, record.sessionId, `${seq}-${sanitizedCategory}.json`, ); - const fullPath = path.join(this.dir, filepath); - - try { - await fs.promises.access(fullPath); - throw new Error( - `Duplicate error record: ${record.sessionId}/${seq}-${sanitizedCategory}`, - ); - } catch (e) { - if (e instanceof Error && "code" in e && e.code === "ENOENT") { - // Expected: file does not exist yet. - } else { - throw e; - } - } - - planned.push({ record, filepath }); + planned.push({ + filepath, + directory: this.runtime.path.join( + this.dir, + ERRORS_DIR, + record.sessionId, + ), + contents: JSON.stringify(record, null, 2), + duplicateMessage: `Duplicate error record: ${record.sessionId}/${seq}-${sanitizedCategory}`, + }); } - - // Write phase: all validation passed, safe to write files. - for (const { record, filepath } of planned) { - const sessionDir = path.join(this.dir, ERRORS_DIR, record.sessionId); - await fs.promises.mkdir(sessionDir, { recursive: true }); - const fullPath = path.join(this.dir, filepath); - await fs.promises.writeFile(fullPath, JSON.stringify(record, null, 2)); - await git.add({ fs, dir: this.dir, filepath }); - } - - const count = records.length; - const noun = count === 1 ? "record" : "records"; - await git.commit({ - fs, - dir: this.dir, - message: `Record ${count} error ${noun}`, - author: AUTHOR, - ...this.signingArgs(), + await this.commitRecordFiles(planned, (count) => { + const noun = count === 1 ? "record" : "records"; + return `Record ${count} error ${noun}`; }); - await this.maybeGC(); }); } @@ -731,11 +830,11 @@ export class IsogitStore _signal?: AbortSignal, ): Promise { assertSafeSegment(sessionId, "sessionId"); - const sessionDir = path.join(this.dir, AUDIT_DIR, sessionId); + const sessionDir = this.runtime.path.join(this.dir, AUDIT_DIR, sessionId); let entries: string[]; try { - entries = await fs.promises.readdir(sessionDir); + entries = await this.runtime.fs.readdir(sessionDir); } catch (cause) { if ( cause instanceof Error && @@ -750,8 +849,8 @@ export class IsogitStore const records: AuditRecordType[] = []; for (const entry of entries) { if (!entry.endsWith(".json")) continue; - const fullPath = path.join(sessionDir, entry); - const raw = await fs.promises.readFile(fullPath, "utf-8"); + const fullPath = this.runtime.path.join(sessionDir, entry); + const raw = await this.runtime.fs.readTextFile(fullPath); const parsed = JSON.parse(raw) as unknown; const result = AuditRecord(parsed); if (result instanceof type.errors) { diff --git a/vendor/intx-types/README.md b/vendor/intx-types/README.md index 70a144fa8..238601a8a 100644 --- a/vendor/intx-types/README.md +++ b/vendor/intx-types/README.md @@ -21,7 +21,7 @@ pull in the shapes they need: (models, model providers, offerings, and append-only pricing, plus the model-requirement and invoker-preference shapes and the model discovery view), observability, sidecar status enums (distinct from - the wire frames under `@intx/types/sidecar` below), agent addresses, + the wire frames under `@intx/types/sidecar` below), run addresses, hex and base64 helpers, and the `hasCode` error guard. - `@intx/types/authz` — grant rules, condition contexts, and authorization result shapes shared between `@intx/authz` and the diff --git a/vendor/intx-types/package.json b/vendor/intx-types/package.json index 90002f986..b3b8779a9 100644 --- a/vendor/intx-types/package.json +++ b/vendor/intx-types/package.json @@ -43,6 +43,14 @@ "./package-json": { "types": "./src/package-json.ts", "default": "./src/package-json.ts" + }, + "./wire-definition-hash": { + "types": "./src/wire-definition-hash.ts", + "default": "./src/wire-definition-hash.ts" + }, + "./workflow-sources": { + "types": "./src/workflow-sources.ts", + "default": "./src/workflow-sources.ts" } }, "dependencies": { diff --git a/vendor/intx-types/src/agent-address.test.ts b/vendor/intx-types/src/agent-address.test.ts index b52f6451c..43ba62f80 100644 --- a/vendor/intx-types/src/agent-address.test.ts +++ b/vendor/intx-types/src/agent-address.test.ts @@ -1,68 +1,68 @@ import { describe, test, expect } from "bun:test"; import { - formatAgentAddress, - isAgentAddress, - parseAgentAddress, + formatRunAddress, + isRunAddress, + parseRunAddress, } from "./agent-address"; -describe("formatAgentAddress", () => { - test("joins instanceId and domain with @", () => { - expect(formatAgentAddress("ins_abc123", "tenant.example")).toBe( - "ins_abc123@tenant.example", +describe("formatRunAddress", () => { + test("joins runId and domain with @", () => { + expect(formatRunAddress("run_abc123", "tenant.example")).toBe( + "run_abc123@tenant.example", ); }); }); -describe("parseAgentAddress", () => { +describe("parseRunAddress", () => { test("splits a well-formed address", () => { - expect(parseAgentAddress("ins_abc123@tenant.example")).toEqual({ - instanceId: "ins_abc123", + expect(parseRunAddress("run_abc123@tenant.example")).toEqual({ + runId: "run_abc123", domain: "tenant.example", }); }); - test("returns null when instance prefix is missing", () => { - expect(parseAgentAddress("usr_alice@tenant.example")).toBeNull(); + test("returns null when the run prefix is missing", () => { + expect(parseRunAddress("usr_alice@tenant.example")).toBeNull(); }); test("returns null when the @ is missing", () => { - expect(parseAgentAddress("ins_abc123")).toBeNull(); + expect(parseRunAddress("run_abc123")).toBeNull(); }); test("returns null when the local part is empty", () => { - expect(parseAgentAddress("@tenant.example")).toBeNull(); + expect(parseRunAddress("@tenant.example")).toBeNull(); }); test("returns null when the domain part is empty", () => { - expect(parseAgentAddress("ins_abc123@")).toBeNull(); + expect(parseRunAddress("run_abc123@")).toBeNull(); }); test("does not validate the shape of the domain", () => { - expect(parseAgentAddress("ins_abc123@not a real domain")).toEqual({ - instanceId: "ins_abc123", + expect(parseRunAddress("run_abc123@not a real domain")).toEqual({ + runId: "run_abc123", domain: "not a real domain", }); }); test("splits on the first @ and treats the rest as the domain", () => { - expect(parseAgentAddress("ins_abc123@foo@bar")).toEqual({ - instanceId: "ins_abc123", + expect(parseRunAddress("run_abc123@foo@bar")).toEqual({ + runId: "run_abc123", domain: "foo@bar", }); }); }); -describe("isAgentAddress", () => { - test("true for ins_-prefixed addresses with a domain", () => { - expect(isAgentAddress("ins_abc123@tenant.example")).toBe(true); +describe("isRunAddress", () => { + test("true for run_-prefixed addresses with a domain", () => { + expect(isRunAddress("run_abc123@tenant.example")).toBe(true); }); - test("false for non-agent local parts", () => { - expect(isAgentAddress("usr_alice@tenant.example")).toBe(false); + test("false for non-run local parts", () => { + expect(isRunAddress("usr_alice@tenant.example")).toBe(false); }); - test("false for bare instance IDs without a domain", () => { - expect(isAgentAddress("ins_abc123")).toBe(false); + test("false for bare run IDs without a domain", () => { + expect(isRunAddress("run_abc123")).toBe(false); }); }); diff --git a/vendor/intx-types/src/agent-address.ts b/vendor/intx-types/src/agent-address.ts index 8327fc578..2af45e560 100644 --- a/vendor/intx-types/src/agent-address.ts +++ b/vendor/intx-types/src/agent-address.ts @@ -1,34 +1,34 @@ -// Agent instance addresses are "@" where instanceId is -// the `ins_`-prefixed identifier produced by generateId("instance"). These -// helpers are the single source of truth for that format. +// Run addresses are "@" where runId is the local part: the +// `run_`-prefixed identifier that names the run. These helpers are the single +// source of truth for that format. // // The shape of the right-hand side of the "@" is not validated beyond the // requirement that it be non-empty: tightening the contract (DNS-ish // validation, normalisation, etc.) is a separate follow-up. // // `@intx/hub-sessions`'s `parseAgentId` is the canonical throwing wrapper -// over `parseAgentAddress` — call it when a `null` return would +// over `parseRunAddress` — call it when a `null` return would // propagate as a silent bug, and keep this parser's `null` return // reserved for callers that already have a structured fallback. -const INSTANCE_PREFIX = "ins_"; +const RUN_PREFIX = "run_"; -export function formatAgentAddress(instanceId: string, domain: string): string { - return `${instanceId}@${domain}`; +export function formatRunAddress(runId: string, domain: string): string { + return `${runId}@${domain}`; } -export function parseAgentAddress( +export function parseRunAddress( address: string, -): { instanceId: string; domain: string } | null { +): { runId: string; domain: string } | null { const atIdx = address.indexOf("@"); if (atIdx <= 0) return null; - const instanceId = address.slice(0, atIdx); + const runId = address.slice(0, atIdx); const domain = address.slice(atIdx + 1); - if (!instanceId.startsWith(INSTANCE_PREFIX)) return null; + if (!runId.startsWith(RUN_PREFIX)) return null; if (domain.length === 0) return null; - return { instanceId, domain }; + return { runId, domain }; } -export function isAgentAddress(address: string): boolean { - return parseAgentAddress(address) !== null; +export function isRunAddress(address: string): boolean { + return parseRunAddress(address) !== null; } diff --git a/vendor/intx-types/src/approvals.ts b/vendor/intx-types/src/approvals.ts index 42725c4e1..0441e934b 100644 --- a/vendor/intx-types/src/approvals.ts +++ b/vendor/intx-types/src/approvals.ts @@ -3,8 +3,8 @@ import { type } from "arktype"; export const ApprovalResponse = type({ id: "string", tenantId: "string", - deploymentId: type("string").describe( - "The workflow deployment the approval originates from. Every approval is raised during a workflow run; there is no launched single agent or agent-definition row behind it.", + anchorRunId: type("string").describe( + "The anchor run the approval originates from. Every approval is raised during a workflow run; there is no launched single agent or agent-definition row behind it.", ), runId: "string", agentAddress: "string", diff --git a/vendor/intx-types/src/message-id.ts b/vendor/intx-types/src/message-id.ts index 723f70060..523afffe8 100644 --- a/vendor/intx-types/src/message-id.ts +++ b/vendor/intx-types/src/message-id.ts @@ -8,9 +8,9 @@ // call sites import. // // A workflow run's id is NOT this value -- a deployment's one addressable -// top-level run uses the deployment mail address as its stable runId (see -// `deriveWorkflowRunId`). The two ids are distinct: this one is per-message, -// while the top-level runId is per-deployment. +// top-level run uses the local part of its mail address as its stable runId +// (see `deriveWorkflowRunId`). The two ids are distinct: this one is +// per-message, while the top-level runId is per-deployment. // // The identifier is the `Message-ID` header value when the message // carries one, and a sha256 of the raw bytes otherwise -- so a message diff --git a/vendor/intx-types/src/package-json.test.ts b/vendor/intx-types/src/package-json.test.ts index 8b50af12d..483ee11e6 100644 --- a/vendor/intx-types/src/package-json.test.ts +++ b/vendor/intx-types/src/package-json.test.ts @@ -74,3 +74,86 @@ describe("PackageJSON interchange.credentials", () => { ).toBe(false); }); }); + +describe("PackageJSON", () => { + test("accepts a package with an interchange.tools entry", () => { + const result = PackageJSON({ + name: "@intx/tools-posix", + version: "1.2.3", + interchange: { tools: "./dist/tools.js" }, + }); + expect(result instanceof type.errors).toBe(false); + expect(result).toMatchObject({ + interchange: { tools: "./dist/tools.js" }, + }); + }); + + test("accepts a package with an interchange.workflow entry", () => { + const result = PackageJSON({ + name: "@intx/workflow-example", + version: "1.2.3", + interchange: { workflow: "./dist/wf.js" }, + }); + expect(result instanceof type.errors).toBe(false); + expect(result).toMatchObject({ + interchange: { workflow: "./dist/wf.js" }, + }); + }); + + test("accepts a package with both tools and workflow", () => { + const result = PackageJSON({ + name: "@intx/combined", + version: "1.2.3", + interchange: { tools: "./dist/tools.js", workflow: "./dist/wf.js" }, + }); + expect(result instanceof type.errors).toBe(false); + expect(result).toMatchObject({ + interchange: { tools: "./dist/tools.js", workflow: "./dist/wf.js" }, + }); + }); + + test("accepts a package with neither tools nor workflow", () => { + const result = PackageJSON({ + name: "left-pad", + version: "1.3.0", + }); + expect(result instanceof type.errors).toBe(false); + }); + + test("accepts a package with an empty interchange object", () => { + const result = PackageJSON({ + name: "left-pad", + version: "1.3.0", + interchange: {}, + }); + expect(result instanceof type.errors).toBe(false); + }); + + test("rejects a non-string interchange.workflow", () => { + const result = PackageJSON({ + name: "@intx/workflow-example", + version: "1.2.3", + interchange: { workflow: 42 }, + }); + expect(result instanceof type.errors).toBe(true); + }); + + test("rejects a non-string interchange.tools", () => { + const result = PackageJSON({ + name: "@intx/tools-posix", + version: "1.2.3", + interchange: { tools: 42 }, + }); + expect(result instanceof type.errors).toBe(true); + }); + + test("ignores undeclared keys alongside workflow", () => { + const result = PackageJSON({ + name: "@intx/workflow-example", + version: "1.2.3", + description: "an upstream npm field", + interchange: { workflow: "./dist/wf.js", extra: "passthrough" }, + }); + expect(result instanceof type.errors).toBe(false); + }); +}); diff --git a/vendor/intx-types/src/package-json.ts b/vendor/intx-types/src/package-json.ts index 49e6a9c59..feabc8ddd 100644 --- a/vendor/intx-types/src/package-json.ts +++ b/vendor/intx-types/src/package-json.ts @@ -48,10 +48,13 @@ export type ToolCredentialDeclarationArray = typeof ToolCredentialDeclarationArray.infer; /** - * Required fields plus the `interchange` extension used to identify tool - * packages: `tools` names the sidecar-bundle entry, and `credentials` + * Required fields plus the `interchange` extensions used to identify + * interchange packages: `tools` names the sidecar-bundle entry, `credentials` * statically declares the provider-backed credentials the package's tools may - * need. `onUndeclaredKey("ignore")` lets the arbitrary upstream npm fields pass + * need, `workflow` names the module whose evaluation produces a workflow + * package's `WorkflowDefinition`, and `directors` names the module whose + * exports are the package's custom `defineDirector` factories. + * `onUndeclaredKey("ignore")` lets the arbitrary upstream npm fields pass * through without listing them. */ export const PackageJSON = type({ @@ -60,6 +63,8 @@ export const PackageJSON = type({ "interchange?": type({ "tools?": "string", "credentials?": ToolCredentialDeclarationArray, + "workflow?": "string", + "directors?": "string", }).onUndeclaredKey("ignore"), }).onUndeclaredKey("ignore"); export type PackageJSON = typeof PackageJSON.infer; diff --git a/vendor/intx-types/src/sessions.ts b/vendor/intx-types/src/sessions.ts index 856195da8..1917f6789 100644 --- a/vendor/intx-types/src/sessions.ts +++ b/vendor/intx-types/src/sessions.ts @@ -54,7 +54,7 @@ export const MailResponse = type({ sessionId: type("string").describe( "Internal session channel identifier, not a user-facing session resource.", ), - instanceId: "string | null", + runId: "string | null", direction: type("'inbound' | 'outbound'").describe( "Whether the message was sent to the agent (`inbound`) or emitted by the agent (`outbound`).", ), @@ -91,7 +91,7 @@ export const MailResponse = type({ }); export type MailResponse = typeof MailResponse.infer; -// Structured attachment-rejection errors returned by POST /:instanceId/mail. +// Structured attachment-rejection errors returned by POST /:runId/mail. // Each variant carries a machine-actionable `code` plus the fields a client // needs to locate and explain the rejection, alongside a human-readable // `message`. This is the wire contract for the route's attachment 400s; the @@ -135,7 +135,7 @@ export const InferenceTurnResponse = type({ sessionId: type("string").describe( "Internal session channel identifier, not a user-facing session resource.", ), - instanceId: "string", + runId: "string", model: "string", status: "'running' | 'completed' | 'failed'", startedAt: "string", diff --git a/vendor/intx-types/src/sidecar.test.ts b/vendor/intx-types/src/sidecar.test.ts index 6f19d1a59..7dbc1115e 100644 --- a/vendor/intx-types/src/sidecar.test.ts +++ b/vendor/intx-types/src/sidecar.test.ts @@ -91,7 +91,7 @@ describe("AgentDeployFrame", () => { id: "wf_demo", triggers: [{ type: "manual" }], stepOrder: ["plan", "act"], - steps: { plan: {}, act: {} }, + steps: { plan: { kind: "step" }, act: { kind: "step" } }, }, sources: { plan: [stepSource], act: [stepSource] }, }, @@ -107,7 +107,7 @@ describe("AgentDeployFrame", () => { id: "wf_demo", triggers: [{ type: "manual" }], stepOrder: ["plan"], - steps: { plan: {} }, + steps: { plan: { kind: "step" } }, }, }, }); @@ -122,7 +122,7 @@ describe("AgentDeployFrame", () => { id: "wf_demo", triggers: [{ type: "manual" }], stepOrder: ["plan", "act"], - steps: { plan: {}, act: {} }, + steps: { plan: { kind: "step" }, act: { kind: "step" } }, }, sources: { plan: [stepSource] }, }, @@ -141,7 +141,7 @@ describe("AgentDeployFrame", () => { definition: { id: "wf_demo", stepOrder: ["plan"], - steps: { plan: {} }, + steps: { plan: { kind: "step" } }, }, sources: { plan: [stepSource] }, }, @@ -237,8 +237,8 @@ describe("SignalCorrelationRegisterFrame snapshot requirement", () => { type: "signal.correlation.register", correlationId: "corr-1", runId: "run-1", - deploymentId: "dep-1", - agentAddress: "ins_dep@integration.interchange", + anchorRunId: "dep-1", + agentAddress: "run_dep@integration.interchange", kind: "approval", }; const snapshot = { diff --git a/vendor/intx-types/src/sidecar.ts b/vendor/intx-types/src/sidecar.ts index 88a276c46..3f71915ae 100644 --- a/vendor/intx-types/src/sidecar.ts +++ b/vendor/intx-types/src/sidecar.ts @@ -17,6 +17,8 @@ import { InferenceSource, } from "./runtime"; import { SignalKind } from "./signals"; +import { ToolPackageManifest } from "./tool-packages"; +import { WorkflowDefinitionSource } from "./workflow-sources"; // --------------------------------------------------------------------------- // Sidecar → Hub @@ -52,7 +54,7 @@ export const ReconnectFrame = type({ export type ReconnectFrame = typeof ReconnectFrame.infer; /** - * Response to a challenge frame. Contains a signature per agent address + * Response to a challenge frame. Contains a signature per run address * proving the sidecar holds the private key. Each signature is computed * over `nonce || utf8(agentAddress)`. */ @@ -107,7 +109,7 @@ export type MailOutboundFrame = typeof MailOutboundFrame.infer; /** * An InferenceEvent from the reactor, forwarded for UI consumption. Tagged - * with the agent address so the hub can route to the correct UI client. + * with the run address so the hub can route to the correct UI client. */ export const AgentEventFrame = type({ type: "'agent.event'", @@ -183,14 +185,14 @@ export type AgentUndeployAckFrame = typeof AgentUndeployAckFrame.infer; * `signalName` is deliberately NOT on the wire: it is a pure function of * `correlationId` (`signalName(correlationId)` in `./signals`), so the hub * computes it rather than trusting a value the sidecar could disagree on. - * `deploymentId` is the workflow deployment the run belongs to; `agentAddress` - * is the deployment's routable address the hub resolves tenancy from. + * `anchorRunId` is the anchor run the parked run belongs to; `agentAddress` + * is the anchor run's routable address the hub resolves tenancy from. */ export const SignalCorrelationRegisterFrame = type({ type: "'signal.correlation.register'", correlationId: "string", runId: "string", - deploymentId: "string", + anchorRunId: "string", agentAddress: "string", kind: SignalKind, // Approver-facing snapshot of the suspended tool call, size-capped at this @@ -340,65 +342,16 @@ export const DrainDeliverFrame = type({ }); export type DrainDeliverFrame = typeof DrainDeliverFrame.infer; -/** - * Workflow projection carried on an `agent.deploy` frame. Its presence - * at the deploy router routes the frame to the workflow deploy path -- - * single- or multi-step, both of which spawn the workflow-process child - * -- as opposed to a per-step provision frame. - * - * `definition` is the wire projection of `WorkflowDefinition` from - * `@intx/workflow`. The arktype validator enforces the structural - * envelope the workflow-process child re-parses on the sidecar after - * materialization (`packages/hub-sessions/src/workflow-kind.ts`'s - * `workflowDefinitionEnvelopeSchema`): `id`, `triggers`, `steps`, - * `stepOrder`, optional `state`. The wire validator MUST require every - * field the envelope requires — the sidecar's deploy router serializes - * `projection.definition` verbatim into `workflow.json` and the child - * rejects a tree missing any envelope-required field. Deeper validation - * of authoring-time primitive shape lives on the workflow definition - * surface in `@intx/workflow`, not on the wire. - * - * `sources` pins an ordered, non-empty inference-source list per step in - * `definition.stepOrder` so the workflow-process child can resolve inference - * at step invocation without a round trip to the hub. The list is the step's - * failover chain: element 0 is the active source (its id is the step's - * `defaultSource`), and the reactor fails over forward through the tail on a - * transient inference error. A workflow step pins a single-element list (no - * per-step failover); a single-agent instance pins the instance's full - * ordered source chain. Every `stepOrder` entry must have a matching - * `sources` entry; the validator rejects frames that violate this at the - * boundary. - */ -const WorkflowProjectionDefinition = type({ - id: "string > 0", - triggers: "unknown[]", - stepOrder: "string[]", - steps: { "[string]": "unknown" }, - "state?": "Record", - "+": "delete", -}); - -/** - * A workflow projection paired with its per-step inference-source pins, with - * the invariant that every `stepOrder` entry has a `sources` failover chain. - * The narrow here is the same coverage check the top-level `AgentDeployWorkflow` - * applies to its own definition; this reusable form carries it into each - * extracted onTrigger body under `referencedDefinitions`, so a body's sources - * cover the body's stepOrder just as the top-level's cover the top-level's. - */ -const WorkflowProjectionWithSources = type({ - definition: WorkflowProjectionDefinition, - sources: { "[string]": InferenceSource.array().atLeastLength(1) }, -}).narrow((value, ctx) => { - for (const stepId of value.definition.stepOrder) { - if (!Object.prototype.hasOwnProperty.call(value.sources, stepId)) { - return ctx.mustBe( - `a workflow projection whose sources cover every step in stepOrder; ${JSON.stringify(stepId)} is missing`, - ); - } - } - return true; -}); +import { + WorkflowProjectionDefinition, + WorkflowProjectionWithSources, +} from "./wire-workflow"; +// Re-export the wire-step/projection contracts that moved to `./wire-workflow` +// so existing `@intx/types/sidecar` consumers keep resolving them here. Each +// name is an arktype schema, so the single re-export carries both its value and +// its inferred type. +export { WorkflowStep } from "./wire-workflow"; +export { WorkflowProjectionDefinition, WorkflowProjectionWithSources }; /** * The decrypted credential material and per-handle binding descriptors @@ -436,34 +389,58 @@ export const CredentialDelivery = type({ }); export type CredentialDelivery = typeof CredentialDelivery.infer; -export const AgentDeployWorkflow = type({ - definition: WorkflowProjectionDefinition, - sources: { "[string]": InferenceSource.array().atLeastLength(1) }, - // Extracted onTrigger section bodies, materialized to their own workflow - // assets on the sidecar so a body child's spawn-child resolves the body by - // ref without a hub round-trip (the body id IS the asset ref). Optional: only - // an onTrigger deploy carries it, and every existing non-onTrigger deploy - // omits it and still validates. Each entry carries the body definition AND - // the body's own per-step inference-source pins, materialized beside the body - // on disk (`sources.json`) so a body child -- in-process, its env lost across - // a restart -- resolves inference durably without a hub round-trip. - "referencedDefinitions?": WorkflowProjectionWithSources.array(), - // Initial credential material for the deployment's tools, decrypted hub-side - // and delivered on the deploy frame so it is resident before any step runs - // (closing the race where a tool resolves a credential before a push lands). - // Run-global: a credential's secret is stored once, keyed by credentialId. - // Optional -- a deploy whose definition binds no credentials omits it. - "credentials?": CredentialDelivery, -}).narrow((value, ctx) => { - for (const stepId of value.definition.stepOrder) { - if (!Object.prototype.hasOwnProperty.call(value.sources, stepId)) { - return ctx.mustBe( - `a workflow projection whose sources cover every step in stepOrder; ${JSON.stringify(stepId)} is missing`, - ); - } - } - return true; -}); +/** + * The source-ref pin: where a code-sourced (npm) workflow definition's bytes + * come from (`source`) plus the frozen dependency closure the hub resolved for + * that pin (`closure`, concrete versions + integrity SRIs). The two ALWAYS + * travel together -- the sidecar re-materializes the exact `closure` from + * `source` and re-evaluates the pinned code -- so they are one co-required + * object rather than two independently-optional fields (which would let a + * "source without closure" state exist and be silently read as live-authored, + * downgrading the source-ref evaluate-the-pinned-code guarantee to trusting the + * inline projection). This is the same shape `WorkflowProbeRequestFrame` + * co-requires. A live-authored deploy carries no pin. + */ +export const SourceRefPin = type({ + source: WorkflowDefinitionSource, + closure: ToolPackageManifest, +}); +export type SourceRefPin = typeof SourceRefPin.infer; + +/** + * A full workflow deploy frame: the shared `WorkflowProjectionWithSources` base + * (definition + per-step sources + approved hash, carrying the + * stepOrder-covered-by-sources narrow) intersected with the top-level-only + * extras. Sharing the base via `.and()` means the field set and the coverage + * narrow are defined once, not restated here. + */ +export const AgentDeployWorkflow = WorkflowProjectionWithSources.and( + type({ + // Extracted onTrigger section bodies, materialized to their own workflow + // assets on the sidecar so a body child's spawn-child resolves the body by + // ref without a hub round-trip (the body id IS the asset ref). Optional: + // only an onTrigger deploy carries it, and every existing non-onTrigger + // deploy omits it and still validates. Each entry carries the body + // definition AND the body's own per-step inference-source pins, materialized + // beside the body on disk (`sources.json`) so a body child -- in-process, + // its env lost across a restart -- resolves inference durably without a hub + // round-trip. + "referencedDefinitions?": WorkflowProjectionWithSources.array(), + // Initial credential material for the deployment's tools, decrypted hub-side + // and delivered on the deploy frame so it is resident before any step runs + // (closing the race where a tool resolves a credential before a push lands). + // Run-global: a credential's secret is stored once, keyed by credentialId. + // Optional -- a deploy whose definition binds no credentials omits it. + "credentials?": CredentialDelivery, + // The source-ref pin (`source` + frozen `closure`) the sidecar + // re-materializes and re-evaluates the pinned code from, instead of trusting + // the inline projection. The two co-travel (see `SourceRefPin`), so presence + // of the pin is the single signal that this is a code-sourced deploy. + // Optional: only a code-sourced (npm) deploy carries it; a live-authored + // deploy has no pin. + "sourceRef?": SourceRefPin, + }), +); export type AgentDeployWorkflow = typeof AgentDeployWorkflow.infer; /** @@ -584,7 +561,7 @@ export type CredentialsUpdateFrame = typeof CredentialsUpdateFrame.infer; // // - `repoId` identifies the source repo at the hub. The hub maps `repoId` // to the originating entry in its kind-keyed RepoStore. For -// `repoId.kind === "agent-state"`, `repoId.id` is the agent address +// `repoId.kind === "agent-state"`, `repoId.id` is the run address // (the deploy/state repo and the destination agent are the same), so // the two fields carry the same value. Future kinds (e.g. assets) use // `repoId` to name a non-agent source while `agentAddress` continues @@ -820,6 +797,76 @@ export const SyncRequestFrame = type({ }); export type SyncRequestFrame = typeof SyncRequestFrame.infer; +// --------------------------------------------------------------------------- +// Workflow probe (bidirectional) +// --------------------------------------------------------------------------- +// +// A probe asks a connected sidecar to inspect a code-sourced workflow WITHOUT +// deploying it: materialize the frozen dependency closure, evaluate the entry +// module to a live `WorkflowDefinition`, project it to its inert needs +// surface, and return that projection plus the derived grant set and content +// hash. The request/result/error trio is correlated by `requestId`, entirely +// independent of the address maps -- a token-authed sidecar can serve a probe +// in its pre-deploy state, with no agent deployed and no routable address. + +/** + * Hub asks a connected sidecar to probe a code-sourced workflow. Correlated by + * `requestId`; the sidecar answers with `workflow.probe.result` on success or + * `workflow.probe.error` on failure, both carrying the same `requestId`. + * + * The frame carries everything the sidecar's probe child needs to run the + * probe with no further hub round-trip: + * - `source` names where the definition's bytes come from (the registry that + * publishes the definition package). + * - `closure` is the frozen dependency closure the hub already resolved -- + * concrete versions and integrity SRIs -- so the child materializes the + * exact tree the hub pinned. + * - `entry` is the `interchange.workflow` module path within the package + * whose evaluation produces the `WorkflowDefinition`. + */ +export const WorkflowProbeRequestFrame = type({ + type: "'workflow.probe.request'", + requestId: "string", + source: WorkflowDefinitionSource, + closure: ToolPackageManifest, + entry: "string", +}); +export type WorkflowProbeRequestFrame = typeof WorkflowProbeRequestFrame.infer; + +/** + * A connected sidecar's answer to a `workflow.probe.request`: the inert + * needs-surface projection of the probed workflow, the inert grant set derived + * from it, and the content hash of the projection. Correlated to the request + * by `requestId`. + * + * `projection` is the same closed `WorkflowProjectionDefinition` a deploy frame + * carries. `grants` is the deployment-wide inert grant surface -- the set of + * capability-grant strings the workflow requires -- for pre-deploy operator + * inspection. `wireHash` is the hex SHA-256 of the projection's canonical JSON + * (`computeWireDefinitionHash` in `@intx/types/wire-definition-hash`), the + * deployment's content-addressed handle. + */ +export const WorkflowProbeResultFrame = type({ + type: "'workflow.probe.result'", + requestId: "string", + projection: WorkflowProjectionDefinition, + grants: "string[]", + wireHash: "string", +}); +export type WorkflowProbeResultFrame = typeof WorkflowProbeResultFrame.infer; + +/** + * A connected sidecar reports that a `workflow.probe.request` failed -- + * materialization, evaluation, projection, or hashing threw. Correlated to the + * request by `requestId`; `error` describes the failure. + */ +export const WorkflowProbeErrorFrame = type({ + type: "'workflow.probe.error'", + requestId: "string", + error: "string", +}); +export type WorkflowProbeErrorFrame = typeof WorkflowProbeErrorFrame.infer; + // --------------------------------------------------------------------------- // Discriminated frame unions // --------------------------------------------------------------------------- @@ -841,7 +888,9 @@ export const SidecarFrame = RegisterFrame.or(ReconnectFrame) .or(PackDoneFrame) .or(PackAckFrame) .or(PackRejectFrame) - .or(MailInboundAckFrame); + .or(MailInboundAckFrame) + .or(WorkflowProbeResultFrame) + .or(WorkflowProbeErrorFrame); export type SidecarFrame = typeof SidecarFrame.infer; /** All frame types the hub sends to the sidecar. */ @@ -860,7 +909,8 @@ export const HubFrame = MailInboundFrame.or(AgentDeployFrame) .or(SignalDeliverFrame) .or(RunGrantsFrame) .or(SignalCorrelationRegisterAckFrame) - .or(DrainDeliverFrame); + .or(DrainDeliverFrame) + .or(WorkflowProbeRequestFrame); export type HubFrame = typeof HubFrame.infer; /** Any frame on the wire, regardless of direction. */ diff --git a/vendor/intx-types/src/wire-definition-hash.test.ts b/vendor/intx-types/src/wire-definition-hash.test.ts new file mode 100644 index 000000000..bb65be887 --- /dev/null +++ b/vendor/intx-types/src/wire-definition-hash.test.ts @@ -0,0 +1,91 @@ +import { describe, test, expect } from "bun:test"; + +import { + canonicalJsonStringify, + computeWireDefinitionHash, +} from "./wire-definition-hash"; + +describe("canonicalJsonStringify", () => { + test("is independent of object key order", () => { + const a = { id: "w-1", stepOrder: ["s1"], steps: { s1: { kind: "a" } } }; + const b = { steps: { s1: { kind: "a" } }, stepOrder: ["s1"], id: "w-1" }; + expect(canonicalJsonStringify(a)).toBe(canonicalJsonStringify(b)); + }); + + test("sorts keys at every nesting level", () => { + const value = { b: { d: 1, c: 2 }, a: [3, 2, 1] }; + expect(canonicalJsonStringify(value)).toBe( + '{"a":[3,2,1],"b":{"c":2,"d":1}}', + ); + }); + + test("preserves array order", () => { + expect(canonicalJsonStringify(["z", "a"])).toBe('["z","a"]'); + expect(canonicalJsonStringify(["a", "z"])).not.toBe( + canonicalJsonStringify(["z", "a"]), + ); + }); + + test("serializes primitives via JSON.stringify", () => { + expect(canonicalJsonStringify(null)).toBe("null"); + expect(canonicalJsonStringify(42)).toBe("42"); + expect(canonicalJsonStringify("hi")).toBe('"hi"'); + expect(canonicalJsonStringify(true)).toBe("true"); + }); + + test("drops undefined-valued keys, matching a JSON round-trip", () => { + // The hub hashes a projection parsed off the JSON wire (undefined-valued + // keys already gone); a child hashes the in-memory projection. The two must + // canonicalize identically or re-verify breaks on a legitimate definition. + const withUndefined = { id: "w-1", state: undefined, steps: {} }; + const roundTripped: unknown = JSON.parse(JSON.stringify(withUndefined)); + expect(canonicalJsonStringify(withUndefined)).toBe( + canonicalJsonStringify(roundTripped), + ); + expect(canonicalJsonStringify(withUndefined)).toBe( + '{"id":"w-1","steps":{}}', + ); + }); + + test("renders undefined array elements as null, matching JSON.stringify", () => { + expect(canonicalJsonStringify([1, undefined, 2])).toBe("[1,null,2]"); + }); +}); + +describe("computeWireDefinitionHash", () => { + test("is stable across key-ordering differences", async () => { + const a = { id: "w-1", stepOrder: ["s1"], steps: { s1: { kind: "step" } } }; + const b = { steps: { s1: { kind: "step" } }, stepOrder: ["s1"], id: "w-1" }; + expect(await computeWireDefinitionHash(a)).toBe( + await computeWireDefinitionHash(b), + ); + }); + + test("differs across different definitions", async () => { + const a = { id: "w-1", stepOrder: ["s1"], steps: { s1: {} } }; + const b = { id: "w-2", stepOrder: ["s1"], steps: { s1: {} } }; + expect(await computeWireDefinitionHash(a)).not.toBe( + await computeWireDefinitionHash(b), + ); + }); + + test("matches the pinned known-vector hash", async () => { + // Snapshot of the pre-move output; proves the moved implementation is + // byte-identical to the sidecar's original. Reordering the keys of the + // same definition must yield the same digest. + const definition = { + id: "w-1", + stepOrder: ["s1", "s2"], + steps: { s2: { kind: "b" }, s1: { kind: "a" } }, + }; + const reordered = { + steps: { s1: { kind: "a" }, s2: { kind: "b" } }, + stepOrder: ["s1", "s2"], + id: "w-1", + }; + const expected = + "50b2f0c5a74e704dbc1dc82af4c782b3eed636a89865ce84976dcc77c3ed2dfc"; + expect(await computeWireDefinitionHash(definition)).toBe(expected); + expect(await computeWireDefinitionHash(reordered)).toBe(expected); + }); +}); diff --git a/vendor/intx-types/src/wire-definition-hash.ts b/vendor/intx-types/src/wire-definition-hash.ts new file mode 100644 index 000000000..dfe9fe7a8 --- /dev/null +++ b/vendor/intx-types/src/wire-definition-hash.ts @@ -0,0 +1,82 @@ +// Content-addressed hash of a wire-projected workflow definition. +// +// The deploy gate, the install-time probe, and re-verify must all agree +// on the deployment's content handle, so they hash the exact same +// canonical form. This module is the single source of truth those call +// sites import; the hash is a hex SHA-256 of the definition's canonical +// JSON. + +import { hexEncode } from "./hex"; + +/** + * Project a value into a canonical JSON string with deterministically + * sorted object keys. Object key order and surrounding whitespace do not + * affect the output, so two structurally equal values serialize to + * byte-identical strings and therefore hash equal. + * + * Values follow `JSON.stringify`'s semantics for what JSON can represent: + * an object key whose value is `undefined`, a function, or a symbol is + * dropped, and such an array element renders as `null`. The only intended + * difference from `JSON.stringify` is the deterministic key order. So for + * JSON-representable values the output is invariant across a JSON + * round-trip; a value with a custom `toJSON` (e.g. a `Date`) is NOT + * round-trip invariant here, because this canonicalizer does not invoke + * `toJSON` -- a caller needing round-trip invariance must pass it + * already-JSON values. + */ +export function canonicalJsonStringify(value: unknown): string { + if (value === null) return "null"; + if (typeof value !== "object") { + // Primitives serialize as JSON does. A value JSON cannot represent + // (`undefined`, a function, a symbol) has no JSON text, so + // `JSON.stringify` returns `undefined`; canonicalize it to `null`, matching + // how JSON renders such a value as an array element (see below). A + // top-level such value never reaches a definition hash. + return JSON.stringify(value) ?? "null"; + } + if (Array.isArray(value)) { + // JSON renders an `undefined`/function/symbol array element as `null`; the + // scalar branch above produces exactly that, so a plain recursive map keeps + // parity. + return `[${value.map((v) => canonicalJsonStringify(v)).join(",")}]`; + } + // Drop keys whose value JSON.stringify would omit (`undefined`, functions, + // symbols), mirroring how JSON serializes an object. + const entries = Object.entries(value) + .filter( + ([, v]) => + v !== undefined && typeof v !== "function" && typeof v !== "symbol", + ) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return `{${entries + .map(([k, v]) => `${JSON.stringify(k)}:${canonicalJsonStringify(v)}`) + .join(",")}}`; +} + +/** + * Compute the content hash for a wire-projected workflow definition: + * SHA-256 of the canonical JSON of the `WorkflowDefinition` projection, + * hex-encoded. This is the deployment's content-addressed handle; every + * party that binds identity, approval, or re-verify to a deployment + * derives it from the same canonical form so their values compare by + * byte equality. + * + * Invariance across the boundary is load-bearing: the hub hashes a + * projection parsed off the JSON wire (where `undefined`-valued keys are + * already gone) while a child hashes an in-memory projection, and the two + * must produce byte-identical strings or re-verify would fail on a + * legitimately-approved definition. The canonicalizer's Date/`toJSON` + * caveat is moot here: both sides hash a projection parsed from the JSON + * wire, so no custom-`toJSON` value ever reaches the canonicalizer and the + * two sides agree. + */ +export async function computeWireDefinitionHash( + definition: unknown, +): Promise { + const canonical = canonicalJsonStringify(definition); + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(canonical), + ); + return hexEncode(new Uint8Array(digest)); +} diff --git a/vendor/intx-types/src/wire-workflow.ts b/vendor/intx-types/src/wire-workflow.ts new file mode 100644 index 000000000..369b48d58 --- /dev/null +++ b/vendor/intx-types/src/wire-workflow.ts @@ -0,0 +1,167 @@ +// Wire contracts for a workflow definition projected onto an `agent.deploy` +// frame: the per-step schema and the projection shapes the sidecar deploy +// router and the workflow-process child validate. Extracted from `sidecar.ts` +// so both files stay focused; `sidecar.ts` re-exports the public names, so +// `@intx/types/sidecar` consumers are unaffected. + +import { type } from "arktype"; + +import { CredentialBinding } from "./credentials"; +import { InferenceSource } from "./runtime"; + +/** + * Fields every wire step carries regardless of `kind`. All other keys pass + * through unmodified (arktype's default), including the nested `agent`, inner + * `step`, `body`, `on`, and selector fields -- typed nowhere here on purpose, + * because two producers feed this schema: the live-deploy passthrough ships a + * step whose `agent.toolFactories` are functions that JSON-encode to `null`, + * while the live->inert projector in `@intx/workflow-deploy` ships a reified + * plain-data agent. Both must validate; reifying the grant surface into plain + * data is the projector's job, not this envelope's. + */ +const commonStepFields = { + "id?": "string", + "after?": "string[]", +} as const; + +/** + * A wire step: its `kind` must be one of the ten known primitives, plus the + * common `id`/`after` fields; all other keys pass through unmodified. Exported + * so the live->inert projector's producer and its mutation-test suite validate + * a single step against the same schema the deploy frame applies to every step. + * + * The load-bearing check here is the KIND discriminant -- a step with no `kind` + * or a `kind` outside the set is rejected at this boundary rather than carried + * through opaque, and the membership is what makes a step's canonical JSON + * deterministic across the child->hub boundary. Per-variant field validation is + * deliberately NOT done here (deeper authoring-time validation -- required + * fields, selector resolvability, DAG shape -- lives on `@intx/workflow`), so + * the ten variants collapse to one schema over the kind enum rather than ten + * near-identical arms whose fields were all optional passthrough anyway. + */ +export const WorkflowStep = type({ + kind: "'step' | 'map' | 'gate' | 'awaitSignal' | 'sleep' | 'childWorkflow' | 'escalation' | 'action' | 'loop' | 'onTrigger'", + ...commonStepFields, +}); +export type WorkflowStep = typeof WorkflowStep.infer; + +/** + * The `steps` record on a wire projection: every value must validate + * against the closed `WorkflowStep` union. The runtime constraint runs + * through a `.narrow` over a `Record` rather than a typed + * `{ "[string]": WorkflowStep }` on purpose: the inferred type stays + * `Record` so the existing live-deploy producer + * (`toWireWorkflowDefinition`, which hands a `Record` + * steps map to `sendAgentDeploy`) still typechecks, while the runtime + * validation is fully closed over the primitive-kind set. + */ +const WorkflowSteps = type({ "[string]": "unknown" }).narrow((steps, ctx) => { + for (const [stepId, step] of Object.entries(steps)) { + const parsed = WorkflowStep(step); + if (parsed instanceof type.errors) { + return ctx.mustBe( + `a record whose every step matches a known workflow primitive ` + + `variant; step ${JSON.stringify(stepId)} did not (${parsed.summary})`, + ); + } + } + return true; +}); + +/** + * Workflow projection carried on an `agent.deploy` frame. Its presence + * at the deploy router routes the frame to the workflow deploy path -- + * single- or multi-step, both of which spawn the workflow-process child + * -- as opposed to a per-step provision frame. + * + * `definition` is the wire projection of `WorkflowDefinition` from + * `@intx/workflow`. The arktype validator enforces the structural + * envelope the workflow-process child re-parses on the sidecar after + * materialization (`packages/hub-sessions/src/workflow-kind.ts`'s + * `workflowDefinitionEnvelopeSchema`): `id`, `triggers`, `steps`, + * `stepOrder`, optional `state`. The wire validator MUST require every + * field the envelope requires — the sidecar's deploy router serializes + * `projection.definition` verbatim into `workflow.json` and the child + * rejects a tree missing any envelope-required field. Deeper validation + * of authoring-time primitive shape lives on the workflow definition + * surface in `@intx/workflow`, not on the wire. + * + * `sources` pins an ordered, non-empty inference-source list per step in + * `definition.stepOrder` so the workflow-process child can resolve inference + * at step invocation without a round trip to the hub. The list is the step's + * failover chain: element 0 is the active source (its id is the step's + * `defaultSource`), and the reactor fails over forward through the tail on a + * transient inference error. A workflow step pins a single-element list (no + * per-step failover); a single-agent instance pins the instance's full + * ordered source chain. Every `stepOrder` entry must have a matching + * `sources` entry; the validator rejects frames that violate this at the + * boundary. + */ +export const WorkflowProjectionDefinition = type({ + id: "string > 0", + triggers: "unknown[]", + stepOrder: "string[]", + steps: WorkflowSteps, + "state?": "Record", + // The definition's credential bindings, projected verbatim by the + // live->inert projector (`projectDefinition`). This MUST stay in sync with + // that projector: because of the `"+": "delete"` below, a binding the + // projector emits but this schema omits would be silently stripped at the + // wire boundary, desyncing the hub-resolved bindings from the projection + // the sidecar validates and re-verifies. Bindings are the operator-approved + // credential request surface (no secret material), so they belong in the + // hashed projection. + "credentialBindings?": CredentialBinding.array(), + "+": "delete", +}).narrow((value, ctx) => { + // Every `stepOrder` entry must name a defined step. A legitimately projected + // definition always satisfies this (the authoring validator enforces it), so + // this rejects only a projector-bypassing or tampered wire frame -- closing a + // phantom-stepOrder entry at the trust boundary for every consumer, rather + // than letting a downstream reader index `steps[missing]` as `undefined` and + // silently take a default path. + for (const stepId of value.stepOrder) { + if (!Object.prototype.hasOwnProperty.call(value.steps, stepId)) { + return ctx.mustBe( + `a workflow projection whose stepOrder names only defined steps; ${JSON.stringify(stepId)} has no matching entry in steps`, + ); + } + } + return true; +}); +export type WorkflowProjectionDefinition = + typeof WorkflowProjectionDefinition.infer; + +/** + * A workflow projection paired with its per-step inference-source pins and the + * hub-approved wire hash, with the invariant that every `stepOrder` entry has a + * `sources` failover chain. This is the shared base for BOTH the top-level + * deploy frame (`AgentDeployWorkflow`, which intersects its extras onto this) + * AND each extracted onTrigger body under `referencedDefinitions` -- so the + * field set and the coverage narrow are defined once and a body's sources cover + * the body's stepOrder just as the top-level's cover the top-level's. + */ +export const WorkflowProjectionWithSources = type({ + definition: WorkflowProjectionDefinition, + sources: { "[string]": InferenceSource.array().atLeastLength(1) }, + // The hub-approved wire hash of `definition`'s projection -- the freeze anchor + // the hub gate wrote (`computeWireDefinitionHash`). The sidecar feeds it to + // the child as the `DEFINITION_HASH` it re-verifies its own recompute + // against, rather than trusting a sidecar-computed hash. At the top level it + // pins the deployment's content handle; per body it pins the body's + // projection. Optional on the wire so a frame built before the source-ref + // hand-off (raw-frame paths) still validates; the production hub builder + // always stamps it. + "approvedWireHash?": "string > 0", +}).narrow((value, ctx) => { + for (const stepId of value.definition.stepOrder) { + if (!Object.prototype.hasOwnProperty.call(value.sources, stepId)) { + return ctx.mustBe( + `a workflow projection whose sources cover every step in stepOrder; ${JSON.stringify(stepId)} is missing`, + ); + } + } + return true; +}); +export type WorkflowProjectionWithSources = + typeof WorkflowProjectionWithSources.infer; diff --git a/vendor/intx-types/src/workflow-run-id.test.ts b/vendor/intx-types/src/workflow-run-id.test.ts new file mode 100644 index 000000000..d8901c419 --- /dev/null +++ b/vendor/intx-types/src/workflow-run-id.test.ts @@ -0,0 +1,33 @@ +import { describe, test, expect } from "bun:test"; + +import { deriveWorkflowRunId } from "./workflow-run-id"; + +describe("deriveWorkflowRunId", () => { + test("returns the local part before the @", () => { + expect(deriveWorkflowRunId("run_abc123@tenant.example")).toBe("run_abc123"); + }); + + test("throws when there is no @", () => { + expect(() => deriveWorkflowRunId("run_abc123")).toThrow( + "Invalid run address", + ); + }); + + test("throws when the domain is empty", () => { + expect(() => deriveWorkflowRunId("run_abc123@")).toThrow( + "Invalid run address", + ); + }); + + test("throws when the local part lacks the run_ prefix", () => { + expect(() => deriveWorkflowRunId("usr_abc123@tenant.example")).toThrow( + "Invalid run address", + ); + }); + + test("splits on the first @ when the address carries several", () => { + expect(deriveWorkflowRunId("run_abc123@foo@tenant.example")).toBe( + "run_abc123", + ); + }); +}); diff --git a/vendor/intx-types/src/workflow-run-id.ts b/vendor/intx-types/src/workflow-run-id.ts index 1bc1a2238..7eee721a2 100644 --- a/vendor/intx-types/src/workflow-run-id.ts +++ b/vendor/intx-types/src/workflow-run-id.ts @@ -1,13 +1,13 @@ // Canonical runId derivation for a workflow deployment's top-level run. // // A workflow deployment has ONE addressable top-level run, whose stable runId -// is the deployment's mail address (`ins_@`). The -// supervisor's dispatch loop keys its per-run state, its grants barrier, -// and its terminal wait on this id. Every producer of a run's grants -- -// the hub-api trigger route and the sidecar's mail-deliver path -- must -// stage those grants under the SAME id, or they land under a run id the -// supervisor never looks up and the run fails closed on its `onRunStart` -// barrier. +// is the local part of the deployment's mail address -- the `` in +// `@`. The supervisor's dispatch loop keys its per-run state, +// its grants barrier, and its terminal wait on this id. Every producer of a +// run's grants -- the hub-api trigger route and the sidecar's mail-deliver +// path -- must stage those grants under the SAME id, or they land under a run +// id the supervisor never looks up and the run fails closed on its +// `onRunStart` barrier. // // This module is the single source of truth those producers import, so // their derivations cannot diverge. It exists to end the divergence that @@ -16,13 +16,25 @@ // trigger occurrence. Internal section/body runs still receive their own // synthetic run ids and are not externally addressable. +import { parseRunAddress } from "./agent-address"; + /** * The stable runId for a workflow deployment's one addressable top-level run: - * its mail address. Callers hold the deployment mail address in different forms -- - * a routing recipient, a supervisor binding, a route-derived address -- - * and route it through this one function so the runId contract is stated - * in exactly one place. + * the local part of its mail address, before the `@`. Callers hold the + * deployment mail address in different forms -- a routing recipient, a + * supervisor binding, a route-derived address -- and route it through this one + * function so the runId contract is stated in exactly one place. + * + * Delegates to `parseRunAddress` so a single function owns the `@`-split: the + * runId is the parsed local part. A malformed address (no `run_` marker, no + * `@`, or an empty domain) is a caller bug, not a value to key state under, so + * this throws rather than returning a fabricated id that would land run state + * under an id the supervisor never looks up. */ -export function deriveWorkflowRunId(deploymentMailAddress: string): string { - return deploymentMailAddress; +export function deriveWorkflowRunId(address: string): string { + const parsed = parseRunAddress(address); + if (parsed === null) { + throw new Error(`Invalid run address: ${JSON.stringify(address)}`); + } + return parsed.runId; } diff --git a/vendor/intx-types/src/workflow-sources.test.ts b/vendor/intx-types/src/workflow-sources.test.ts new file mode 100644 index 000000000..6f75500f5 --- /dev/null +++ b/vendor/intx-types/src/workflow-sources.test.ts @@ -0,0 +1,66 @@ +import { describe, test, expect } from "bun:test"; +import { type } from "arktype"; +import { + WorkflowDefinitionRegistrySource, + WorkflowDefinitionSource, +} from "./workflow-sources"; + +describe("WorkflowDefinitionRegistrySource", () => { + test("accepts a registry source", () => { + const result = WorkflowDefinitionRegistrySource({ + kind: "registry", + registry: "npmjs", + }); + expect(result instanceof type.errors).toBe(false); + }); + + test("rejects a missing kind", () => { + const result = WorkflowDefinitionRegistrySource({ registry: "npmjs" }); + expect(result instanceof type.errors).toBe(true); + }); + + test("rejects a missing registry", () => { + const result = WorkflowDefinitionRegistrySource({ kind: "registry" }); + expect(result instanceof type.errors).toBe(true); + }); + + test("rejects a non-string registry", () => { + const result = WorkflowDefinitionRegistrySource({ + kind: "registry", + registry: 42, + }); + expect(result instanceof type.errors).toBe(true); + }); +}); + +describe("WorkflowDefinitionSource", () => { + test("accepts a registry source", () => { + const result = WorkflowDefinitionSource({ + kind: "registry", + registry: "npmjs", + }); + expect(result instanceof type.errors).toBe(false); + }); + + test("rejects an unknown kind", () => { + const result = WorkflowDefinitionSource({ + kind: "asset", + assetId: "asset_abc", + path: "definitions/foo.json", + }); + expect(result instanceof type.errors).toBe(true); + }); + + test("rejects a missing kind", () => { + const result = WorkflowDefinitionSource({ registry: "npmjs" }); + expect(result instanceof type.errors).toBe(true); + }); + + test("rejects a non-string registry", () => { + const result = WorkflowDefinitionSource({ + kind: "registry", + registry: 42, + }); + expect(result instanceof type.errors).toBe(true); + }); +}); diff --git a/vendor/intx-types/src/workflow-sources.ts b/vendor/intx-types/src/workflow-sources.ts new file mode 100644 index 000000000..a46893b69 --- /dev/null +++ b/vendor/intx-types/src/workflow-sources.ts @@ -0,0 +1,30 @@ +// Schemas for where a code-sourced workflow definition's bytes come from. +// +// A workflow install carries a `WorkflowDefinitionSource` to say where the +// definition should be fetched from at apply time. The registry variant +// names an npm registry that publishes the definition package; the sidecar's +// registry config maps `registry` to a URL and credentials. + +import { type } from "arktype"; + +/** + * A workflow definition published to the named npm registry, fetched at + * apply time. The sidecar's registry config maps `registry` to a URL and + * credentials. + */ +export const WorkflowDefinitionRegistrySource = type({ + kind: "'registry'", + registry: "string", +}); +export type WorkflowDefinitionRegistrySource = + typeof WorkflowDefinitionRegistrySource.infer; + +/** + * Discriminated union over where a workflow definition's bytes come from. + * + * Only the registry variant exists today. The union is kept in union form so + * asset- and git-sourced variants can be added later by widening it with + * `.or(...)` without a breaking change to the exported type's shape. + */ +export const WorkflowDefinitionSource = WorkflowDefinitionRegistrySource; +export type WorkflowDefinitionSource = typeof WorkflowDefinitionSource.infer; From 90dd97ea2b7e3cedc118f5b4c4f817e02de8c76a Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 22 Aug 2026 17:29:01 -0700 Subject: [PATCH 2/2] Ignore intentionally corrupt broken-toolchain fixture in lint The eval fixture ships a truncated config.ts so the grader can require a real repair. Prettier and ESLint both hard-fail on the parse error against the continue-on-error lint baseline. Exclude the fixture tree from both tools; the fixture itself stays corrupt for the eval. --- .prettierignore | 2 ++ eslint.config.js | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.prettierignore b/.prettierignore index d3a449bfe..a4db21604 100644 --- a/.prettierignore +++ b/.prettierignore @@ -4,3 +4,5 @@ vendor/ scratch/ node_modules/ CHANGELOG.md +# Intentionally corrupt eval fixture (CL-6882) — Prettier cannot parse it. +tests/fixtures/broken-toolchain/ diff --git a/eslint.config.js b/eslint.config.js index a05bd1ac7..190fae2c9 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -14,6 +14,8 @@ export default tseslint.config( "**/scratch/**", "node_modules/**", "**/node_modules/**", + // Intentionally corrupt eval fixture (CL-6882) — parser cannot load it. + "tests/fixtures/broken-toolchain/**", ], }, js.configs.recommended,