diff --git a/bootstrap/CHANGELOG.md b/bootstrap/CHANGELOG.md index 9b6589f..5e22008 100644 --- a/bootstrap/CHANGELOG.md +++ b/bootstrap/CHANGELOG.md @@ -2,6 +2,16 @@ ## 0.1.0 +- WP-22: `assembleManifest` writes `SELF_CID_PLACEHOLDER` on the current + `versions[]` entry and stamps the previous tail with the on-chain CID + (issue #30). +- WP-22: `createProductionDeps` uses `gatewaysFromEnv` so `ENSPACK_IPFS_GATEWAYS` + is prepended (then Pinata, then SPEC defaults). +- WP-22: `planBootstrap` records per-entry `failed` on resolve/publish throws + and continues; totals count only `ok` rows. Already-published models plan + `1.1.0` via `bumpMinor` / previous-manifest (same as `run`). +- WP-22: Hugging Bay lock HTTP 409/404 is treated as no lock; files step logs + "no Hugging Bay lock; cross-check skipped" and continues. - WP-21: `ENSPACK_BOOTSTRAP_STATE` overrides the `state.json` path so the one-box seed volume can persist resume state. `bootstrap/Dockerfile` is the infra compose `bootstrap` profile image (aria2c + `pnpm deploy --filter @enspack/bootstrap`). diff --git a/bootstrap/src/constants.ts b/bootstrap/src/constants.ts index 1a79b30..c5dcaca 100644 --- a/bootstrap/src/constants.ts +++ b/bootstrap/src/constants.ts @@ -1,4 +1,4 @@ -import { LICENSE_ALLOWLIST, MIRROR_NAMESPACE } from "@enspack/core"; +import { LICENSE_ALLOWLIST, MIRROR_NAMESPACE, SELF_CID_PLACEHOLDER } from "@enspack/core"; /** SPEC §3: MAGNET_RE-valid 40-hex zeros used only to drive the downloader. */ export const PLACEHOLDER_INFOHASH = "0".repeat(40); @@ -6,8 +6,8 @@ export const PLACEHOLDER_INFOHASH = "0".repeat(40); /** SPEC §4: placeholder magnet matching `MAGNET_RE`. */ export const PLACEHOLDER_MAGNET = `magnet:?xt=urn:btih:${PLACEHOLDER_INFOHASH}`; -/** Schema-valid CID used as a versions[] placeholder before the real pin. */ -export const PLACEHOLDER_CID = "bafkreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +/** Schema-valid CID used as a torrent-cid / draft placeholder (same bytes as issue #30). */ +export const PLACEHOLDER_CID = SELF_CID_PLACEHOLDER; export const DEFAULT_PUBLISHER = MIRROR_NAMESPACE; diff --git a/bootstrap/src/deps.ts b/bootstrap/src/deps.ts index 220f62b..15e7001 100644 --- a/bootstrap/src/deps.ts +++ b/bootstrap/src/deps.ts @@ -20,7 +20,7 @@ export interface BootstrapHf { export interface BootstrapHb { resolve(repo: string): Promise; - lock(artifactId: string): Promise; + lock(artifactId: string): Promise; submitFallback( artifactId: string, input: HbFallbackInput & { sourceUrl?: string; filePath?: string }, diff --git a/bootstrap/src/manifest.ts b/bootstrap/src/manifest.ts index 9a0ba11..09e9081 100644 --- a/bootstrap/src/manifest.ts +++ b/bootstrap/src/manifest.ts @@ -1,6 +1,12 @@ -import { MIRROR_NAMESPACE, type Manifest, type ManifestFile, SPEC_STRING } from "@enspack/core"; +import { + MIRROR_NAMESPACE, + type Manifest, + type ManifestFile, + SELF_CID_PLACEHOLDER, + SPEC_STRING, +} from "@enspack/core"; import { magnetFor } from "@enspack/torrent"; -import { PLACEHOLDER_CID, PLACEHOLDER_INFOHASH, PLACEHOLDER_MAGNET } from "./constants.js"; +import { PLACEHOLDER_INFOHASH, PLACEHOLDER_MAGNET } from "./constants.js"; import { canonicalNameFor, modelNameFor, versionNameFor } from "./names.js"; function hasLicenseFile(files: ManifestFile[]): boolean { @@ -39,11 +45,18 @@ export function assembleManifest(input: AssembleInput): Manifest { const thisVersion = { version: input.version, name, - cid: PLACEHOLDER_CID, + cid: SELF_CID_PLACEHOLDER, createdAt: input.createdAt, }; - const versions = - input.previousVersions !== undefined ? [...input.previousVersions, thisVersion] : [thisVersion]; + const prior = + input.previousVersions === undefined + ? [] + : input.previousVersions.map((v, i, arr) => + i === arr.length - 1 && input.previous !== undefined + ? { ...v, cid: input.previous } + : { ...v }, + ); + const versions = [...prior, thisVersion]; const distribution: Manifest["distribution"] = { infohash: input.infohash, @@ -107,7 +120,7 @@ export function draftManifestForDownload( }, files: files as Manifest["files"], totalSize, - versions: [{ version: "0.0.0", name, cid: PLACEHOLDER_CID, createdAt }], + versions: [{ version: "0.0.0", name, cid: SELF_CID_PLACEHOLDER, createdAt }], }; } diff --git a/bootstrap/src/plan.ts b/bootstrap/src/plan.ts index 0311ccf..ef88a17 100644 --- a/bootstrap/src/plan.ts +++ b/bootstrap/src/plan.ts @@ -1,8 +1,10 @@ import { + type Manifest, type PublishCall, type PublishResult, canonicalJson, formatPublishPlan, + isEnspackError, manifestCid, validateManifest, } from "@enspack/core"; @@ -105,14 +107,22 @@ export async function planBootstrap( const modelName = modelNameFor(gate.data.org, gate.data.repoName ?? ""); let version = "1.0.0"; + let previousVersions: Manifest["versions"] | undefined; + let previous: string | undefined; try { const resolved = await deps.resolver.resolve(modelName, { chain }); - if (resolved.manifest !== null) { + if (resolved.manifest !== null && resolved.cid !== null) { + previousVersions = resolved.manifest.versions; + previous = resolved.cid; const last = resolved.manifest.versions[resolved.manifest.versions.length - 1]; version = bumpMinor(last?.version ?? resolved.manifest.version); } - } catch { - /* first publish */ + } catch (err) { + if (!isEnspackError(err) || err.code !== "RESOLVE") { + const message = err instanceof Error ? err.message : String(err); + out.push({ repo: entry.repo, status: "failed", snapshotSize, license, reason: message }); + continue; + } } const webseeds = [deps.hfWebseed(entry.repo, revision)]; @@ -140,34 +150,49 @@ export async function planBootstrap( version, createdAt: deps.now(), }; - const manifest = assembleManifest(assemble); - validateManifest(manifest); - const cid = await manifestCid(canonicalJson(manifest)); - const published = await deps.publisher.publish({ - manifest, - manifestCid: cid, - chain, - dryRun: true, - }); - const calls = allPublishCalls(published); - const gas = sumGas(calls); - totalBytes += snapshotSize; - totalGas += gas; - const row: PlanEntryJson = { - repo: entry.repo, - status: "ok", - snapshotSize, - fileCount: files.length, - license, - gasEstimate: gas.toString(), - version, - name: manifest.name, - plan: formatPublishPlan(calls), - }; - if (filesResult.data.hbCrossCheck !== undefined) { - row.hbCrossCheck = filesResult.data.hbCrossCheck; + if (previousVersions !== undefined) assemble.previousVersions = previousVersions; + if (previous !== undefined) assemble.previous = previous; + + try { + const manifest = assembleManifest(assemble); + validateManifest(manifest); + const cid = await manifestCid(canonicalJson(manifest)); + const published = await deps.publisher.publish({ + manifest, + manifestCid: cid, + chain, + dryRun: true, + }); + const calls = allPublishCalls(published); + const gas = sumGas(calls); + totalBytes += snapshotSize; + totalGas += gas; + const row: PlanEntryJson = { + repo: entry.repo, + status: "ok", + snapshotSize, + fileCount: files.length, + license, + gasEstimate: gas.toString(), + version, + name: manifest.name, + plan: formatPublishPlan(calls), + }; + if (filesResult.data.hbCrossCheck !== undefined) { + row.hbCrossCheck = filesResult.data.hbCrossCheck; + } + out.push(row); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + out.push({ + repo: entry.repo, + status: "failed", + snapshotSize, + license, + reason: message, + version, + }); } - out.push(row); } return { entries: out, totals: { bytes: totalBytes, gasEstimate: totalGas.toString() } }; diff --git a/bootstrap/src/production.ts b/bootstrap/src/production.ts index d606b47..0f0e5e9 100644 --- a/bootstrap/src/production.ts +++ b/bootstrap/src/production.ts @@ -4,6 +4,7 @@ import { createManifestStore, createPublisher, createResolver, + gatewaysFromEnv, kuboPinner, publicClientFor, seedNodePinner, @@ -90,7 +91,7 @@ export function createProductionDeps(opts: ProductionOpts): BootstrapDeps { pinner = seedNodePinner({ baseUrl: seedUrl }); } - const store = createManifestStore({ pinner }); + const store = createManifestStore({ gateways: gatewaysFromEnv(env), pinner }); const resolver = createResolver({ client, store, diff --git a/bootstrap/src/steps/files.ts b/bootstrap/src/steps/files.ts index 7bd8d1d..d0ab7d2 100644 --- a/bootstrap/src/steps/files.ts +++ b/bootstrap/src/steps/files.ts @@ -10,7 +10,7 @@ import type { EntrySnapshot, ModelEntry, StepOutcome } from "../types.js"; export async function stepFiles( entry: ModelEntry, revision: string, - deps: Pick, + deps: Pick, ): Promise { const files = await deps.hf.buildFiles(entry.repo, revision); const artifact = await deps.hb.resolve(entry.repo); @@ -24,6 +24,15 @@ export async function stepFiles( } try { const lock = await deps.hb.lock(artifact.id); + if (lock === null) { + deps.log.info("no Hugging Bay lock; cross-check skipped"); + const data: Partial = { + files, + fileCount: files.length, + hbCrossCheck: "skipped", + }; + return { outcome: "ok", data }; + } crossCheck(files, lock); } catch (err) { const message = err instanceof EnspackError ? err.message : String(err); diff --git a/bootstrap/test/cross-check.test.ts b/bootstrap/test/cross-check.test.ts index 9b3b8a8..a993255 100644 --- a/bootstrap/test/cross-check.test.ts +++ b/bootstrap/test/cross-check.test.ts @@ -4,7 +4,17 @@ import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import type { BootstrapDeps } from "../src/deps.js"; import { runBootstrap } from "../src/run.js"; -import { disagreeingHb, fakeHf, loadModels, resolverThatThrows, silentLog } from "./helpers.js"; +import { stepFiles } from "../src/steps/files.js"; +import { + TINY_REPO, + TINY_SHA, + disagreeingHb, + fakeHf, + loadModels, + noLockHb, + resolverThatThrows, + silentLog, +} from "./helpers.js"; describe("cross-check (BOOTSTRAP.md §2 rule 3)", () => { const dirs: string[] = []; @@ -80,4 +90,26 @@ describe("cross-check (BOOTSTRAP.md §2 rule 3)", () => { expect(publishCalls).toEqual([]); expect(records[0]?.completedSteps).toEqual(["gate"]); }); + + it("treats a missing HB lock as ok and skips the cross-check", async () => { + const logs: string[] = []; + const result = await stepFiles( + { repo: TINY_REPO, tier: 1, expected_license: "apache-2.0", revision: "abcdef12" }, + TINY_SHA, + { + hf: fakeHf(), + hb: noLockHb(), + log: { + info(message) { + logs.push(message); + }, + warn() {}, + }, + }, + ); + expect(result.outcome).toBe("ok"); + if (result.outcome !== "ok") return; + expect(result.data.hbCrossCheck).toBe("skipped"); + expect(logs).toContain("no Hugging Bay lock; cross-check skipped"); + }); }); diff --git a/bootstrap/test/helpers.ts b/bootstrap/test/helpers.ts index 78e13de..26de974 100644 --- a/bootstrap/test/helpers.ts +++ b/bootstrap/test/helpers.ts @@ -130,6 +130,25 @@ export function emptyHb(): BootstrapDeps["hb"] { }; } +export function noLockHb(): BootstrapDeps["hb"] { + return { + async resolve() { + return { + id: "hf-model-enspack-tiny-model", + repo: TINY_REPO, + digest: "aa".repeat(32), + raw: {}, + }; + }, + async lock() { + return null; + }, + async submitFallback() { + return {}; + }, + }; +} + export function disagreeingHb(): BootstrapDeps["hb"] { return { async resolve() { diff --git a/bootstrap/test/manifest.test.ts b/bootstrap/test/manifest.test.ts new file mode 100644 index 0000000..e502429 --- /dev/null +++ b/bootstrap/test/manifest.test.ts @@ -0,0 +1,62 @@ +import { SELF_CID_PLACEHOLDER } from "@enspack/core"; +import { describe, expect, it } from "vitest"; +import { assembleManifest } from "../src/manifest.js"; +import { tinyModelFiles } from "./helpers.js"; + +const REAL_CID = `bafkrei${"b".repeat(52)}`; +const PREV_CID = `bafkrei${"c".repeat(52)}`; +const WEBSEED = + "https://huggingface.co/enspack/tiny-model/resolve/abcdef1200000000000000000000000000000000/"; + +describe("assembleManifest versions[] cid (issue #30)", () => { + it("uses SELF_CID_PLACEHOLDER for the current entry", () => { + const manifest = assembleManifest({ + org: "enspack", + repoName: "tiny-model", + repo: "enspack/tiny-model", + revision: "abcdef1200000000000000000000000000000000", + license: "apache-2.0", + displayName: "tiny-model", + files: tinyModelFiles(), + infohash: "0".repeat(40), + magnet: `magnet:?xt=urn:btih:${"0".repeat(40)}`, + torrentCid: SELF_CID_PLACEHOLDER, + webseeds: [WEBSEED], + version: "1.0.0", + createdAt: "2026-09-14T00:00:00Z", + }); + expect(manifest.versions).toHaveLength(1); + expect(manifest.versions[0]?.cid).toBe(SELF_CID_PLACEHOLDER); + }); + + it("stamps the previous tail with previous CID and placeholders the new last entry", () => { + const manifest = assembleManifest({ + org: "enspack", + repoName: "tiny-model", + repo: "enspack/tiny-model", + revision: "abcdef1200000000000000000000000000000000", + license: "apache-2.0", + displayName: "tiny-model", + files: tinyModelFiles(), + infohash: "0".repeat(40), + magnet: `magnet:?xt=urn:btih:${"0".repeat(40)}`, + torrentCid: SELF_CID_PLACEHOLDER, + webseeds: [WEBSEED], + version: "1.1.0", + createdAt: "2026-09-15T00:00:00Z", + previous: PREV_CID, + previousVersions: [ + { + version: "1.0.0", + name: "v1-0-0.enspack--tiny-model.mirrors.enspack.eth", + cid: REAL_CID, + createdAt: "2026-09-14T00:00:00Z", + }, + ], + }); + expect(manifest.versions).toHaveLength(2); + expect(manifest.versions[0]?.cid).toBe(PREV_CID); + expect(manifest.versions[1]?.cid).toBe(SELF_CID_PLACEHOLDER); + expect(manifest.previous).toBe(PREV_CID); + }); +}); diff --git a/bootstrap/test/plan.test.ts b/bootstrap/test/plan.test.ts index 03105c6..4efd68b 100644 --- a/bootstrap/test/plan.test.ts +++ b/bootstrap/test/plan.test.ts @@ -1,15 +1,17 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import type { PublishInput } from "@enspack/core"; +import type { PublishInput, Resolved } from "@enspack/core"; +import { EnspackError, namehashOf } from "@enspack/core"; import { afterEach, describe, expect, it } from "vitest"; import { runCli } from "../src/cli.js"; import type { BootstrapDeps } from "../src/deps.js"; -import type { PlanJson } from "../src/plan.js"; +import { type PlanJson, planBootstrap } from "../src/plan.js"; import { ZERO_ADDR, emptyHb, fakeHf, + loadFixtureManifest, loadModels, modelsYamlPath, recordingPublisher, @@ -177,3 +179,117 @@ describe("plan --json (MVP.md WP-12 dry run)", () => { ); }); }); + +describe("planBootstrap resilience (WP-22)", () => { + function basePlanDeps(overrides: Partial = {}): BootstrapDeps { + return { + hf: fakeHf(), + hb: emptyHb(), + store: { + async put() { + throw new Error("plan must not pin"); + }, + async getVerified() { + throw new Error("unused"); + }, + }, + publisher: recordingPublisher([]), + downloader: { + async fetch() { + throw new Error("plan must not download"); + }, + }, + verifier: { + async verify() { + throw new Error("verify"); + }, + async quarantine() { + throw new Error("quarantine"); + }, + }, + seedNode: { + async seed() { + throw new Error("seed"); + }, + async status() { + throw new Error("status"); + }, + }, + resolver: resolverThatThrows(), + now: () => "2026-09-14T00:00:00.000Z", + hfWebseed: (repo, revision) => `https://huggingface.co/${repo}/resolve/${revision}/`, + log: silentLog(), + ...overrides, + }; + } + + it("records a throwing publisher as failed and continues other entries", async () => { + const config = loadModels(); + const first = config.models[0]; + if (first === undefined) throw new Error("fixture"); + const second = { ...first, repo: "enspack/other-model" }; + const inner = recordingPublisher([]); + const deps = basePlanDeps({ + publisher: { + async publish(input) { + if (input.manifest.model.includes("tiny-model")) { + throw new EnspackError("PUBLISH", "version name already points at cid"); + } + return inner.publish(input); + }, + }, + }); + const plan = await planBootstrap(config, [first, second], "sepolia", deps); + expect(plan.entries).toHaveLength(2); + expect(plan.entries[0]?.status).toBe("failed"); + expect(plan.entries[0]?.reason).toMatch(/already points/); + expect(plan.entries[1]?.status).toBe("ok"); + expect(plan.entries[1]?.version).toBe("1.0.0"); + expect(plan.totals.bytes).toBe(plan.entries[1]?.snapshotSize); + expect(plan.totals.gasEstimate).toBe(plan.entries[1]?.gasEstimate); + }); + + it("plans 1.1.0 when the model name already resolves", async () => { + const config = loadModels(); + const entry = config.models[0]; + if (entry === undefined) throw new Error("fixture"); + const prevCid = `bafkrei${"c".repeat(52)}`; + const deps = basePlanDeps({ + resolver: { + async resolve(ref): Promise { + if (!ref.includes("enspack--tiny-model")) { + throw new EnspackError("RESOLVE", `unresolved ${ref}`); + } + const previous = loadFixtureManifest(); + return { + name: ref, + node: namehashOf(ref), + cid: prevCid, + magnet: previous.distribution.magnet, + spec: previous.spec, + manifest: { + ...previous, + name: "v1-0-0.enspack--tiny-model.mirrors.enspack.eth", + model: "enspack--tiny-model.mirrors.enspack.eth", + version: "1.0.0", + versions: [ + { + version: "1.0.0", + name: "v1-0-0.enspack--tiny-model.mirrors.enspack.eth", + cid: previous.versions[0]?.cid ?? `bafkrei${"a".repeat(52)}`, + createdAt: previous.createdAt, + }, + ], + }, + manifestBytes: null, + }; + }, + }, + }); + const plan = await planBootstrap(config, [entry], "sepolia", deps); + expect(plan.entries).toHaveLength(1); + expect(plan.entries[0]?.status).toBe("ok"); + expect(plan.entries[0]?.version).toBe("1.1.0"); + expect(plan.entries[0]?.name).toMatch(/^v1-1-0\./); + }); +}); diff --git a/bootstrap/test/production.test.ts b/bootstrap/test/production.test.ts new file mode 100644 index 0000000..435eec0 --- /dev/null +++ b/bootstrap/test/production.test.ts @@ -0,0 +1,25 @@ +import { DEFAULT_GATEWAYS, PINATA_GATEWAY, gatewaysFromEnv } from "@enspack/core"; +import { describe, expect, it } from "vitest"; +import { createProductionDeps } from "../src/production.js"; + +const ANVIL_0_KEY = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; + +describe("createProductionDeps gateways (WP-22)", () => { + it("honours ENSPACK_IPFS_GATEWAYS via gatewaysFromEnv", () => { + const extra = "https://custom.example/ipfs/{cid}"; + const env: NodeJS.ProcessEnv = { + ENSPACK_OPERATOR_KEY: ANVIL_0_KEY, + SEPOLIA_RPC_URL: "http://127.0.0.1:1", + ENSPACK_IPFS_GATEWAYS: extra, + }; + expect(gatewaysFromEnv(env)).toEqual([extra, PINATA_GATEWAY, ...DEFAULT_GATEWAYS]); + const deps = createProductionDeps({ + chain: "sepolia", + env, + pin: "kubo", + dryRun: true, + }); + expect(deps.store).toBeDefined(); + expect(typeof deps.store.getVerified).toBe("function"); + }); +}); diff --git a/docs/publishing.md b/docs/publishing.md index 3bc9f7c..b9bbeba 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -52,6 +52,21 @@ and `ENSPACK_ENS_VERSION` override. Mainnet stays v1. manifest, CID, and calldata/gas on stderr (`formatPublishPlan`). No pins, no txs. +### `versions[].cid` (issue #30) + +A manifest cannot contain its own CID: writing the CID changes the bytes. +`enspack publish` (and bootstrap `assembleManifest`) put the schema-valid +placeholder `SELF_CID_PLACEHOLDER` (`bafkrei` + 52×`a`) on the **last** +`versions[]` entry. Previous entries keep real CIDs (the previous tail is +stamped with the on-chain CID of that version). Clients take the current +version's CID from `contenthash`. `enspack versions --json` adds +`latestCidFromContenthash`. Human `inspect` / `versions` mark the latest +entry as `(this manifest — see contenthash)`. + +`enspack get --http-only` skips torrent metainfo (`distribution.torrent.cid`) +and fetches files from `webseeds[]` with SHA-256 verification (SPEC §4 step +7c). Without `--http-only`, metainfo is still fetched and checked. + Publisher names under `mirrors.enspack.eth` use `--` labels and set `canonical` to `..enspack.eth`. Agents must not publish on Sepolia under any label other than `*.mirrors.enspack.eth` (FLEET.md). diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index d13effb..a24c250 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -2,6 +2,16 @@ ## 0.1.0 +- WP-22: `publish` writes `SELF_CID_PLACEHOLDER` on `versions[last].cid` instead + of the draft CID (issue #30). Previous entries keep (or are stamped with) + real CIDs. `inspect` / `versions` human output marks the latest cid as + "(this manifest — see contenthash)"; `versions --json` adds + `latestCidFromContenthash`. +- WP-22: `--http-only` skips torrent metainfo fetch/verify (files still SHA-256 + checked from webseeds). Logs `http-only: skipping metainfo` on stderr. +- WP-22: Hugging Bay lock 404/409 skips the publish cross-check. +- WP-22: default store prepends `https://gateway.pinata.cloud/ipfs/{cid}` + (`gatewaysFromEnv`); `ENSPACK_IPFS_GATEWAYS` still comes first. - WP-18: `enspack ens-setup --chain sepolia --name enspack.eth` one-shot ENSv2 on-chain setup. Signer is `ENSPACK_OPERATOR_KEY` (falls back to `ENSPACK_PUBLISHER_KEY`). `--dry-run` / `--json` / `--subname` / `--operator`. diff --git a/packages/cli/README.md b/packages/cli/README.md index b9c9827..ba2be83 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -57,6 +57,10 @@ enspack get [--dir ] [--select ...] [--http-only] Default install target is `$HF_HOME/hub/models--{org}--{repo}/` (SPEC §5). `--dir` copies files flat into that directory. +`--http-only` fetches each `files[]` entry from `webseeds[]` and verifies +SHA-256. It does **not** fetch `distribution.torrent.cid` metainfo from IPFS +(SPEC §4 step 7c). The torrent path still checks metainfo. + If `enspack.lock` in the current working directory already has an entry for the name, a CID mismatch is a hard error (exit 3) unless `--update`. The lockfile is rewritten only when that entry already exists, or when `--save` is passed @@ -93,7 +97,7 @@ webseed; see `packages/cli/test/get.e2e.test.ts`. | `HF_TOKEN` | Optional Hugging Face read token | | `PINATA_JWT` | `--pin pinata` | | `ENSPACK_SEED_NODE` | Default seed-node base URL (`--pin seed`, `seed`) | -| `ENSPACK_IPFS_GATEWAYS` | Comma-separated gateway templates (`{cid}`), prepended to core `DEFAULT_GATEWAYS` | +| `ENSPACK_IPFS_GATEWAYS` | Comma-separated gateway templates (`{cid}`), tried first. Then Pinata (`https://gateway.pinata.cloud/ipfs/{cid}`), then core `DEFAULT_GATEWAYS` (dweb.link, ipfs.io, w3s.link) | | `ENSPACK_KUBO_API` | Kubo HTTP API base for `--pin kubo` | | `HF_HOME` | Hugging Face cache root (default `~/.cache/huggingface`) | diff --git a/packages/cli/src/commands/inspect.ts b/packages/cli/src/commands/inspect.ts index 77cb9c8..617d4c9 100644 --- a/packages/cli/src/commands/inspect.ts +++ b/packages/cli/src/commands/inspect.ts @@ -40,6 +40,14 @@ export async function runInspect( `webseeds ${manifest.distribution.webseeds.join(" ")}`, `canonical ${manifest.canonical ?? "-"}`, ]; + const lastIdx = manifest.versions.length - 1; + for (const [i, v] of manifest.versions.entries()) { + const cid = i === lastIdx ? "(this manifest — see contenthash)" : v.cid; + lines.push(`version ${v.version} ${v.name} ${cid}`); + } + if (resolved.cid !== null) { + lines.push(`contenthash ${resolved.cid}`); + } for (const line of lines) { human(deps.stderr, line); } diff --git a/packages/cli/src/commands/publish.ts b/packages/cli/src/commands/publish.ts index b88c3aa..05d94e4 100644 --- a/packages/cli/src/commands/publish.ts +++ b/packages/cli/src/commands/publish.ts @@ -2,17 +2,18 @@ import { mkdir, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { - DEFAULT_GATEWAYS, EnspackError, MIRROR_NAMESPACE, type Manifest, type ManifestStore, type Pinner, type PublishResult, + SELF_CID_PLACEHOLDER, canonicalJson, createManifestStore, ensVersionFor, formatPublishPlan, + gatewaysFromEnv, isEnspackError, kuboPinner, manifestCid, @@ -58,18 +59,6 @@ function splitRepo(repo: string): { org: string; name: string } { return { org: repo.slice(0, slash), name: repo.slice(slash + 1) }; } -function parseGateways(env: NodeJS.ProcessEnv): string[] { - const extra = env.ENSPACK_IPFS_GATEWAYS; - const extras = - extra !== undefined && extra !== "" - ? extra - .split(",") - .map((s) => s.trim()) - .filter((s) => s.length > 0) - : []; - return [...extras, ...DEFAULT_GATEWAYS]; -} - function pinnerFor( kind: "kubo" | "pinata" | "seed", deps: CliDeps, @@ -103,7 +92,7 @@ function pinStore(deps: CliDeps, pinner: Pinner): ManifestStore { if (deps.storeWithPinner !== undefined) { return deps.storeWithPinner(pinner); } - return createManifestStore({ gateways: parseGateways(deps.env), pinner }); + return createManifestStore({ gateways: gatewaysFromEnv(deps.env), pinner }); } async function putBytes( @@ -186,7 +175,11 @@ export async function runPublish(deps: CliDeps, flags: PublishFlags): Promise new Date()))().toISOString(); const prev = await previousVersions(deps, chain, modelName); - const placeholderCid = `bafkrei${"p".repeat(52)}`; + // Issue #30: a manifest cannot contain its own CID. Keep a schema-valid + // placeholder on the last entry; stamp the previous tail with the on-chain CID. + const prior = prev.versions.map((v, i, arr) => + i === arr.length - 1 && prev.previous !== undefined ? { ...v, cid: prev.previous } : { ...v }, + ); const thisVersion = { version: flags.version, name: versionName, - cid: placeholderCid, + cid: SELF_CID_PLACEHOLDER, createdAt, }; - const versions = [...prev.versions, thisVersion] as Manifest["versions"]; + const versions = [...prior, thisVersion] as unknown as Manifest["versions"]; const manifestDraft: Record = { spec: "enspack/0.1", @@ -284,13 +281,7 @@ export async function runPublish(deps: CliDeps, flags: PublishFlags): Promise s.trim()) - .filter((s) => s.length > 0) - : []; - return [...extras, ...DEFAULT_GATEWAYS]; -} - /** Secret stores often strip the `0x`; accept both forms, never log the value. */ function normalizePrivateKey(value: string): `0x${string}` | null { const trimmed = value.trim(); @@ -190,7 +178,7 @@ function isWriter(value: NodeJS.WritableStream | Writer): value is Writer { */ export function createDefaultDeps(opts: DefaultDepsOpts = {}): CliDeps { const env = opts.env ?? process.env; - const gateways = parseGateways(env); + const gateways = gatewaysFromEnv(env); const store = createManifestStore({ gateways }); const stdout = opts.stdout === undefined diff --git a/packages/cli/src/pipeline.ts b/packages/cli/src/pipeline.ts index 53f876d..0a5c8ff 100644 --- a/packages/cli/src/pipeline.ts +++ b/packages/cli/src/pipeline.ts @@ -229,16 +229,21 @@ export async function runPipeline(deps: CliDeps, opts: PipelineOptions): Promise human(deps.stderr, `a publisher-verified name exists: ${manifest.canonical}`); } - const metainfo = await fetchTorrentVerified(asIpfsStore(deps.store), manifest, deps.fetch); - if (metainfo !== null) { - const parsed = await parseTorrent(metainfo); - if (parsed.infohash !== manifest.distribution.infohash) { - throw new EnspackError( - "VERIFY", - `torrent infohash ${parsed.infohash} !== manifest.distribution.infohash ${manifest.distribution.infohash}`, - ); + let metainfo: Uint8Array | null = null; + if (opts.httpOnly === true) { + human(deps.stderr, "http-only: skipping metainfo"); + } else { + metainfo = await fetchTorrentVerified(asIpfsStore(deps.store), manifest, deps.fetch); + if (metainfo !== null) { + const parsed = await parseTorrent(metainfo); + if (parsed.infohash !== manifest.distribution.infohash) { + throw new EnspackError( + "VERIFY", + `torrent infohash ${parsed.infohash} !== manifest.distribution.infohash ${manifest.distribution.infohash}`, + ); + } + metainfoMatchesManifest(parsed, manifest); } - metainfoMatchesManifest(parsed, manifest); } const infohash = manifest.distribution.infohash; diff --git a/packages/cli/src/types.ts b/packages/cli/src/types.ts index 7d10b5a..8a2671e 100644 --- a/packages/cli/src/types.ts +++ b/packages/cli/src/types.ts @@ -39,7 +39,7 @@ export interface CliHf { licenseGate(license: string | null | undefined, opts?: LicenseGateOptions): void; huggingBay: { resolve(repo: string): Promise; - lock(artifactId: string): Promise; + lock(artifactId: string): Promise; submitFallback(artifactId: string, input: HbFallbackInput): Promise; }; } diff --git a/packages/cli/test/__snapshots__/json.test.ts.snap b/packages/cli/test/__snapshots__/json.test.ts.snap index b88a2b0..8d2a2c5 100644 --- a/packages/cli/test/__snapshots__/json.test.ts.snap +++ b/packages/cli/test/__snapshots__/json.test.ts.snap @@ -40,7 +40,7 @@ exports[`json payloads > verify --json on a matching directory 1`] = ` " `; -exports[`json payloads > versions --json marks the latest 1`] = ` -"{"versions":[{"version":"1.0.0","name":"v1-0-0.tiny-model.mirrors.enspack.eth","cid":"bafkreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","createdAt":"2026-09-14T00:00:00Z","latest":true}]} +exports[`json payloads > versions --json marks the latest and adds latestCidFromContenthash 1`] = ` +"{"versions":[{"version":"1.0.0","name":"v1-0-0.tiny-model.mirrors.enspack.eth","cid":"bafkreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","createdAt":"2026-09-14T00:00:00Z","latest":true}],"latestCidFromContenthash":"bafkreihsttlkrfbniygoxztnhtdxhhpk4uerx6lpceyx4plsxnfuellp3i"} " `; diff --git a/packages/cli/test/defaults.test.ts b/packages/cli/test/defaults.test.ts index 990e48b..975d9c8 100644 --- a/packages/cli/test/defaults.test.ts +++ b/packages/cli/test/defaults.test.ts @@ -1,5 +1,5 @@ import type { Manifest } from "@enspack/core"; -import { ensVersionFor } from "@enspack/core"; +import { DEFAULT_GATEWAYS, PINATA_GATEWAY, ensVersionFor, gatewaysFromEnv } from "@enspack/core"; import { describe, expect, it } from "vitest"; import { createDefaultDeps } from "../src/defaults.js"; import { applyEnsVersionFlag, ensOptsFor } from "../src/ens.js"; @@ -102,3 +102,26 @@ describe("ensOptsFor (WP-17)", () => { expect(() => applyEnsVersionFlag({}, "v3")).toThrow(/must be v1 or v2/); }); }); + +describe("CLI configured gateway order (WP-22)", () => { + it("uses gatewaysFromEnv: env extras, Pinata, then SPEC DEFAULT_GATEWAYS", () => { + expect([...DEFAULT_GATEWAYS]).toEqual([ + "https://{cid}.ipfs.dweb.link", + "https://ipfs.io/ipfs/{cid}", + "https://{cid}.ipfs.w3s.link", + ]); + expect(gatewaysFromEnv({})).toEqual([PINATA_GATEWAY, ...DEFAULT_GATEWAYS]); + const extra = "https://custom.example/ipfs/{cid}"; + expect(gatewaysFromEnv({ ENSPACK_IPFS_GATEWAYS: extra })).toEqual([ + extra, + PINATA_GATEWAY, + ...DEFAULT_GATEWAYS, + ]); + const deps = createDefaultDeps({ + stdout: capturingWriter(), + stderr: capturingWriter(), + env: { ETH_RPC_URL: "http://127.0.0.1:1", ENSPACK_IPFS_GATEWAYS: extra }, + }); + expect(deps.store).toBeDefined(); + }); +}); diff --git a/packages/cli/test/http-only.test.ts b/packages/cli/test/http-only.test.ts new file mode 100644 index 0000000..b0b5af1 --- /dev/null +++ b/packages/cli/test/http-only.test.ts @@ -0,0 +1,50 @@ +import { join } from "node:path"; +import { EnspackError } from "@enspack/core"; +import { describe, expect, it } from "vitest"; +import { baseDeps, runCli, withTmp } from "./helpers.js"; + +describe("get --http-only skips torrent metainfo (WP-22)", () => { + it("succeeds when the torrent CID fetch throws FETCH, and get without --http-only fails", async () => { + await withTmp(async (tmp) => { + const { deps, manifest } = await baseDeps(tmp); + const torrentCid = manifest.distribution.torrent?.cid; + expect(torrentCid).toBeDefined(); + const inner = deps.store; + deps.store = { + async getVerified(cid: string) { + if (cid === torrentCid) { + throw new EnspackError("FETCH", `torrent CID ${cid} unavailable`); + } + return inner.getVerified(cid); + }, + async put(bytes: Uint8Array) { + return inner.put(bytes); + }, + }; + + const dir = join(tmp, "http-only"); + const httpOnly = await runCli(deps, [ + "get", + manifest.model, + "--dir", + dir, + "--http-only", + "--json", + ]); + expect(httpOnly.code, httpOnly.stderr).toBe(0); + expect(httpOnly.stderr).toContain("http-only: skipping metainfo"); + expect(JSON.parse(httpOnly.stdout)).toMatchObject({ verified: true }); + + const torrentPath = await runCli(deps, [ + "get", + manifest.model, + "--dir", + join(tmp, "torrent"), + "--json", + ]); + expect(torrentPath.code).toBe(2); + expect(torrentPath.stderr).toContain("torrent CID"); + expect(torrentPath.stderr).toContain("unavailable"); + }); + }); +}); diff --git a/packages/cli/test/json.test.ts b/packages/cli/test/json.test.ts index acb4699..f5d6588 100644 --- a/packages/cli/test/json.test.ts +++ b/packages/cli/test/json.test.ts @@ -54,16 +54,18 @@ describe("json payloads", () => { }); }); - it("versions --json marks the latest", async () => { + it("versions --json marks the latest and adds latestCidFromContenthash", async () => { await withTmp(async (tmp) => { - const { deps, manifest } = await baseDeps(tmp); - const { code, stdout } = await runCli(deps, ["versions", manifest.model, "--json"]); + const { deps, manifest, cid } = await baseDeps(tmp); + const { code, stdout, stderr } = await runCli(deps, ["versions", manifest.model, "--json"]); expect(code).toBe(0); + expect(stderr).toBe(""); expect(JSON.parse(stdout)).toEqual({ versions: manifest.versions.map((v, i) => ({ ...v, latest: i === manifest.versions.length - 1, })), + latestCidFromContenthash: cid, }); expect(stdout).toMatchSnapshot(); }); diff --git a/packages/cli/test/publish.test.ts b/packages/cli/test/publish.test.ts new file mode 100644 index 0000000..f8351a4 --- /dev/null +++ b/packages/cli/test/publish.test.ts @@ -0,0 +1,202 @@ +import { createHash } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { + EnspackError, + type Manifest, + type PublishInput, + SELF_CID_PLACEHOLDER, + namehashOf, +} from "@enspack/core"; +import { describe, expect, it } from "vitest"; +import type { CliHf } from "../src/types.js"; +import { baseDeps, runCli, stubHf, stubPublisher, withTmp } from "./helpers.js"; + +const REAL_CID = `bafkrei${"b".repeat(52)}`; +const PREV_CID = `bafkrei${"c".repeat(52)}`; + +function helloHf(): { hf: CliHf; hello: Buffer } { + const hello = Buffer.from("hello-enspack\n"); + const digest = createHash("sha256").update(hello).digest("hex"); + const hf: CliHf = { + ...stubHf(), + async info() { + return { + gated: false, + private: false, + license: "apache-2.0", + sha: "b".repeat(40), + cardData: { license: "apache-2.0" }, + }; + }, + async resolveRevision() { + return "b".repeat(40); + }, + async buildFiles() { + return [{ path: "hello.txt", size: hello.length, sha256: digest, role: "other" }]; + }, + }; + return { hf, hello }; +} + +async function wirePublish( + deps: Awaited>["deps"], + hello: Buffer, + onPublish: (input: PublishInput) => void, +): Promise { + deps.hf = helloHf().hf; + deps.downloader = { + async fetch(_m: Manifest, dest: string) { + await mkdir(dest, { recursive: true }); + await writeFile(join(dest, "hello.txt"), hello); + }, + }; + const inner = stubPublisher(); + deps.publisherFactory = () => ({ + async publish(input) { + onPublish(input); + return inner.publish(input); + }, + }); +} + +describe("publish versions[] cid (issue #30)", () => { + it("writes SELF_CID_PLACEHOLDER on the last entry for a first publish", async () => { + await withTmp(async (tmp) => { + const { deps } = await baseDeps(tmp); + const { hello } = helloHf(); + const seen: PublishInput[] = []; + await wirePublish(deps, hello, (input) => { + seen.push(input); + }); + deps.resolverFactory = () => ({ + async resolve() { + throw new EnspackError("RESOLVE", "no previous"); + }, + }); + const { code, stderr } = await runCli(deps, [ + "publish", + "--from-hf", + "Qwen/Qwen2.5-7B-Instruct", + "--publisher", + "mirrors.enspack.eth", + "--version", + "1.0.0", + "--dry-run", + "--json", + ]); + expect(code, stderr).toBe(0); + expect(seen).toHaveLength(1); + const versions = seen[0]?.manifest.versions ?? []; + expect(versions).toHaveLength(1); + expect(versions[0]?.cid).toBe(SELF_CID_PLACEHOLDER); + expect(seen[0]?.manifestCid).not.toBe(SELF_CID_PLACEHOLDER); + }); + }); + + it("keeps previous real CIDs and placeholders only the new last entry", async () => { + await withTmp(async (tmp) => { + const { deps } = await baseDeps(tmp); + const { hello } = helloHf(); + const seen: PublishInput[] = []; + await wirePublish(deps, hello, (input) => { + seen.push(input); + }); + const digest = createHash("sha256").update(hello).digest("hex"); + const previous: Manifest["versions"] = [ + { + version: "1.0.0", + name: "v1-0-0.qwen--qwen2-5-7b-instruct.mirrors.enspack.eth", + cid: REAL_CID, + createdAt: "2026-09-14T00:00:00Z", + }, + { + version: "1.0.1", + name: "v1-0-1.qwen--qwen2-5-7b-instruct.mirrors.enspack.eth", + cid: SELF_CID_PLACEHOLDER, + createdAt: "2026-09-15T00:00:00Z", + }, + ]; + const lastPrev = previous[1]; + if (lastPrev === undefined) { + throw new Error("fixture"); + } + deps.resolverFactory = () => ({ + async resolve(ref: string) { + return { + name: ref, + node: namehashOf(ref), + cid: PREV_CID, + magnet: null, + spec: "enspack/0.1" as const, + manifest: { + spec: "enspack/0.1" as const, + name: lastPrev.name, + model: "qwen--qwen2-5-7b-instruct.mirrors.enspack.eth", + publisher: "mirrors.enspack.eth", + version: "1.0.1", + createdAt: "2026-09-15T00:00:00Z", + license: "apache-2.0", + distribution: { + infohash: "0".repeat(40), + magnet: `magnet:?xt=urn:btih:${"0".repeat(40)}`, + webseeds: [ + "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct/resolve/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/", + ], + }, + files: [{ path: "hello.txt", size: hello.length, sha256: digest, role: "other" }], + totalSize: hello.length, + versions: previous, + } as unknown as Manifest, + manifestBytes: null, + }; + }, + }); + const { code, stderr } = await runCli(deps, [ + "publish", + "--from-hf", + "Qwen/Qwen2.5-7B-Instruct", + "--publisher", + "mirrors.enspack.eth", + "--version", + "1.1.0", + "--dry-run", + "--json", + ]); + expect(code, stderr).toBe(0); + const versions = seen[0]?.manifest.versions ?? []; + expect(versions).toHaveLength(3); + expect(versions[0]?.cid).toBe(REAL_CID); + expect(versions[1]?.cid).toBe(PREV_CID); + expect(versions[2]?.cid).toBe(SELF_CID_PLACEHOLDER); + expect(versions[2]?.version).toBe("1.1.0"); + expect(seen[0]?.manifest.previous).toBe(PREV_CID); + }); + }); +}); + +describe("inspect / versions human cid annotation (issue #30)", () => { + it("inspect human marks the latest cid and prints contenthash", async () => { + await withTmp(async (tmp) => { + const { deps, manifest, cid } = await baseDeps(tmp); + const { code, stdout, stderr } = await runCli(deps, ["inspect", manifest.model]); + expect(code).toBe(0); + expect(stdout).toBe(""); + expect(stderr).toContain("(this manifest — see contenthash)"); + expect(stderr).toContain(`contenthash ${cid}`); + expect(stderr).not.toContain(` ${SELF_CID_PLACEHOLDER}\n`); + }); + }); + + it("versions human marks the latest cid and prints latestCidFromContenthash", async () => { + await withTmp(async (tmp) => { + const { deps, manifest, cid } = await baseDeps(tmp); + const { code, stdout, stderr } = await runCli(deps, ["versions", manifest.model]); + expect(code).toBe(0); + expect(stdout).toBe(""); + expect(stderr).toContain("(this manifest — see contenthash)"); + expect(stderr).toContain(`latestCidFromContenthash ${cid}`); + expect(stderr).toContain("latest"); + }); + }); +}); diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index 3a9f89d..24e2897 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -2,6 +2,10 @@ ## 0.1.0 +- WP-22: `SELF_CID_PLACEHOLDER` (`bafkrei` + 52×`a`) for `versions[last].cid` + (issue #30). A manifest cannot contain its own CID. +- WP-22: `PINATA_GATEWAY` and `gatewaysFromEnv(env)` (env extras, Pinata, then + SPEC `DEFAULT_GATEWAYS`). `DEFAULT_GATEWAYS` itself is unchanged. - WP-18: `planEnsSetup` / `runEnsSetup` for one-shot ENSv2 publisher setup (resolver + UserRegistry + optional operator grants + subname registries). Idempotent; pending proxies resolved from `ProxyDeployed`. diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts index 0489ba4..fcded2e 100644 --- a/packages/core/src/constants.ts +++ b/packages/core/src/constants.ts @@ -38,6 +38,36 @@ export const DEFAULT_GATEWAYS = [ "https://{cid}.ipfs.w3s.link", ] as const; +/** + * Pinata public gateway. CLI and bootstrap configured lists prepend this + * (manifests are pinned there); it is not part of SPEC `DEFAULT_GATEWAYS`. + */ +export const PINATA_GATEWAY = "https://gateway.pinata.cloud/ipfs/{cid}"; + +type EnvLike = Record; + +/** + * SPEC §4 step 3 order for configured clients: `ENSPACK_IPFS_GATEWAYS`, then + * Pinata, then `DEFAULT_GATEWAYS`. + */ +export function gatewaysFromEnv(env: EnvLike = process.env): string[] { + const extra = env.ENSPACK_IPFS_GATEWAYS; + const extras = + extra !== undefined && extra !== "" + ? extra + .split(",") + .map((s) => s.trim()) + .filter((s) => s.length > 0) + : []; + return [...extras, PINATA_GATEWAY, ...DEFAULT_GATEWAYS]; +} + +/** + * Schema-valid placeholder for `versions[last].cid` (issue #30). A manifest + * cannot contain its own CID; clients take the current version from contenthash. + */ +export const SELF_CID_PLACEHOLDER = `bafkrei${"a".repeat(52)}`; + /** SPEC / MVP WP-03: manifest size cap (1 MiB). */ export const MANIFEST_MAX_BYTES = 1024 * 1024; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 5935cd9..5c4c9bd 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -7,10 +7,13 @@ export { MAGNET_RE, MANIFEST_MAX_BYTES, MIRROR_NAMESPACE, + PINATA_GATEWAY, ROOT_NAME, + SELF_CID_PLACEHOLDER, SPEC_STRING, TEXT_KEYS, TORRENT_MAX_BYTES, + gatewaysFromEnv, } from "./constants.js"; export { EnspackError, EXIT_CODES, isEnspackError } from "./error.js"; export type { EnspackErrorCode } from "./error.js"; diff --git a/packages/core/test/constants.test.ts b/packages/core/test/constants.test.ts new file mode 100644 index 0000000..a29b676 --- /dev/null +++ b/packages/core/test/constants.test.ts @@ -0,0 +1,52 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { + DEFAULT_GATEWAYS, + PINATA_GATEWAY, + SELF_CID_PLACEHOLDER, + gatewaysFromEnv, + validateManifest, +} from "../src/index.js"; + +const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "../../.."); + +describe("SELF_CID_PLACEHOLDER (issue #30)", () => { + it("is bafkrei plus 52 a's and matches the tiny-model fixture", () => { + expect(SELF_CID_PLACEHOLDER).toBe(`bafkrei${"a".repeat(52)}`); + const fixture = JSON.parse( + readFileSync(join(repoRoot, "test/fixtures/tiny-model.enspack.json"), "utf8"), + ) as { versions: { cid: string }[] }; + expect(fixture.versions[fixture.versions.length - 1]?.cid).toBe(SELF_CID_PLACEHOLDER); + expect(validateManifest(fixture).versions.at(-1)?.cid).toBe(SELF_CID_PLACEHOLDER); + }); +}); + +describe("gatewaysFromEnv (WP-22)", () => { + it("leaves DEFAULT_GATEWAYS as the SPEC §4 step 3 list", () => { + expect([...DEFAULT_GATEWAYS]).toEqual([ + "https://{cid}.ipfs.dweb.link", + "https://ipfs.io/ipfs/{cid}", + "https://{cid}.ipfs.w3s.link", + ]); + }); + + it("prepends Pinata then SPEC defaults, with ENSPACK_IPFS_GATEWAYS first", () => { + expect(gatewaysFromEnv({})).toEqual([PINATA_GATEWAY, ...DEFAULT_GATEWAYS]); + expect( + gatewaysFromEnv({ + ENSPACK_IPFS_GATEWAYS: "https://a.example/{cid}, https://b.example/{cid}", + }), + ).toEqual([ + "https://a.example/{cid}", + "https://b.example/{cid}", + PINATA_GATEWAY, + ...DEFAULT_GATEWAYS, + ]); + expect(gatewaysFromEnv({ ENSPACK_IPFS_GATEWAYS: " " })).toEqual([ + PINATA_GATEWAY, + ...DEFAULT_GATEWAYS, + ]); + }); +}); diff --git a/packages/core/test/ipfs/store.test.ts b/packages/core/test/ipfs/store.test.ts index af34c06..aaa068a 100644 --- a/packages/core/test/ipfs/store.test.ts +++ b/packages/core/test/ipfs/store.test.ts @@ -62,6 +62,35 @@ describe("createManifestStore.getVerified", () => { } }); + it("rotates on HTTP 429 like other gateway failures", async () => { + const cid = await manifestCid(PAYLOAD); + const { origin: limited } = await listen((_req, res) => { + res.writeHead(429, { "content-type": "text/plain" }); + res.end("rate-limit-body-must-not-leak"); + }); + const good = await serveBytes(PAYLOAD); + const store = createManifestStore({ + gateways: [gatewayTemplate(limited), gatewayTemplate(good)], + timeoutMs: 2000, + }); + const got = await store.getVerified(cid); + expect(got).toEqual(PAYLOAD); + + const only429 = createManifestStore({ + gateways: [gatewayTemplate(limited)], + timeoutMs: 2000, + }); + try { + await only429.getVerified(cid); + expect.unreachable("should throw"); + } catch (err) { + const message = (err as EnspackError).message; + expect(err).toMatchObject({ code: "FETCH" }); + expect(message).toContain("HTTP 429"); + expect(message).not.toContain("rate-limit-body-must-not-leak"); + } + }); + it("rotates on HTTP 500 without including the response body", async () => { const cid = await manifestCid(PAYLOAD); const { origin: bad } = await listen((_req, res) => { diff --git a/packages/hf/CHANGELOG.md b/packages/hf/CHANGELOG.md index be523cf..d1a4de5 100644 --- a/packages/hf/CHANGELOG.md +++ b/packages/hf/CHANGELOG.md @@ -2,6 +2,8 @@ ## 0.1.0 +- WP-22: `HuggingBayClient.lock` returns `null` on HTTP 404 and 409 (no lock); + other non-2xx still FETCH. Callers skip the cross-check when there is no lock. - Hugging Face Hub client: tree (paginated), revision resolve, model info, license lookup, size-capped download, `buildFiles` (SPEC §8 step 1). - Hugging Bay client: `resolve`, `lock`, `submitFallback` (SPEC §8 step 6, §10). - `crossCheck` against a Hugging Bay lock (SPEC §8 step 1) and `hfWebseed` (SPEC §3). diff --git a/packages/hf/src/huggingbay-client.ts b/packages/hf/src/huggingbay-client.ts index e7f22d2..bde7b72 100644 --- a/packages/hf/src/huggingbay-client.ts +++ b/packages/hf/src/huggingbay-client.ts @@ -158,10 +158,12 @@ export class HuggingBayClient { /** * Fetch a Hugging Bay lock. Observed 2026-09-14: `GET /api/artifacts/{id}/lock` * returns `{ artifacts: [{ files: [{ path, sha256, sizeBytes }] }] }` (SPEC §8 step 1). + * 404 and 409 mean no lock (BOOTSTRAP.md §2 rule 3 only cross-checks where a lock exists). */ - async lock(artifactId: string): Promise { + async lock(artifactId: string): Promise { const url = `${this.baseUrl}/api/artifacts/${encodeURIComponent(artifactId)}/lock`; const res = await this.#get(url); + if (res.status === 404 || res.status === 409) return null; if (!res.ok) { throw new EnspackError("FETCH", `Hugging Bay lock ${artifactId} HTTP ${res.status}`); } diff --git a/packages/hf/test/cross-check.test.ts b/packages/hf/test/cross-check.test.ts index 00d8864..9e077d4 100644 --- a/packages/hf/test/cross-check.test.ts +++ b/packages/hf/test/cross-check.test.ts @@ -11,6 +11,7 @@ describe("crossCheck", () => { }; const hb = new HuggingBayClient({ fetch: createReplayFetch() }); const lock = await hb.lock("hf-model-qwen-qwen2-5-7b-instruct"); + if (lock === null) throw new Error("expected lock"); expect(crossCheck(example.files, lock)).toEqual({ ok: true, compared: example.files.length }); }); @@ -20,6 +21,7 @@ describe("crossCheck", () => { }; const hb = new HuggingBayClient({ fetch: createReplayFetch() }); const lock = await hb.lock("hf-model-qwen-qwen2-5-7b-instruct"); + if (lock === null) throw new Error("expected lock"); const tampered = { ...lock, files: lock.files.map((f) => @@ -43,6 +45,7 @@ describe("crossCheck", () => { }; const hb = new HuggingBayClient({ fetch: createReplayFetch() }); const lock = await hb.lock("hf-model-qwen-qwen2-5-7b-instruct"); + if (lock === null) throw new Error("expected lock"); const tampered = { ...lock, files: lock.files.map((f) => (f.path === "README.md" ? { ...f, size: f.size + 1 } : f)), diff --git a/packages/hf/test/huggingbay.test.ts b/packages/hf/test/huggingbay.test.ts index 0a82d67..30290c2 100644 --- a/packages/hf/test/huggingbay.test.ts +++ b/packages/hf/test/huggingbay.test.ts @@ -38,11 +38,47 @@ describe("HuggingBayClient", () => { it("parses lock files from artifacts[].files sizeBytes", async () => { const client = new HuggingBayClient({ fetch: createReplayFetch() }); const lock = await client.lock("hf-model-qwen-qwen2-5-7b-instruct"); + expect(lock).not.toBeNull(); + if (lock === null) return; expect(lock.files.length).toBeGreaterThan(0); expect(lock.files.some((f) => f.path === "config.json" && f.size === 663)).toBe(true); expect(lock.raw).toBeTruthy(); }); + it("returns null on lock HTTP 404 and 409", async () => { + for (const status of [404, 409]) { + const client = new HuggingBayClient({ + fetch: async (input) => { + const url = + typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + if (url.includes("/lock")) { + return new Response("{}", { + status, + headers: { "content-type": "application/json" }, + }); + } + return new Response("{}", { status: 500 }); + }, + }); + expect(await client.lock("hf-model-missing")).toBeNull(); + } + }); + + it("throws FETCH on lock HTTP 500", async () => { + const client = new HuggingBayClient({ + fetch: async () => + new Response("{}", { status: 500, headers: { "content-type": "application/json" } }), + }); + try { + await client.lock("hf-model-broken"); + expect.unreachable("expected FETCH"); + } catch (err) { + expect(err).toBeInstanceOf(EnspackError); + expect((err as EnspackError).code).toBe("FETCH"); + expect((err as EnspackError).message).toContain("HTTP 500"); + } + }); + it("rejects magnets that fail MAGNET_RE", async () => { const client = new HuggingBayClient({ fetch: createReplayFetch() }); try { diff --git a/test/e2e/test/sepolia.e2e.test.ts b/test/e2e/test/sepolia.e2e.test.ts index 744e733..07ca26d 100644 --- a/test/e2e/test/sepolia.e2e.test.ts +++ b/test/e2e/test/sepolia.e2e.test.ts @@ -190,6 +190,12 @@ describe.skipIf(skip)("sepolia e2e", { timeout: 600_000 }, () => { }); it("enspack get --json from a fresh HF_HOME, then verify", async () => { + const kind = pinKind(process.env); + const existing = process.env.ENSPACK_E2E_NAME; + if (kind === null && (existing === undefined || existing === "")) { + // First test did not publish; there is no live name to get. + return; + } const hfHome = await mkdtemp(join(tmpdir(), "enspack-sepolia-hf-")); tmps.push(hfHome); const env: NodeJS.ProcessEnv = {