diff --git a/CHANGELOG.md b/CHANGELOG.md index 7910cf2680..d7ae4c9091 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ checkpoint, and status-only commits are intentionally omitted. ### Changed +- Live verification now installs a missing target package manager on demand after execution is approved, publishes installer failures as verification results, and guides plans toward stable assertions the run can satisfy. - Live verification now runs immediately after review in the same job and exact reviewed checkout; review judgment gates execution, target children receive a denylist-and-heuristic-sanitized environment, package installs suppress lifecycle scripts unless a repository explicitly opts in, and review jobs default to `ubuntu-latest` without requiring Linux namespaces. Existing publication jobs still validate and upload media before publishing the normal record and comment. - Live verification comments now keep terminal captures but render browser proof as sanitized per-step outcomes with explicit failure reasons, never document-wide page text or empty assertion sections. - Live verification now runs real PR behavior by default, publishes bounded command output and assertion results even without video, and treats recordings as optional presentation. diff --git a/docs/live-proof.md b/docs/live-proof.md index 36128b3505..888451146f 100644 --- a/docs/live-proof.md +++ b/docs/live-proof.md @@ -22,6 +22,15 @@ The planner gates execution in order: the repository must opt in with still runs and publishes verification, but it bypasses recording, transcoding, and poster generation. +Only after those gates pass does the verification child resolve the target +repository profile's `package_manager`. If the configured Bun, pnpm, or npm +executable is missing, the child runs that package manager's official installer +inside the same sanitized scratch profile and verifies that the executable is +available before target setup. Installer failures become a failed +`live-verification.json` result and are published through the normal artifact +path; they do not fail the review itself. Reviews that do not verify never probe +or install a target package manager. + ## Review-job execution After the review command returns, the job inspects the produced reports before @@ -63,6 +72,13 @@ and is unaffected by this live-proof policy. HOME, package-manager caches, and temporary files point into the scratch profile. +Plans must use assertions the demonstration can satisfy. Browser interactions +should derive search or filter values from content the page already renders, +and terminal plans should assert stable output such as a header, flag, or error +string rather than counts, timings, or run-dependent numbers. When no exact +value is certain, the planner must choose a more stable assertion instead of +inventing one. + Browser plans are serialized as JSON data into a generated plain `playwright-core` script; plan values are never inserted as source code. Recorded browser runs use installed Chrome with a 1280x800 video context and diff --git a/prompts/review-item.md b/prompts/review-item.md index 2e8ea4bb35..a9d586fbe3 100644 --- a/prompts/review-item.md +++ b/prompts/review-item.md @@ -459,6 +459,14 @@ plan must be demonstrable from the PR head alone without external accounts, credentials, or third-party services. Step values must never contain secrets or tokens of any kind. +Every assertion must name something the demonstration can actually satisfy. +Assert values that the page or command will genuinely produce: for a search +box, search for a value the page itself already displays; for a command, assert +a stable substring of its output such as a header, flag name, or error string, +not a count, timing, or number that varies per run. If you cannot name a value +the run will certainly print or render, assert something more stable rather +than inventing one. + Judge whether the run has something worth watching, solely to choose its presentation. Choose `payoff.kind: "static_text"` when the whole demonstration is a short burst of plain text that a reader can understand better in a quoted diff --git a/src/live-proof/execute.ts b/src/live-proof/execute.ts index af44b7915e..c045c3c21f 100644 --- a/src/live-proof/execute.ts +++ b/src/live-proof/execute.ts @@ -155,6 +155,13 @@ export async function executeLiveProof( try { let drive: ReturnType; try { + ensureLiveProofPackageManager( + profile.packageManager, + runner, + checkout, + targetEnvironment, + log, + ); for (const configuredCommand of liveTest.setup) { const command = liveProofSetupCommand(configuredCommand, liveTest.allowInstallScripts); requireSuccess("sh", ["-lc", command], runner("sh", ["-lc", command], { cwd: checkout })); @@ -329,6 +336,68 @@ export function liveProofSetupCommand(command: string, allowInstallScripts: bool return command.replace(install[0], `${install[0]} --ignore-scripts`); } +export function liveProofPackageManagerInstallCommand(packageManager: string): string { + switch (packageManager) { + case "bun": + return "curl -fsSL https://bun.sh/install | bash"; + case "pnpm": + return "curl -fsSL https://get.pnpm.io/install.sh | sh -"; + case "npm": + return "curl -fsSL https://www.npmjs.com/install.sh | sh"; + default: + throw new Error( + `unsupported live-proof package manager ${JSON.stringify(packageManager)}; expected bun, pnpm, or npm`, + ); + } +} + +export function ensureLiveProofPackageManager( + packageManager: string, + runner: MediaProofCommandRunner, + checkout: string, + environment: NodeJS.ProcessEnv, + log: (message: string) => void = console.log, +): void { + const installCommand = liveProofPackageManagerInstallCommand(packageManager); + addPackageManagerToPath(packageManager, environment); + const probe = () => + runner("sh", ["-lc", `command -v ${packageManager} >/dev/null 2>&1`], { cwd: checkout }); + if (probe().status === 0) return; + + const installed = runner("sh", ["-lc", installCommand], { + cwd: checkout, + timeoutMs: 2 * 60_000, + }); + if (installed.status !== 0) { + throw new Error( + `could not install live-proof package manager ${packageManager} with official installer (${installCommand}): ${mediaProofSpawnDetail(installed)}`, + ); + } + addPackageManagerToPath(packageManager, environment); + const verified = probe(); + if (verified.status !== 0) { + throw new Error( + `live-proof package manager ${packageManager} is unavailable after its official installer (${installCommand}): ${mediaProofSpawnDetail(verified)}`, + ); + } + log(`[live-proof] installed target package manager ${packageManager}: ${installCommand}`); +} + +function addPackageManagerToPath(packageManager: string, environment: NodeJS.ProcessEnv): void { + const home = environment.HOME?.trim(); + if (!home) return; + const directory = + packageManager === "bun" + ? join(home, ".bun", "bin") + : packageManager === "pnpm" + ? environment.PNPM_HOME?.trim() || join(home, ".local", "share", "pnpm") + : undefined; + if (!directory) return; + const path = environment.PATH ?? ""; + if (!path.split(":").includes(directory)) + environment.PATH = path ? `${directory}:${path}` : directory; +} + function writeVerificationResult( options: Parameters[0] & { path: string }, ): void { diff --git a/src/repository-profiles.ts b/src/repository-profiles.ts index b6779302f7..710ce492a1 100644 --- a/src/repository-profiles.ts +++ b/src/repository-profiles.ts @@ -4,6 +4,7 @@ import { fileURLToPath } from "node:url"; export type RepositoryItemKind = "issue" | "pull_request"; export type RepositoryLiveTestSurface = "browser" | "terminal"; +export type RepositoryPackageManager = "bun" | "pnpm" | "npm"; export interface RepositoryLiveTestConfig { enabled: boolean; @@ -40,6 +41,7 @@ export interface RepositoryProfile { slug: string; displayName: string; checkoutDir: string; + packageManager: RepositoryPackageManager; docsUrl?: string; communityUrl?: string; promptNote: string; @@ -57,6 +59,7 @@ interface ConfiguredRepositoryProfile { targetRepo: string; displayName: string; checkoutDir: string; + packageManager: RepositoryPackageManager; docsUrl?: string; communityUrl?: string; promptNote: string; @@ -68,6 +71,7 @@ interface GenericFallbackConfig { owner: string; denyRepositories: readonly string[]; allowRepoNamePattern: RegExp; + packageManager: RepositoryPackageManager; promptNote: string; applyCloseRules: Partial>; liveTest?: RepositoryLiveTestConfig; @@ -103,6 +107,7 @@ const CORE_OPENCLAW_PROFILE: RepositoryProfile = { slug: "openclaw-openclaw", displayName: "OpenClaw", checkoutDir: "openclaw", + packageManager: "pnpm", docsUrl: "https://docs.openclaw.ai", communityUrl: "https://clawhub.ai/", promptNote: @@ -182,6 +187,7 @@ function configuredRepositoryProfile(profile: ConfiguredRepositoryProfile): Repo slug: slugForRepo(targetRepo), displayName: profile.displayName, checkoutDir: profile.checkoutDir, + packageManager: profile.packageManager, promptNote: profile.promptNote, applyCloseRules: profile.applyCloseRules, }; @@ -210,6 +216,7 @@ function fallbackRepositoryProfile(normalizedTargetRepo: string): RepositoryProf slug: slugForRepo(normalizedTargetRepo), displayName: repoName, checkoutDir: repoName, + packageManager: fallback.packageManager, promptNote: fallback.promptNote .replaceAll("{target_repo}", normalizedTargetRepo) .replaceAll("{repo_name}", repoName), @@ -302,6 +309,7 @@ function validateConfiguredRepositoryProfile( targetRepo: repoValue(profile.target_repo, `${label}.target_repo`), displayName: stringValue(profile.display_name, `${label}.display_name`), checkoutDir: pathSegmentValue(profile.checkout_dir, `${label}.checkout_dir`), + packageManager: packageManagerValue(profile.package_manager, `${label}.package_manager`), promptNote: stringValue(profile.prompt_note, `${label}.prompt_note`), applyCloseRules: closeRulesValue(profile.apply_close_rules, `${label}.apply_close_rules`), }; @@ -390,6 +398,7 @@ function validateGenericFallbackConfig( (entry, index) => normalizeRepo(repoValue(entry, `${label}.deny_repositories[${index}]`)), ), allowRepoNamePattern: new RegExp(pattern), + packageManager: packageManagerValue(fallback.package_manager, `${label}.package_manager`), promptNote: stringValue(fallback.prompt_note, `${label}.prompt_note`), applyCloseRules: closeRulesValue(fallback.apply_close_rules, `${label}.apply_close_rules`), }; @@ -436,6 +445,14 @@ function pathSegmentValue(value: unknown, label: string): string { return segment; } +function packageManagerValue(value: unknown, label: string): RepositoryPackageManager { + const packageManager = value === undefined ? "pnpm" : stringValue(value, label).toLowerCase(); + if (packageManager !== "bun" && packageManager !== "pnpm" && packageManager !== "npm") { + throw new Error(`${label} must be bun, pnpm, or npm`); + } + return packageManager; +} + function stringValue(value: unknown, label: string): string { if (typeof value !== "string" || value.trim() === "") throw new Error(`${label} must be a string`); diff --git a/test/live-proof-review-environment.test.ts b/test/live-proof-review-environment.test.ts index b7ec7652aa..9f0b953dd5 100644 --- a/test/live-proof-review-environment.test.ts +++ b/test/live-proof-review-environment.test.ts @@ -82,6 +82,7 @@ test( slug: "openclaw-sanitized-fixture", displayName: "fixture", checkoutDir: "fixture", + packageManager: "pnpm", promptNote: "fixture", applyCloseRules: {}, liveTest: { diff --git a/test/live-proof.test.ts b/test/live-proof.test.ts index d20aea2c46..e807f25b37 100644 --- a/test/live-proof.test.ts +++ b/test/live-proof.test.ts @@ -22,7 +22,12 @@ import { generatePlaywrightScript, terminalCommandPlan, } from "../dist/live-proof/drivers.js"; -import { executeLiveProof, liveProofSetupCommand } from "../dist/live-proof/execute.js"; +import { + ensureLiveProofPackageManager, + executeLiveProof, + liveProofPackageManagerInstallCommand, + liveProofSetupCommand, +} from "../dist/live-proof/execute.js"; import { assertLiveProofEnvironmentSanitized, sanitizedLiveProofEnvironment, @@ -71,6 +76,7 @@ function profile(enabled = true): RepositoryProfile { slug: "example-repo", displayName: "Example", checkoutDir: "repo", + packageManager: "pnpm", promptNote: "Example profile.", applyCloseRules: {}, liveTest: { @@ -329,6 +335,53 @@ test("live-proof install setup disables lifecycle scripts unless the profile opt ); }); +test("live-proof installs a missing Bun toolchain with the official installer", () => { + const calls: Array<{ command: string; args: readonly string[]; path?: string }> = []; + const logs: string[] = []; + const environment: NodeJS.ProcessEnv = { HOME: "/tmp/live-proof-home", PATH: "/usr/bin" }; + let probes = 0; + ensureLiveProofPackageManager( + "bun", + (command, args, options) => { + calls.push({ command, args, path: options?.env?.PATH ?? environment.PATH }); + if (String(args[1]).startsWith("command -v bun")) { + probes += 1; + return { status: probes === 1 ? 1 : 0 }; + } + return { status: 0 }; + }, + "/tmp/checkout", + environment, + (message) => logs.push(message), + ); + + assert.deepEqual( + calls.map(({ command, args }) => [command, ...args].join(" ")), + [ + "sh -lc command -v bun >/dev/null 2>&1", + "sh -lc curl -fsSL https://bun.sh/install | bash", + "sh -lc command -v bun >/dev/null 2>&1", + ], + ); + assert.match(environment.PATH ?? "", /^\/tmp\/live-proof-home\/\.bun\/bin:/); + assert.match(logs.join("\n"), /installed target package manager bun/); +}); + +test("live-proof reports an unsupported package manager clearly", () => { + assert.throws( + () => + ensureLiveProofPackageManager("yarn", () => ({ status: 1 }), "/tmp/checkout", { + HOME: "/tmp/live-proof-home", + PATH: "/usr/bin", + }), + /unsupported live-proof package manager "yarn"; expected bun, pnpm, or npm/, + ); + assert.equal( + liveProofPackageManagerInstallCommand("bun"), + "curl -fsSL https://bun.sh/install | bash", + ); +}); + test("Playwright generation keeps quotes, backticks, and newlines inside JSON data", () => { const script = generatePlaywrightScript([ { @@ -777,10 +830,13 @@ test("execution setup failures still produce a failed verification result", asyn }, { env: { CLAWSWEEPER_LIVE_PROOF_ENABLED: "1" }, - runner: (command) => - command === "git" - ? { status: 0, stdout: `${HEAD}\n` } - : { status: 1, stderr: "setup exploded" }, + runner: (command, args) => { + if (command === "git") return { status: 0, stdout: `${HEAD}\n` }; + if (command === "sh" && String(args[1]).startsWith("command -v pnpm")) { + return { status: 0 }; + } + return { status: 1, stderr: "setup exploded" }; + }, repositoryProfileFor: () => ({ ...profile(), liveTest: { ...profile().liveTest!, setup: ["pnpm install"] }, @@ -812,6 +868,54 @@ test("execution setup failures still produce a failed verification result", asyn assert.equal(existsSync(join(outputDir, "live-proof-manifest.json")), false); }); +test("toolchain installer failures produce a published verification result", async () => { + const directory = mkdtempSync(join(tmpdir(), "clawsweeper-live-toolchain-failure-")); + const outputDir = join(directory, "output"); + const planPath = join(directory, "plan.json"); + const plan = recommendedPlan("terminal"); + writeFileSync(planPath, JSON.stringify(plan), "utf8"); + + await executeLiveProof( + { + repo: "example/repo", + item: 42, + outputDir, + planPath, + checkoutPath: directory, + }, + { + env: { CLAWSWEEPER_LIVE_PROOF_ENABLED: "1" }, + runner: (command, args) => { + if (command === "git") return { status: 0, stdout: `${HEAD}\n` }; + if (String(args[1]).startsWith("command -v bun")) return { status: 1 }; + if (String(args[1]) === "curl -fsSL https://bun.sh/install | bash") { + return { status: 1, stderr: "network unavailable" }; + } + return { status: 0 }; + }, + repositoryProfileFor: () => ({ ...profile(), packageManager: "bun" }), + reportLiveProofPlan: () => plan, + parseLiveProofPlan: () => plan, + fetchPullRequest: async () => { + throw new Error("local checkout must not fetch the pull request"); + }, + now: () => new Date("2026-08-17T12:00:00.000Z"), + log: () => undefined, + }, + ); + + const verification = parseLiveVerificationResult( + JSON.parse(readFileSync(join(outputDir, "live-verification.json"), "utf8")) as unknown, + ); + assert.equal(verification.overall_pass, false); + assert.match( + verification.failure?.reason ?? "", + /could not install live-proof package manager bun with official installer/, + ); + assert.match(verification.output, /curl -fsSL https:\/\/bun\.sh\/install \| bash/); + assert.match(verification.output, /network unavailable/); +}); + test("live proof manifest is metadata-only and rejects URL-bearing extensions", () => { const manifest = validManifest(); assert.deepEqual(parseLiveProofManifest(manifest), manifest); diff --git a/test/repository-profiles.test.ts b/test/repository-profiles.test.ts index 917703b055..5e0bbd73ad 100644 --- a/test/repository-profiles.test.ts +++ b/test/repository-profiles.test.ts @@ -78,6 +78,7 @@ test("repositoryProfileFor matches mixed-case input against canonical profiles", assert.equal(profile.targetRepo, "openclaw/clawhub"); assert.equal(profile.slug, "openclaw-clawhub"); + assert.equal(profile.packageManager, "bun"); assert.deepEqual(profile.applyCloseRules.issue, ["implemented_on_main"]); assert.deepEqual(profile.applyCloseRules.pull_request, [ "implemented_on_main", @@ -113,6 +114,7 @@ test("generic OpenClaw fallback supports conservative event-only onboarding", () "mostly_implemented_on_main", ]); assert.deepEqual(profile.liveTest, TERMINAL_LIVE_TEST); + assert.equal(profile.packageManager, "pnpm"); }); test("generic steipete fallback starts review-only", () => { @@ -200,6 +202,16 @@ test("schema v2 repository profiles strictly validate optional live_test config" () => validateTargetRepositoryConfigForTest(targetRepositoryConfig(liveTest, 1)), /live_test requires schema_version 2/, ); + assert.throws( + () => + validateTargetRepositoryConfigForTest({ + ...targetRepositoryConfig(liveTest), + repositories: [ + { ...targetRepositoryConfig(liveTest).repositories[0], package_manager: "yarn" }, + ], + }), + /package_manager must be bun, pnpm, or npm/, + ); }); test("terminal live_test profiles may omit browser start and URL fields", () => { diff --git a/test/review-prompt-policy.test.ts b/test/review-prompt-policy.test.ts index b3c53c8cb1..a9a83288ce 100644 --- a/test/review-prompt-policy.test.ts +++ b/test/review-prompt-policy.test.ts @@ -240,6 +240,22 @@ test("review prompt accepts real production transport-boundary proof for reliabi ); }); +test("review prompt requires live-proof assertions the demonstration can satisfy", () => { + const prompt = readFileSync("prompts/review-item.md", "utf8"); + + assert.match( + prompt, + /Every assertion must name something the demonstration can actually satisfy/, + ); + assert.match(prompt, /search for a value the page itself already displays/); + assert.match( + prompt, + /stable substring of its output such as a header, flag name, or error string/, + ); + assert.match(prompt, /not a count, timing, or number that varies per run/); + assert.match(prompt, /assert something more stable rather\s+than inventing one/); +}); + test("generated shared-channel review prompt preserves scoped policy and real fault evidence", () => { const proof = "The real production owner and grammY HTTP client produced a recorded 429 older → 200 newest trace against a local HTTP server.";