diff --git a/README.md b/README.md index cc14fa6..fd33b2c 100644 --- a/README.md +++ b/README.md @@ -453,6 +453,13 @@ agentforge list-targets selected Claude publications, and for a `root-manifest` publication also validates the marketplace root. It is opt-in so the default check does not require Claude Code to be installed. +- `--json` emits the result as a single JSON document on stdout instead of the + human lines, for a CI job or a consuming tool that would otherwise parse the + diagnostic stream back apart. The document carries a `schemaVersion`, a + hoisted `status` (`ok` / `failed`), a per-publication status and file count, + the issues in their stable path-then-code order, and the compilation + diagnostics. Read `status` rather than inferring success from an empty + `issues` array. Exit codes are unchanged. ### `root-manifest` publications diff --git a/package.json b/package.json index 7be6ab4..90488e0 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "./definitions": "./src/definitions.ts", "./materializer": "./src/materializer.ts", "./marketplace-adapters": "./src/marketplace-adapters.ts", + "./report": "./src/report.ts", "./render": "./src/render.ts" }, "scripts": { diff --git a/src/cli.ts b/src/cli.ts index b0f22f4..ff69cf7 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -10,7 +10,7 @@ import { type LoadedMarketplace, loadMarketplaceDefinition } from './definitions import { claudeMarketplaceAdapter, codexMarketplaceAdapter } from './marketplace-adapters.ts'; import { materializeCompilation } from './materializer.ts'; import { render } from './render.ts'; -import { formatFromPath, type ReportFormat, renderReport } from './report.ts'; +import { buildCheckReport, formatFromPath, type ReportFormat, renderReport } from './report.ts'; import { rootDisplayPath } from './root-manifest.ts'; import { ARTIFACT_DEFS } from './schema.ts'; import { allTargets } from './targets/index.ts'; @@ -245,10 +245,11 @@ program '--claude-native', 'cross-check selected Claude publications with claude plugin validate --strict', ) + .option('--json', 'emit the result as a JSON document on stdout instead of human lines') .action( async ( marketplace: string, - opts: { out: string; publication: string[]; claudeNative?: boolean }, + opts: { out: string; publication: string[]; claudeNative?: boolean; json?: boolean }, ) => { try { const loaded = await loadMarketplaceDefinition(resolve(marketplace)); @@ -257,20 +258,31 @@ program const plan = compileSelectedMarketplace(selected, outputRoot); const result = checkMarketplace(plan, outputRoot); - for (const publication of selected.definition.publications.toSorted((left, right) => - compareStrings(left.id, right.id), - )) { - const count = - result.filesChecked.filter((path) => path.startsWith(`${publication.id}/`)).length + - result.rootFilesChecked.filter(({ publicationId }) => publicationId === publication.id) - .length; - const status = result.issues.some(({ publicationId }) => publicationId === publication.id) - ? 'failed' - : 'ok'; - console.log(`[${publication.id}] ${status}: ${count} managed files`); + // `--json` owns stdout entirely: a consumer parses the whole stream, so + // one stray human line makes the document unparseable. The native + // cross-check below still writes through, because it is a separate + // tool's output and suppressing it would hide why the run failed. + if (opts.json) { + console.log(JSON.stringify(buildCheckReport(plan, result), null, 2)); + } else { + for (const publication of selected.definition.publications.toSorted((left, right) => + compareStrings(left.id, right.id), + )) { + const count = + result.filesChecked.filter((path) => path.startsWith(`${publication.id}/`)).length + + result.rootFilesChecked.filter( + ({ publicationId }) => publicationId === publication.id, + ).length; + const status = result.issues.some( + ({ publicationId }) => publicationId === publication.id, + ) + ? 'failed' + : 'ok'; + console.log(`[${publication.id}] ${status}: ${count} managed files`); + } + for (const line of formatCompilationDiagnostics(plan)) console.log(line); + for (const issue of result.issues) console.error(formatCheckIssue(issue)); } - for (const line of formatCompilationDiagnostics(plan)) console.log(line); - for (const issue of result.issues) console.error(formatCheckIssue(issue)); let failed = result.issues.length > 0; if (opts.claudeNative) { diff --git a/src/report.ts b/src/report.ts index dd6361f..f0256f1 100644 --- a/src/report.ts +++ b/src/report.ts @@ -1,4 +1,5 @@ import { dirname } from 'node:path'; +import type { MarketplaceCheckIssue, MarketplaceCheckResult } from './check.ts'; import type { CompilationDiagnostic, CompilationPlan } from './compiler.ts'; import { portableRelativePath } from './definitions.ts'; import { rootDisplayPath } from './root-manifest.ts'; @@ -350,3 +351,90 @@ function groupSection(heading: string, group: ReportGroup): string[] { lines.push(''); return lines; } + +// The check report: what `check --json` emits. +// +// Kept apart from `CompilationReport` rather than folded into it. That report +// answers "what became of each construct" and groups by disposition +// (ndr:71jgk2); this one answers "is this compiled tree fit to publish", and its +// codes are already dispositional — `missing-output`, `changed-output`, +// `unsafe-output-content` each name what happened. There is no severity axis to +// reorganise, so imposing the disposition grouping would add a layer that +// classifies nothing. +// +// It carries `schemaVersion` because it is a machine-targeted format, which is +// exactly what ndr:r51yhr binds. +const CHECK_SCHEMA_VERSION = 1; + +export interface CheckReportPublication { + id: string; + status: 'ok' | 'failed'; + // Managed files compared against the plan, including any root-anchored + // manifest this publication owns. + filesChecked: number; +} + +export interface CheckReport { + schemaVersion: number; + marketplaceId: string; + // `ok` only when no publication failed. Hoisted so a consumer can branch + // without walking the issue list, and so an empty `issues` array is never the + // only signal — a reader who mistakes "no issues parsed" for "passed" is the + // failure mode the text format already had. + status: 'ok' | 'failed'; + publications: CheckReportPublication[]; + issues: MarketplaceCheckIssue[]; + diagnostics: ReportedCheckDiagnostic[]; +} + +export interface ReportedCheckDiagnostic { + code: string; + severity: CompilationDiagnostic['severity']; + target: string; + publicationId: string; + packageId?: string; + message: string; +} + +export function buildCheckReport( + plan: CompilationPlan, + result: MarketplaceCheckResult, +): CheckReport { + const failed = new Set(result.issues.map(({ publicationId }) => publicationId)); + const publicationIds = new Set([ + ...plan.outputs.map(({ provenance }) => provenance.publicationId), + ...plan.rootOutputs.map(({ provenance }) => provenance.publicationId), + ]); + + const publications = [...publicationIds].toSorted(compareReportStrings).map((id) => ({ + id, + status: (failed.has(id) ? 'failed' : 'ok') as 'ok' | 'failed', + filesChecked: + result.filesChecked.filter((path) => path.startsWith(`${id}/`)).length + + result.rootFilesChecked.filter(({ publicationId }) => publicationId === id).length, + })); + + return { + schemaVersion: CHECK_SCHEMA_VERSION, + marketplaceId: plan.marketplaceId, + status: result.issues.length > 0 ? 'failed' : 'ok', + publications, + // Already ordered by path then code in `checkMarketplace`, and left that + // way: a stable order is what makes two runs diffable. + issues: result.issues, + diagnostics: plan.diagnostics.map((diagnostic) => ({ + code: diagnostic.code, + severity: diagnostic.severity, + target: diagnostic.target, + publicationId: diagnostic.provenance.publicationId, + ...(diagnostic.provenance.packageId === undefined + ? {} + : { packageId: diagnostic.provenance.packageId }), + message: diagnostic.message, + })), + }; +} + +function compareReportStrings(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} diff --git a/tests/check.test.ts b/tests/check.test.ts index c0979ac..3c5a0a4 100644 --- a/tests/check.test.ts +++ b/tests/check.test.ts @@ -15,6 +15,7 @@ import { join } from 'node:path'; import { checkMarketplace } from 'agentforge/check'; import type { CompilationPlan } from 'agentforge/compiler'; import { materializeCompilation } from 'agentforge/materializer'; +import { buildCheckReport } from 'agentforge/report'; let temporaryRoot: string; @@ -435,6 +436,75 @@ describe('marketplace check', () => { expect(result.issues).toEqual([]); }); + + test('builds a machine-readable report for a clean publication', () => { + const source = join(temporaryRoot, 'source.txt'); + const outputRoot = join(temporaryRoot, 'output'); + writeFileSync(source, 'copied\n'); + const plan = fixturePlan(source); + materializeCompilation(plan, outputRoot); + + const report = buildCheckReport(plan, checkMarketplace(plan, outputRoot)); + + expect(report.schemaVersion).toBe(1); + expect(report.status).toBe('ok'); + expect(report.issues).toEqual([]); + expect(report.publications).toEqual([{ id: 'claude', status: 'ok', filesChecked: 2 }]); + }); + + // The hoisted status is what keeps a consumer from reading "no issues parsed" + // as "passed" — the exact failure the text format invited. + test('reports failed status and the issues alongside it', () => { + const source = join(temporaryRoot, 'source.txt'); + const outputRoot = join(temporaryRoot, 'output'); + writeFileSync(source, 'copied\n'); + const plan = fixturePlan(source); + materializeCompilation(plan, outputRoot); + writeFileSync(join(outputRoot, 'claude', 'generated.json'), '{"ok":false}\n'); + + const report = buildCheckReport(plan, checkMarketplace(plan, outputRoot)); + + expect(report.status).toBe('failed'); + expect(report.publications).toEqual([{ id: 'claude', status: 'failed', filesChecked: 2 }]); + expect(report.issues.map(({ code }) => code)).toContain('changed-output'); + }); + + test('carries compilation diagnostics with their provenance flattened', () => { + const source = join(temporaryRoot, 'source.txt'); + const outputRoot = join(temporaryRoot, 'output'); + writeFileSync(source, 'copied\n'); + const base = fixturePlan(source); + const plan: CompilationPlan = { + ...base, + diagnostics: [ + { + code: 'claude-only-frontmatter-stripped', + severity: 'warning', + message: 'Skill "demo": stripped allowed-tools.', + target: 'codex', + provenance: { + marketplacePath: '/fixture/MARKETPLACE.yaml', + publicationId: 'claude', + packageId: 'example', + }, + }, + ], + }; + materializeCompilation(plan, outputRoot); + + const report = buildCheckReport(plan, checkMarketplace(plan, outputRoot)); + + expect(report.diagnostics).toEqual([ + { + code: 'claude-only-frontmatter-stripped', + severity: 'warning', + target: 'codex', + publicationId: 'claude', + packageId: 'example', + message: 'Skill "demo": stripped allowed-tools.', + }, + ]); + }); }); function jsonResourcePlan(sourcePath: string): CompilationPlan {