diff --git a/app/api/game-assets/[slug]/[file]/route.ts b/app/api/game-assets/[slug]/[file]/route.ts index d5bc219..f67931a 100644 --- a/app/api/game-assets/[slug]/[file]/route.ts +++ b/app/api/game-assets/[slug]/[file]/route.ts @@ -16,7 +16,7 @@ function corsHeaders(extra: Record = {}): Record }; } -export async function GET(_request: NextRequest, context: RouteContext) { +export async function GET(request: NextRequest, context: RouteContext) { const { slug: rawSlug, file: rawFile } = await context.params; const slug = rawSlug.trim().toLowerCase(); const filename = rawFile.trim(); @@ -37,8 +37,10 @@ export async function GET(_request: NextRequest, context: RouteContext) { status: 200, headers: corsHeaders({ "Content-Type": "model/gltf-binary", - "Content-Disposition": `attachment; filename="${filename}"`, - "Cache-Control": "public, max-age=86400, s-maxage=86400", + "Content-Disposition": `inline; filename="${filename}"`, + "Cache-Control": request.nextUrl.searchParams.has("v") + ? "private, no-store" + : "public, max-age=86400, s-maxage=86400", }), }); } diff --git a/app/api/reverse-game/route.ts b/app/api/reverse-game/route.ts index dc5dfb0..5ec7e4b 100644 --- a/app/api/reverse-game/route.ts +++ b/app/api/reverse-game/route.ts @@ -68,6 +68,7 @@ async function executeGameReverse(opts: { gameName, force, onStatus: (message) => send("status", { message }), + onHero: (hero) => send("hero", hero), }); if (!result.ok) { diff --git a/components/game-reverse-page.tsx b/components/game-reverse-page.tsx index f9bcc44..ed20df4 100644 --- a/components/game-reverse-page.tsx +++ b/components/game-reverse-page.tsx @@ -4,10 +4,16 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { useRouter } from "next/navigation"; import { GameSpecFlavorText } from "@/components/game-spec-flavor-text"; import { HeroKernelPreview } from "@/components/hero-kernel-preview"; +import { MeshyLiveStage } from "@/components/meshy-live-stage"; import { Navbar } from "@/components/navbar"; import { PromptMarkdown } from "@/components/prompt-markdown"; import { nameToSlug, parseGameInput } from "@/lib/parse-game-input"; import type { StoredHeroAsset } from "@/lib/game-asset-storage"; +import { + mergeHeroProgress, + parseHeroProgressEvent, + type HeroProgressEvent, +} from "@/lib/meshy-progress"; type GameReversePageProps = { gameSlug: string; @@ -29,6 +35,7 @@ export function GameReversePage({ gameSlug, gameName }: GameReversePageProps) { const [heroAssets, setHeroAssets] = useState< Array >([]); + const [liveHeroes, setLiveHeroes] = useState([]); const started = useRef(false); const resultsRef = useRef(null); @@ -53,6 +60,7 @@ export function GameReversePage({ gameSlug, gameName }: GameReversePageProps) { setError(null); setPrompt(null); setHeroAssets([]); + setLiveHeroes([]); setStatusLine("Checking if it's cached…"); try { @@ -116,11 +124,17 @@ export function GameReversePage({ gameSlug, gameName }: GameReversePageProps) { prompt?: string; fromCache?: boolean; error?: string; - }; + } & Partial; if (event === "status" && typeof json.message === "string") { setStatusLine(json.message); } + if (event === "hero") { + const hero = parseHeroProgressEvent(json); + if (hero) { + setLiveHeroes((prev) => mergeHeroProgress(prev, hero)); + } + } if (event === "done" && typeof json.prompt === "string") { setPrompt(json.prompt); if (json.fromCache) setStatusLine("Loaded from cache"); @@ -303,6 +317,9 @@ export function GameReversePage({ gameSlug, gameName }: GameReversePageProps) { ) : null} + {loading && liveHeroes.length > 0 ? ( + + ) : null} {prompt ? ( diff --git a/components/hero-kernel-preview.tsx b/components/hero-kernel-preview.tsx index 4dd91de..4edcb2d 100644 --- a/components/hero-kernel-preview.tsx +++ b/components/hero-kernel-preview.tsx @@ -12,6 +12,7 @@ type HeroKernelPreviewProps = { title?: string; subtitle?: string; autoClip?: QuaterniusClip; + compact?: boolean; }; export function HeroKernelPreview({ @@ -19,6 +20,7 @@ export function HeroKernelPreview({ title = "Quaternius kernel", subtitle, autoClip = "Idle_Loop", + compact = false, }: HeroKernelPreviewProps) { const canvasRef = useRef(null); const [clip, setClip] = useState(autoClip); @@ -109,8 +111,14 @@ export function HeroKernelPreview({ if (disposed) return; const root = gltf.scene; + let skin: import("three").SkinnedMesh | null = null; root.traverse((obj) => { - const mesh = obj as import("three").Mesh; + const mesh = obj as import("three").SkinnedMesh; + if (mesh.isSkinnedMesh) { + mesh.bind(mesh.skeleton, mesh.bindMatrix); + mesh.frustumCulled = false; + if (!skin) skin = mesh; + } if (mesh.isMesh) { mesh.castShadow = true; mesh.receiveShadow = true; @@ -122,10 +130,10 @@ export function HeroKernelPreview({ const names = clips.map((c) => c.name).filter(Boolean); setAvailable(names.length ? names : [...KERNEL_PREVIEW_CLIPS]); - mixer = new THREE.AnimationMixer(root); + mixer = new THREE.AnimationMixer(skin ?? root); const actions = new Map(); for (const c of clips) { - const action = mixer.clipAction(c); + const action = mixer.clipAction(c.clone()); action.enabled = true; actions.set(c.name, action); } @@ -198,26 +206,32 @@ export function HeroKernelPreview({ return (
-
-
-

{title}

-

- {subtitle ?? "Quaternius Universal Animation Library kernel"} + {compact ? null : ( +

+
+

{title}

+

+ {subtitle ?? "Quaternius Universal Animation Library kernel"} +

+
+

+ {error ? error : status}

-

- {error ? error : status} -

-
+ )}
{error ? (

{error}

+ ) : compact ? ( +

+ {status} +

) : (
{clipButtons.map((name) => ( diff --git a/components/meshy-live-stage.tsx b/components/meshy-live-stage.tsx new file mode 100644 index 0000000..19b7eb0 --- /dev/null +++ b/components/meshy-live-stage.tsx @@ -0,0 +1,69 @@ +"use client"; + +import { HeroKernelPreview } from "@/components/hero-kernel-preview"; +import type { HeroProgressEvent } from "@/lib/meshy-progress"; + +function stageLabel(hero: HeroProgressEvent): string { + if (hero.stage === "ready") { + return hero.kernel ? "Walking" : "Ready"; + } + if (hero.stage === "kernel") return "Auto-rig"; + if (hero.stage === "refine") return `Texture ${hero.progress}%`; + if (hero.stage === "failed") return "Failed"; + return `Sculpt ${hero.progress}%`; +} + +function LiveHeroCard({ hero }: { hero: HeroProgressEvent }) { + const moving = Boolean(hero.kernel && hero.modelUrl); + return ( +
+
+
+
+

{hero.id}

+

+ {stageLabel(hero)} +

+
+
+
+
+ {hero.modelUrl ? ( + + ) : hero.thumbnailUrl ? ( +
+ {/* eslint-disable-next-line @next/next/no-img-element */} + +
+ ) : ( +
+ {hero.status} +
+ )} +
+
+ ); +} + +export function MeshyLiveStage({ heroes }: { heroes: HeroProgressEvent[] }) { + if (!heroes.length) return null; + return ( +
+ {heroes.map((hero) => ( + + ))} +
+ ); +} diff --git a/lib/game-hero-assets.ts b/lib/game-hero-assets.ts index 874a732..a640ec0 100644 --- a/lib/game-hero-assets.ts +++ b/lib/game-hero-assets.ts @@ -2,6 +2,7 @@ import type { LlmTarget } from "@/lib/quick-llm"; import { callQuickLlm } from "@/lib/quick-llm"; import { generateTexturedGlb, getMeshyApiKey } from "@/lib/meshy-client"; import { autoRigToQuaterniusKernel } from "@/lib/auto-rig-humanoid"; +import type { HeroProgressEvent } from "@/lib/meshy-progress"; import { QUATERNIUS_KERNEL_ID, QUATERNIUS_ROOT_MOTION_PUBLIC_PATH, @@ -24,7 +25,7 @@ type PlannedAsset = { const ASSET_ID_RE = /^[a-z][a-z0-9-]{0,31}$/; const GENERATED_SECTION_RE = - /\n*## 10\. Generated hero assets[\s\S]*$/i; + /\n*## 10\. (?:Generated hero assets|Movement kernel)[\s\S]*$/i; function parseJsonObject(text: string): unknown { const trimmed = text @@ -56,8 +57,6 @@ export function appendGeneratedAssetsSection( assets: StoredHeroAsset[] ): string { const base = stripGeneratedAssetsSection(specMd); - if (!assets.length) return base; - const rows = assets .map((asset) => { const note = asset.kernel @@ -71,15 +70,23 @@ export function appendGeneratedAssetsSection( }) .join("\n"); - const kernelHint = assets.some((a) => a.kernel) - ? ` -- Clips are already embedded. Drive them with Three.js \`AnimationMixer\` using these names: \`Idle_Loop\`, \`Walk_Loop\`, \`Jog_Fwd_Loop\`, \`Sprint_Loop\`, \`Jump_Start\` / \`Jump_Loop\` / \`Jump_Land\`, \`Punch_Jab\`, \`Sword_Attack\`, \`Death01\`. + const kernelHint = ` +- Clips are already embedded on kernel-rigged heroes. Drive them with Three.js \`AnimationMixer\` using these names: \`Idle_Loop\`, \`Walk_Loop\`, \`Jog_Fwd_Loop\`, \`Sprint_Loop\`, \`Jump_Start\` / \`Jump_Loop\` / \`Jump_Land\`, \`Punch_Jab\`, \`Sword_Attack\`, \`Death01\`. - Do not T-pose, invent keyframes, or replace the hero with boxes. -- In-place locomotion is baked in. For traveling root motion, also load \`${getSiteBaseUrl()}${QUATERNIUS_ROOT_MOTION_PUBLIC_PATH}\` (same bone names). -` - : ` -- Play the walk clip on humanoid heroes while moving. Pause it when idle. +- In-place locomotion: \`${getSiteBaseUrl()}${QUATERNIUS_STANDARD_PUBLIC_PATH}\`. Traveling root motion: \`${getSiteBaseUrl()}${QUATERNIUS_ROOT_MOTION_PUBLIC_PATH}\` (same bone names). +`; + + if (!assets.length) { + return `${base} + +## 10. Movement kernel + +Humanoid heroes auto-rig onto the Quaternius Universal skeleton. Load the kernel GLB with Three.js \`GLTFLoader\` if no generated hero file is listed. + +${kernelHint.trim()} +- Keep buildings, roads, and repeating world dressing procedural. `; + } return `${base} @@ -101,7 +108,6 @@ export function appendHeroAssetInstructions( slug: string, assets: StoredHeroAsset[] ): string { - if (!assets.length) return prompt; const lines = assets.map((asset) => { const extra = asset.kernel ? " (Quaternius Universal rig; play Idle_Loop / Walk_Loop / Sprint_Loop — do not T-pose)" @@ -110,10 +116,11 @@ export function appendHeroAssetInstructions( : ""; return `- ${asset.filename}${extra}: ${gameAssetFileUrl(slug, asset.filename)}`; }); - const kernelLine = assets.some((a) => a.kernel) - ? `\nClip names: Idle_Loop (stand), Walk_Loop (move), Jog_Fwd_Loop, Sprint_Loop (run), Jump_Start then Jump_Loop then Jump_Land, Punch_Jab / Punch_Cross, Sword_Attack, Death01. Use AnimationMixer. Optional traveling locomotion: ${getSiteBaseUrl()}${QUATERNIUS_ROOT_MOTION_PUBLIC_PATH} (same skeleton). In-place kernel copy: ${getSiteBaseUrl()}${QUATERNIUS_STANDARD_PUBLIC_PATH}.` - : ""; - const block = `Download these 3D models into public/models/ and load them with GLTFLoader. Do not rebuild these heroes from boxes.\n${lines.join("\n")}${kernelLine}`; + const kernelLine = `Clip names: Idle_Loop (stand), Walk_Loop (move), Jog_Fwd_Loop, Sprint_Loop (run), Jump_Start then Jump_Loop then Jump_Land, Punch_Jab / Punch_Cross, Sword_Attack, Death01. Use AnimationMixer. Optional traveling locomotion: ${getSiteBaseUrl()}${QUATERNIUS_ROOT_MOTION_PUBLIC_PATH} (same skeleton). In-place kernel copy: ${getSiteBaseUrl()}${QUATERNIUS_STANDARD_PUBLIC_PATH}.`; + const downloads = lines.length + ? `${lines.join("\n")}\n${kernelLine}` + : `- Quaternius Universal kernel (auto-rig target; play Idle_Loop / Walk_Loop / Sprint_Loop — do not T-pose): ${getSiteBaseUrl()}${QUATERNIUS_STANDARD_PUBLIC_PATH}\n${kernelLine}`; + const block = `Download these 3D models into public/models/ and load them with GLTFLoader. Do not rebuild these heroes from boxes.\n${downloads}`; const stripped = prompt .replace( /\n*Download these 3D models into public\/models\/[\s\S]*?(?=\n\nUse this game spec:|$)/i, @@ -192,6 +199,21 @@ function heuristicPlan(gameName: string, specMd: string): PlannedAsset[] { ]; } +function liveAssetUrl(slug: string, filename: string, rev: string): string { + return `/api/game-assets/${encodeURIComponent(slug)}/${encodeURIComponent(filename)}?v=${encodeURIComponent(rev)}`; +} + +async function publishHeroGlb( + slug: string, + id: string, + bytes: Buffer, + rev: string +): Promise { + const filename = `${id}.glb`; + await writeGameAssetFile(slug, filename, bytes); + return liveAssetUrl(slug, filename, rev); +} + export async function generateHeroAssets(opts: { llm: LlmTarget; slug: string; @@ -199,6 +221,7 @@ export async function generateHeroAssets(opts: { specMd: string; deadlineAt?: number; onStatus?: (message: string) => void; + onHero?: (event: HeroProgressEvent) => void; }): Promise { const apiKey = getMeshyApiKey(); if (!apiKey) { @@ -220,15 +243,57 @@ export async function generateHeroAssets(opts: { console.warn(`[game-assets] ${item.id} skipped: not enough time left`); break; } + + opts.onHero?.({ + id: item.id, + kind: item.kind, + stage: "preview", + status: "PENDING", + progress: 0, + }); + const sculpt = await generateTexturedGlb({ apiKey, prompt: item.prompt, poseMode: item.kind === "humanoid" ? "t-pose" : "", deadlineAt: opts.deadlineAt, onStatus: opts.onStatus, + onProgress: async (snap) => { + const event: HeroProgressEvent = { + id: item.id, + kind: item.kind, + stage: snap.stage, + status: snap.status, + progress: snap.progress, + }; + if (snap.thumbnailUrl) event.thumbnailUrl = snap.thumbnailUrl; + if (snap.glb) { + try { + event.modelUrl = await publishHeroGlb( + opts.slug, + item.id, + snap.glb, + `${snap.stage}-${snap.progress}` + ); + } catch (e) { + console.warn( + `[game-assets] ${item.id} live save failed:`, + e instanceof Error ? e.message : e + ); + } + } + opts.onHero?.(event); + }, }); if (!sculpt.ok) { console.warn(`[game-assets] ${item.id} sculpt failed: ${sculpt.error}`); + opts.onHero?.({ + id: item.id, + kind: item.kind, + stage: "failed", + status: "FAILED", + progress: 0, + }); continue; } @@ -239,6 +304,13 @@ export async function generateHeroAssets(opts: { let clips: string[] = []; if (item.kind === "humanoid") { opts.onStatus?.("Auto-rigging Quaternius kernel"); + opts.onHero?.({ + id: item.id, + kind: item.kind, + stage: "kernel", + status: "IN_PROGRESS", + progress: 90, + }); const rig = await autoRigToQuaterniusKernel(sculpt.glb); if (rig.ok) { bytes = rig.glb; @@ -254,7 +326,12 @@ export async function generateHeroAssets(opts: { const filename = `${item.id}.glb`; try { - await writeGameAssetFile(opts.slug, filename, bytes); + const modelUrl = await publishHeroGlb( + opts.slug, + item.id, + bytes, + rigged ? "kernel" : "ready" + ); assets.push({ id: item.id, filename, @@ -265,11 +342,28 @@ export async function generateHeroAssets(opts: { kernel, clips, }); + opts.onHero?.({ + id: item.id, + kind: item.kind, + stage: "ready", + status: "SUCCEEDED", + progress: 100, + modelUrl, + kernel: Boolean(kernel), + clips, + }); } catch (e) { console.warn( `[game-assets] ${item.id} save failed:`, e instanceof Error ? e.message : e ); + opts.onHero?.({ + id: item.id, + kind: item.kind, + stage: "failed", + status: "FAILED", + progress: 0, + }); } } diff --git a/lib/game-prompt-utils.ts b/lib/game-prompt-utils.ts index 6470cd3..6838b42 100644 --- a/lib/game-prompt-utils.ts +++ b/lib/game-prompt-utils.ts @@ -1,7 +1,6 @@ import { gameSpecPageUrl } from "@/lib/site-url"; -const GAME_SPEC_SUFFIX_RE = - /\n*Use this game spec:\s*(?:\[[^\]]*\]\([^)]+\)|https?:\/\/\S+)\s*$/i; +const GAME_SPEC_SUFFIX_RE = /(?:\n+Use this game spec:[^\n]*)+$/i; export function stripGameSpecLink(prompt: string): string { return prompt.replace(GAME_SPEC_SUFFIX_RE, "").trimEnd(); @@ -10,7 +9,5 @@ export function stripGameSpecLink(prompt: string): string { export function appendGameSpecLink(prompt: string, slug: string): string { const stripped = stripGameSpecLink(prompt); const link = gameSpecPageUrl(slug); - const suffix = `Use this game spec: ${link}`; - if (stripped.includes(suffix)) return stripped; - return `${stripped}\n\n${suffix}`; + return `${stripped}\n\nUse this game spec: ${link}`; } diff --git a/lib/game-reverse-engine.ts b/lib/game-reverse-engine.ts index f3c73a8..a87ba2b 100644 --- a/lib/game-reverse-engine.ts +++ b/lib/game-reverse-engine.ts @@ -18,6 +18,7 @@ import { generateHeroAssets, } from "@/lib/game-hero-assets"; import { readHeroAssetManifest } from "@/lib/game-asset-storage"; +import type { HeroProgressEvent } from "@/lib/meshy-progress"; export type GameReverseResult = | { @@ -98,9 +99,10 @@ export async function ensureGameReversed(opts: { slug: string; gameName: string; onStatus?: (message: string) => void; + onHero?: (event: HeroProgressEvent) => void; force?: boolean; }): Promise { - const { slug, gameName, onStatus, force } = opts; + const { slug, gameName, onStatus, onHero, force } = opts; if (!force) { const cached = await readGameReverse(slug); @@ -189,6 +191,7 @@ export async function ensureGameReversed(opts: { specMd: specResult.text, deadlineAt: Date.now() + meshyBudgetMs, onStatus, + onHero, }).catch((e) => { console.warn( `[reverse-game] hero assets failed:`, diff --git a/lib/meshy-client.ts b/lib/meshy-client.ts index 2149fd1..2739b11 100644 --- a/lib/meshy-client.ts +++ b/lib/meshy-client.ts @@ -1,4 +1,12 @@ +import { + classifyMeshySsePayload, + meshyTaskSnapshot, + shouldFallBackToPoll, + type MeshyTaskSnapshot, +} from "@/lib/meshy-progress"; + const MESHY_BASE = "https://api.meshy.ai/openapi"; +const POLL_MS = 3_000; export function getMeshyApiKey(): string | null { const key = process.env.MESHY_API_KEY?.trim(); @@ -10,6 +18,8 @@ type MeshyTask = { id?: string; status?: string; progress?: number; + thumbnail_url?: string; + alpha_thumbnail_url?: string; task_error?: { message?: string } | string | null; model_urls?: { glb?: string }; result?: { @@ -20,6 +30,13 @@ type MeshyTask = { }; }; +export type MeshySculptStage = "preview" | "refine"; + +export type MeshySculptProgress = MeshyTaskSnapshot & { + stage: MeshySculptStage; + glb?: Buffer; +}; + function authHeaders(apiKey: string): HeadersInit { return { Authorization: `Bearer ${apiKey}`, @@ -60,11 +77,15 @@ function remainingMs(deadlineAt?: number, fallback = 180_000): number { return Math.max(0, deadlineAt - Date.now()); } +function isTerminal(status: string): boolean { + return status === "SUCCEEDED" || status === "FAILED" || status === "CANCELED"; +} + async function pollTask(opts: { url: string; apiKey: string; timeoutMs: number; - onProgress?: (progress: number, status: string) => void; + onProgress?: (snapshot: MeshyTaskSnapshot) => void | Promise; }): Promise<{ ok: true; task: MeshyTask } | { ok: false; error: string }> { if (opts.timeoutMs <= 0) { return { ok: false, error: "Meshy task timed out" }; @@ -75,20 +96,180 @@ async function pollTask(opts: { headers: { Authorization: `Bearer ${opts.apiKey}` }, }); if (!got.ok) return got; - const status = got.data.status ?? "UNKNOWN"; - opts.onProgress?.(got.data.progress ?? 0, status); - if (status === "SUCCEEDED") return { ok: true, task: got.data }; - if (status === "FAILED" || status === "CANCELED") { + const snapshot = meshyTaskSnapshot(got.data); + await opts.onProgress?.(snapshot); + if (snapshot.status === "SUCCEEDED") return { ok: true, task: got.data }; + if (snapshot.status === "FAILED" || snapshot.status === "CANCELED") { return { ok: false, - error: taskErrorMessage(got.data) || `Meshy task ${status.toLowerCase()}`, + error: taskErrorMessage(got.data) || `Meshy task ${snapshot.status.toLowerCase()}`, }; } - await new Promise((r) => setTimeout(r, 8000)); + await new Promise((r) => setTimeout(r, POLL_MS)); } return { ok: false, error: "Meshy task timed out" }; } +function parseSseDataLine(line: string): unknown | null { + const trimmed = line.trim(); + if (!trimmed.startsWith("data:")) return null; + const json = trimmed.slice(5).trim(); + if (!json || json === "[DONE]") return null; + try { + return JSON.parse(json); + } catch { + return null; + } +} + +async function applyStreamPayload( + payload: unknown, + last: MeshyTask | null, + onProgress?: (snapshot: MeshyTaskSnapshot) => void | Promise +): Promise< + | { last: MeshyTask; terminal: false } + | { last: MeshyTask; terminal: true; ok: true } + | { last: MeshyTask; terminal: true; ok: false; taskFailed: true; error: string } + | { last: MeshyTask; terminal: true; ok: false; protocol: true; error: string } +> { + if (!payload || typeof payload !== "object") { + return { last: last ?? {}, terminal: false }; + } + const classified = classifyMeshySsePayload(payload); + if (classified.kind === "http-error") { + return { + last: last ?? {}, + terminal: true, + ok: false, + protocol: true, + error: classified.message, + }; + } + const merged = { ...last, ...(payload as MeshyTask) }; + await onProgress?.(meshyTaskSnapshot(merged)); + const status = meshyTaskSnapshot(merged).status; + if (!isTerminal(status)) return { last: merged, terminal: false }; + if (status === "SUCCEEDED") return { last: merged, terminal: true, ok: true }; + return { + last: merged, + terminal: true, + ok: false, + taskFailed: true, + error: taskErrorMessage(merged) || `Meshy task ${status.toLowerCase()}`, + }; +} + +type StreamWatchResult = + | { ok: true; task: MeshyTask } + | { ok: false; error: string; taskFailed?: boolean }; + +async function finishStreamEvent( + applied: Awaited>, + reader: ReadableStreamDefaultReader, + finishSucceeded: (task: MeshyTask) => Promise +): Promise { + try { + await reader.cancel(); + } catch { + // ignore + } + if (!applied.terminal) return null; + if (applied.ok) return finishSucceeded(applied.last); + if ("protocol" in applied && applied.protocol) return null; + return { ok: false, error: applied.error, taskFailed: true }; +} + +async function streamTextTo3dTask(opts: { + taskId: string; + apiKey: string; + timeoutMs: number; + onProgress?: (snapshot: MeshyTaskSnapshot) => void | Promise; +}): Promise { + if (opts.timeoutMs <= 0) return null; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), opts.timeoutMs); + try { + const res = await fetch(`${MESHY_BASE}/v2/text-to-3d/${opts.taskId}/stream`, { + headers: { + Authorization: `Bearer ${opts.apiKey}`, + Accept: "text/event-stream", + }, + signal: controller.signal, + }); + if (!res.ok || !res.body) return null; + + const reader = res.body.getReader(); + const dec = new TextDecoder(); + let buf = ""; + let last: MeshyTask | null = null; + + const finishSucceeded = async (task: MeshyTask) => { + const full = await meshyJson( + `${MESHY_BASE}/v2/text-to-3d/${opts.taskId}`, + { headers: { Authorization: `Bearer ${opts.apiKey}` } } + ); + if (full.ok) { + await opts.onProgress?.(meshyTaskSnapshot(full.data)); + return { ok: true as const, task: full.data }; + } + return { ok: true as const, task }; + }; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buf += dec.decode(value, { stream: true }); + const lines = buf.split("\n"); + buf = lines.pop() ?? ""; + for (const line of lines) { + const payload = parseSseDataLine(line); + if (payload == null) continue; + const applied = await applyStreamPayload(payload, last, opts.onProgress); + last = applied.last; + if (!applied.terminal) continue; + return finishStreamEvent(applied, reader, finishSucceeded); + } + } + + const tail = parseSseDataLine(buf); + if (tail != null) { + const applied = await applyStreamPayload(tail, last, opts.onProgress); + last = applied.last; + if (applied.terminal) { + return finishStreamEvent(applied, reader, finishSucceeded); + } + } + + if (last && meshyTaskSnapshot(last).status === "SUCCEEDED") { + return finishSucceeded(last); + } + return null; + } catch { + return null; + } finally { + clearTimeout(timer); + } +} + +async function watchTextTo3dTask(opts: { + taskId: string; + apiKey: string; + timeoutMs: number; + onProgress?: (snapshot: MeshyTaskSnapshot) => void | Promise; +}): Promise<{ ok: true; task: MeshyTask } | { ok: false; error: string }> { + const started = Date.now(); + const streamed = await streamTextTo3dTask(opts); + if (!shouldFallBackToPoll(streamed)) { + return streamed as { ok: true; task: MeshyTask } | { ok: false; error: string }; + } + return pollTask({ + url: `${MESHY_BASE}/v2/text-to-3d/${opts.taskId}`, + apiKey: opts.apiKey, + timeoutMs: Math.max(0, opts.timeoutMs - (Date.now() - started)), + onProgress: opts.onProgress, + }); +} + export async function downloadBinary(url: string): Promise { const res = await fetch(url); if (!res.ok) { @@ -97,6 +278,15 @@ export async function downloadBinary(url: string): Promise { return Buffer.from(await res.arrayBuffer()); } +async function emitSculptProgress( + onProgress: ((progress: MeshySculptProgress) => void | Promise) | undefined, + stage: MeshySculptStage, + snapshot: MeshyTaskSnapshot, + glb?: Buffer +): Promise { + await onProgress?.({ ...snapshot, stage, glb }); +} + export async function generateTexturedGlb(opts: { apiKey: string; prompt: string; @@ -104,11 +294,25 @@ export async function generateTexturedGlb(opts: { timeoutMs?: number; deadlineAt?: number; onStatus?: (message: string) => void; + onProgress?: (progress: MeshySculptProgress) => void | Promise; }): Promise< | { ok: true; glb: Buffer; refineTaskId: string | null } | { ok: false; error: string } > { + const report = async ( + stage: MeshySculptStage, + snapshot: MeshyTaskSnapshot, + glb?: Buffer + ) => { + const label = + stage === "preview" ? "Sculpting hero mesh" : "Painting hero textures"; + opts.onStatus?.(`${label} (${snapshot.progress}% ${snapshot.status})`); + await emitSculptProgress(opts.onProgress, stage, snapshot, glb); + }; + opts.onStatus?.("Sculpting hero mesh"); + await report("preview", { status: "PENDING", progress: 0 }); + const preview = await meshyJson( `${MESHY_BASE}/v2/text-to-3d`, { @@ -121,6 +325,7 @@ export async function generateTexturedGlb(opts: { should_remesh: true, target_polycount: 30000, target_formats: ["glb"], + alpha_thumbnail: true, ai_model: "latest", }), } @@ -129,17 +334,26 @@ export async function generateTexturedGlb(opts: { return { ok: false, error: preview.ok ? "Meshy preview missing id" : preview.error }; } - const previewDone = await pollTask({ - url: `${MESHY_BASE}/v2/text-to-3d/${preview.data.result}`, + const previewDone = await watchTextTo3dTask({ + taskId: preview.data.result, apiKey: opts.apiKey, timeoutMs: remainingMs(opts.deadlineAt, opts.timeoutMs ?? 180_000), - onProgress: (p, s) => opts.onStatus?.(`Sculpting hero mesh (${p}% ${s})`), + onProgress: (snapshot) => report("preview", snapshot), }); if (!previewDone.ok) return previewDone; - const previewGlbUrl = previewDone.task.model_urls?.glb; + const previewSnap = meshyTaskSnapshot(previewDone.task); + let previewGlb: Buffer | undefined; + if (previewSnap.glbUrl) { + previewGlb = await downloadBinary(previewSnap.glbUrl); + await report("preview", previewSnap, previewGlb); + } else { + await report("preview", previewSnap); + } opts.onStatus?.("Painting hero textures"); + await report("refine", { status: "PENDING", progress: 0 }); + const refine = await meshyJson( `${MESHY_BASE}/v2/text-to-3d`, { @@ -151,44 +365,40 @@ export async function generateTexturedGlb(opts: { enable_pbr: true, texture_resolution: "2k", target_formats: ["glb"], + alpha_thumbnail: true, ai_model: "latest", }), } ); if (!refine.ok || !refine.data.result) { - if (previewGlbUrl) { - return { - ok: true, - glb: await downloadBinary(previewGlbUrl), - refineTaskId: null, - }; + if (previewGlb) { + return { ok: true, glb: previewGlb, refineTaskId: null }; } return { ok: false, error: refine.ok ? "Meshy refine missing id" : refine.error }; } - const refineDone = await pollTask({ - url: `${MESHY_BASE}/v2/text-to-3d/${refine.data.result}`, + const refineDone = await watchTextTo3dTask({ + taskId: refine.data.result, apiKey: opts.apiKey, timeoutMs: remainingMs(opts.deadlineAt, opts.timeoutMs ?? 180_000), - onProgress: (p, s) => opts.onStatus?.(`Painting hero textures (${p}% ${s})`), + onProgress: (snapshot) => report("refine", snapshot), }); if (!refineDone.ok) { - if (previewGlbUrl) { - return { - ok: true, - glb: await downloadBinary(previewGlbUrl), - refineTaskId: null, - }; + if (previewGlb) { + return { ok: true, glb: previewGlb, refineTaskId: null }; } return refineDone; } - const glbUrl = refineDone.task.model_urls?.glb ?? previewGlbUrl; + const refineSnap = meshyTaskSnapshot(refineDone.task); + const glbUrl = refineSnap.glbUrl ?? previewSnap.glbUrl; if (!glbUrl) return { ok: false, error: "Meshy refine produced no GLB" }; + const glb = await downloadBinary(glbUrl); + await report("refine", refineSnap, glb); return { ok: true, - glb: await downloadBinary(glbUrl), + glb, refineTaskId: refine.data.result, }; } @@ -217,7 +427,8 @@ export async function rigHumanoidWalk(opts: { url: `${MESHY_BASE}/v1/rigging/${created.data.result}`, apiKey: opts.apiKey, timeoutMs: remainingMs(opts.deadlineAt, opts.timeoutMs ?? 180_000), - onProgress: (p, s) => opts.onStatus?.(`Rigging walk cycle (${p}% ${s})`), + onProgress: (snapshot) => + opts.onStatus?.(`Rigging walk cycle (${snapshot.progress}% ${snapshot.status})`), }); if (!done.ok) return done; diff --git a/lib/meshy-progress.ts b/lib/meshy-progress.ts new file mode 100644 index 0000000..e11a3da --- /dev/null +++ b/lib/meshy-progress.ts @@ -0,0 +1,161 @@ +export type MeshyTaskSnapshot = { + status: string; + progress: number; + thumbnailUrl?: string; + glbUrl?: string; +}; + +export type HeroProgressStage = + | "preview" + | "refine" + | "kernel" + | "ready" + | "failed"; + +export type HeroProgressEvent = { + id: string; + kind: "humanoid" | "vehicle" | "prop"; + stage: HeroProgressStage; + status: string; + progress: number; + thumbnailUrl?: string; + modelUrl?: string; + kernel?: boolean; + clips?: string[]; +}; + +export function httpsUrl(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + const trimmed = value.trim(); + if (!/^https:\/\//i.test(trimmed)) return undefined; + return trimmed; +} + +/** Map a Meshy task payload to UI fields. Never invent URLs or treat in-flight GLBs as ready. */ +export function meshyTaskSnapshot(task: { + status?: string; + progress?: number; + thumbnail_url?: unknown; + alpha_thumbnail_url?: unknown; + model_urls?: { glb?: unknown } | null; +}): MeshyTaskSnapshot { + const status = (task.status ?? "UNKNOWN").toUpperCase(); + const progress = + typeof task.progress === "number" && Number.isFinite(task.progress) + ? Math.max(0, Math.min(100, Math.round(task.progress))) + : 0; + const snapshot: MeshyTaskSnapshot = { status, progress }; + const thumb = + httpsUrl(task.alpha_thumbnail_url) ?? httpsUrl(task.thumbnail_url); + if (thumb) snapshot.thumbnailUrl = thumb; + if (status === "SUCCEEDED") { + const glb = httpsUrl(task.model_urls?.glb); + if (glb) snapshot.glbUrl = glb; + } + return snapshot; +} + +export function parseHeroProgressEvent( + value: unknown +): HeroProgressEvent | null { + if (!value || typeof value !== "object") return null; + const raw = value as Partial; + if (typeof raw.id !== "string" || !raw.id) return null; + if (raw.kind !== "humanoid" && raw.kind !== "vehicle" && raw.kind !== "prop") { + return null; + } + if ( + raw.stage !== "preview" && + raw.stage !== "refine" && + raw.stage !== "kernel" && + raw.stage !== "ready" && + raw.stage !== "failed" + ) { + return null; + } + if (typeof raw.status !== "string") return null; + if (typeof raw.progress !== "number" || !Number.isFinite(raw.progress)) { + return null; + } + const event: HeroProgressEvent = { + id: raw.id, + kind: raw.kind, + stage: raw.stage, + status: raw.status, + progress: Math.max(0, Math.min(100, Math.round(raw.progress))), + }; + const thumb = httpsUrl(raw.thumbnailUrl); + if (thumb) event.thumbnailUrl = thumb; + if (typeof raw.modelUrl === "string" && raw.modelUrl.startsWith("/")) { + event.modelUrl = raw.modelUrl; + } + if (raw.kernel) event.kernel = true; + if (Array.isArray(raw.clips)) { + event.clips = raw.clips.filter( + (name): name is string => typeof name === "string" + ); + } + return event; +} + +export type MeshySseClassification = + | { kind: "task" } + | { kind: "http-error"; statusCode: number; message: string } + | { kind: "ignore" }; + +/** Meshy message events can include status_code without a task status. That is not a task failure. */ +export function classifyMeshySsePayload(payload: unknown): MeshySseClassification { + if (!payload || typeof payload !== "object") return { kind: "ignore" }; + const raw = payload as Record; + const status = typeof raw.status === "string" ? raw.status.trim() : ""; + const statusCode = + typeof raw.status_code === "number" && Number.isFinite(raw.status_code) + ? raw.status_code + : undefined; + if (status) return { kind: "task" }; + if (statusCode !== undefined && statusCode >= 400) { + return { + kind: "http-error", + statusCode, + message: + typeof raw.message === "string" && raw.message.trim() + ? raw.message + : "Meshy stream error", + }; + } + return { kind: "ignore" }; +} + +export type MeshyStreamResult = + | { ok: true } + | { ok: false; taskFailed?: boolean }; + +/** + * Stream is optional. Poll unless the stream finished the Meshy task + * (SUCCEEDED) or the task itself FAILED/CANCELED. + */ +export function shouldFallBackToPoll( + streamed: MeshyStreamResult | null | undefined +): boolean { + if (streamed == null) return true; + if (streamed.ok) return false; + return streamed.taskFailed !== true; +} + +export function mergeHeroProgress( + prev: HeroProgressEvent[], + next: HeroProgressEvent +): HeroProgressEvent[] { + const index = prev.findIndex((item) => item.id === next.id); + if (index < 0) return [...prev, next]; + const current = prev[index]; + const merged: HeroProgressEvent = { + ...current, + ...next, + thumbnailUrl: next.thumbnailUrl || current.thumbnailUrl, + modelUrl: next.modelUrl || current.modelUrl, + clips: next.clips ?? current.clips, + kernel: next.kernel ?? current.kernel, + }; + return prev.map((item, i) => (i === index ? merged : item)); +} diff --git a/lib/quaternius-kernel.ts b/lib/quaternius-kernel.ts index 914938c..a6efe3a 100644 --- a/lib/quaternius-kernel.ts +++ b/lib/quaternius-kernel.ts @@ -158,6 +158,22 @@ export const GAME_ACTION_TO_CLIP: Record { + const mangled = `Build Delta Force. + +Use this game spec: http://localhost:3000/specs/delta-forc (http://localhost:3000/specs/delta-force)e`; + const out = appendGameSpecLink(mangled, "delta-force"); + const lines = out.split("\n").filter((line) => line.startsWith("Use this game spec:")); + assert.equal(lines.length, 1); + assert.match(lines[0] ?? "", /^Use this game spec: https?:\/\/\S+$/); + assert.match(lines[0] ?? "", /\/specs\/delta-force$/); + assert.doesNotMatch(lines[0] ?? "", /\(/); + assert.doesNotMatch(out, /delta-forc \(/); +}); + +test("empty hero assets still append kernel and download instructions", () => { + const out = appendHeroAssetInstructions("Make a vertical slice.", "delta-force", []); + assert.match(out, /Download these 3D models into public\/models\//); + assert.match(out, /Idle_Loop/); + assert.match(out, /Walk_Loop/); + assert.match(out, /UAL1_Standard/); + assert.match(out, /AnimationMixer/); +}); + +test("generated hero assets keep real GLB URLs plus kernel clips", () => { + const assets: StoredHeroAsset[] = [ + { + id: "player", + filename: "player.glb", + kind: "humanoid", + rigged: true, + hasWalk: true, + prompt: "soldier", + kernel: "quaternius-ual1", + clips: ["Idle_Loop", "Walk_Loop"], + }, + ]; + const out = appendHeroAssetInstructions("Make a vertical slice.", "delta-force", assets); + assert.match(out, /player\.glb/); + assert.match(out, /\/api\/game-assets\/delta-force\/player\.glb/); + assert.match(out, /Idle_Loop/); + assert.match(out, /Walk_Loop/); +}); diff --git a/scripts/test-meshy-progress.ts b/scripts/test-meshy-progress.ts new file mode 100644 index 0000000..0a4b54f --- /dev/null +++ b/scripts/test-meshy-progress.ts @@ -0,0 +1,129 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + httpsUrl, + mergeHeroProgress, + meshyTaskSnapshot, + parseHeroProgressEvent, + classifyMeshySsePayload, + shouldFallBackToPoll, + type HeroProgressEvent, +} from "../lib/meshy-progress"; + +test("meshyTaskSnapshot streams status and percent without inventing URLs", () => { + assert.deepEqual( + meshyTaskSnapshot({ status: "IN_PROGRESS", progress: 41 }), + { status: "IN_PROGRESS", progress: 41 } + ); + assert.equal( + meshyTaskSnapshot({ + status: "IN_PROGRESS", + progress: 80, + model_urls: { glb: "https://assets.meshy.ai/output/model.glb" }, + }).glbUrl, + undefined + ); +}); + +test("meshyTaskSnapshot only exposes result URLs after SUCCEEDED", () => { + const snap = meshyTaskSnapshot({ + status: "SUCCEEDED", + progress: 100, + thumbnail_url: "https://assets.meshy.ai/output/preview.png", + alpha_thumbnail_url: "https://assets.meshy.ai/output/preview-alpha.png", + model_urls: { glb: "https://assets.meshy.ai/output/model.glb" }, + }); + assert.equal(snap.thumbnailUrl, "https://assets.meshy.ai/output/preview-alpha.png"); + assert.equal(snap.glbUrl, "https://assets.meshy.ai/output/model.glb"); +}); + +test("meshyTaskSnapshot ignores non-https thumbnail and glb URLs", () => { + const snap = meshyTaskSnapshot({ + status: "SUCCEEDED", + progress: 100, + thumbnail_url: "javascript:alert(1)", + model_urls: { glb: "/relative/model.glb" }, + }); + assert.equal(snap.thumbnailUrl, undefined); + assert.equal(snap.glbUrl, undefined); + assert.equal(httpsUrl("http://insecure.example/preview.png"), undefined); +}); + +test("parseHeroProgressEvent accepts same-origin model URLs only", () => { + const parsed = parseHeroProgressEvent({ + id: "player", + kind: "humanoid", + stage: "preview", + status: "SUCCEEDED", + progress: 100, + thumbnailUrl: "https://assets.meshy.ai/preview.png", + modelUrl: "/api/game-assets/gta/player.glb?v=preview-100", + }); + assert.ok(parsed); + assert.equal(parsed?.modelUrl, "/api/game-assets/gta/player.glb?v=preview-100"); + assert.equal( + parseHeroProgressEvent({ + id: "player", + kind: "humanoid", + stage: "preview", + status: "SUCCEEDED", + progress: 100, + modelUrl: "https://evil.example/model.glb", + })?.modelUrl, + undefined + ); +}); + +test("mergeHeroProgress keeps earlier preview URLs when a later event omits them", () => { + const first: HeroProgressEvent = { + id: "player", + kind: "humanoid", + stage: "preview", + status: "SUCCEEDED", + progress: 100, + thumbnailUrl: "https://assets.meshy.ai/preview.png", + modelUrl: "/api/game-assets/gta/player.glb?v=preview-100", + }; + const next: HeroProgressEvent = { + id: "player", + kind: "humanoid", + stage: "refine", + status: "IN_PROGRESS", + progress: 12, + }; + const merged = mergeHeroProgress([first], next); + assert.equal(merged.length, 1); + assert.equal(merged[0].stage, "refine"); + assert.equal(merged[0].progress, 12); + assert.equal(merged[0].thumbnailUrl, first.thumbnailUrl); + assert.equal(merged[0].modelUrl, first.modelUrl); +}); + +test("SSE status_code without task status is not a task failure", () => { + assert.equal( + classifyMeshySsePayload({ status_code: 200, progress: 0 }).kind, + "ignore" + ); + assert.equal( + classifyMeshySsePayload({ id: "abc", progress: 12 }).kind, + "ignore" + ); + const err = classifyMeshySsePayload({ + status_code: 404, + message: "Task not found", + }); + assert.equal(err.kind, "http-error"); + assert.equal( + classifyMeshySsePayload({ status: "IN_PROGRESS", progress: 40, status_code: 200 }) + .kind, + "task" + ); +}); + +test("stream error falls back to poll unless the Meshy task FAILED or CANCELED", () => { + assert.equal(shouldFallBackToPoll(null), true); + assert.equal(shouldFallBackToPoll({ ok: false }), true); + assert.equal(shouldFallBackToPoll({ ok: false, taskFailed: false }), true); + assert.equal(shouldFallBackToPoll({ ok: true }), false); + assert.equal(shouldFallBackToPoll({ ok: false, taskFailed: true }), false); +}); diff --git a/scripts/test-quaternius-kernel.ts b/scripts/test-quaternius-kernel.ts index a1fb81c..8d02622 100644 --- a/scripts/test-quaternius-kernel.ts +++ b/scripts/test-quaternius-kernel.ts @@ -26,6 +26,17 @@ test("kernel clip and joint tables match the vendored GLB contract", () => { "Jump_Land", ]); assert.equal(GAME_ACTION_TO_CLIP.idle, "Idle_Loop"); + assert.equal(GAME_ACTION_TO_CLIP.drive, "Driving_Loop"); + assert.equal(GAME_ACTION_TO_CLIP.pass, "Interact"); + assert.equal(GAME_ACTION_TO_CLIP.shot, "Interact"); + assert.equal(GAME_ACTION_TO_CLIP.keeperDive, "Roll"); + assert.equal(GAME_ACTION_TO_CLIP.crouch, "Crouch_Idle_Loop"); + assert.deepEqual(resolveGameClip("header"), ["Jump_Start", "Jump_Loop", "Jump_Land"]); + assert.equal(GAME_ACTION_TO_CLIP.celebration, "Dance_Loop"); + assert.equal(GAME_ACTION_TO_CLIP.enterSeat, "Sitting_Enter"); + assert.equal(GAME_ACTION_TO_CLIP.exitSeat, "Sitting_Exit"); + assert.notEqual(GAME_ACTION_TO_CLIP.shot, "Punch_Jab"); + assert.notEqual(GAME_ACTION_TO_CLIP.shot, "Sword_Attack"); assert.equal(isDeformJoint("root"), false); assert.equal(isDeformJoint("index_04_leaf_l"), false); assert.equal(isDeformJoint("upperarm_l"), true);