From 1feb524b7bb43a0e34d46347d4edf1b544dd97d5 Mon Sep 17 00:00:00 2001 From: willhong Date: Fri, 18 Sep 2026 09:27:27 +0900 Subject: [PATCH] fix(threads): title threads that open with a skill invocation The five-word gate in shouldGenerateThreadTitle counted the `/skill-name` token as an ordinary word, so a first message that invokes a skill almost always fell under the threshold and inference never ran. The thread kept a null title and the sidebar fell back to the raw prompt text. Exempt prompts carrying a command mention from the length gate, strip the command ranges out of the text handed to the model, and name the invoked commands in the metadata template so the title describes the work rather than the tool. --- .../src/services/threads/title-generation.ts | 72 ++++++++++++++++++- apps/server/test/helpers/prompt-input.ts | 25 +++++++ .../threads/generated-thread-titles.test.ts | 44 +++++++++++- .../test/threads/title-generation.test.ts | 51 +++++++++++++ .../src/templates/generate-thread-metadata.md | 5 ++ 5 files changed, 193 insertions(+), 4 deletions(-) diff --git a/apps/server/src/services/threads/title-generation.ts b/apps/server/src/services/threads/title-generation.ts index ed6e673fee1..f0fc4610717 100644 --- a/apps/server/src/services/threads/title-generation.ts +++ b/apps/server/src/services/threads/title-generation.ts @@ -1,6 +1,10 @@ import { renderTemplate } from "@bb/templates"; import { getThread, updateThread } from "@bb/db"; -import type { PromptInput } from "@bb/domain"; +import { + removeCommandMentionsFromPromptInput, + type PromptInput, + type PromptMentionCommandTrigger, +} from "@bb/domain"; import type { AppDeps, LoggedWorkSessionDeps } from "../../types.js"; import { Type } from "@earendil-works/pi-ai"; import { @@ -10,6 +14,7 @@ import { } from "../ai/inference.js"; const MIN_TITLE_GENERATION_WORDS = 5; +const MAX_PROMPT_TEXT_LENGTH = 80; const MAX_GENERATED_TITLE_WORDS = 5; const MAX_BRANCH_SLUG_LENGTH = 48; @@ -55,12 +60,64 @@ function cleanPromptText(input: PromptInput[]): string { .trim(); } +function clampPromptText(text: string): string { + return text.length <= MAX_PROMPT_TEXT_LENGTH + ? text + : `${text.slice(0, MAX_PROMPT_TEXT_LENGTH - 3)}...`; +} + export function deriveTitleFallback(input: PromptInput[]): string | null { const text = cleanPromptText(input); if (text.length === 0) { return null; } - return text.length <= 80 ? text : `${text.slice(0, 77)}...`; + return clampPromptText(text); +} + +interface InvokedPromptCommand { + name: string; + trigger: PromptMentionCommandTrigger; +} + +export function collectInvokedPromptCommands( + input: PromptInput[], +): InvokedPromptCommand[] { + const seen = new Set(); + return input.flatMap((part) => + part.type === "text" + ? part.mentions.flatMap((mention) => { + if (mention.resource.kind !== "command") { + return []; + } + const { name, trigger } = mention.resource; + const key = `${trigger}${name}`; + if (seen.has(key)) { + return []; + } + seen.add(key); + return [{ name, trigger }]; + }) + : [], + ); +} + +function promptTextWithoutCommands( + input: PromptInput[], + commands: InvokedPromptCommand[], +): string { + return cleanPromptText( + commands.reduce( + (remaining, command) => + removeCommandMentionsFromPromptInput(remaining, command), + input, + ), + ); +} + +function formatInvokedCommands(commands: InvokedPromptCommand[]): string { + return commands + .map((command) => `${command.trigger}${command.name}`) + .join(", "); } export function shouldGenerateThreadTitle(input: PromptInput[]): boolean { @@ -69,6 +126,10 @@ export function shouldGenerateThreadTitle(input: PromptInput[]): boolean { return false; } + if (collectInvokedPromptCommands(input).length > 0) { + return true; + } + return text.split(/\s+/u).length >= MIN_TITLE_GENERATION_WORDS; } @@ -137,8 +198,13 @@ export async function generateThreadMetadataWithOutcome( return complete(null, "too-short"); } + const commands = collectInvokedPromptCommands(args.input); + const body = promptTextWithoutCommands(args.input, commands); const prompt = renderTemplate("generateThreadMetadata", { - cleanedPrompt: fallback, + cleanedPrompt: body.length > 0 ? clampPromptText(body) : fallback, + ...(commands.length > 0 + ? { invokedCommands: formatInvokedCommands(commands) } + : {}), }); const maxAttempts = Math.max(1, args.timeoutMaxAttempts ?? 1); diff --git a/apps/server/test/helpers/prompt-input.ts b/apps/server/test/helpers/prompt-input.ts index 40969578bad..1d97a10ef46 100644 --- a/apps/server/test/helpers/prompt-input.ts +++ b/apps/server/test/helpers/prompt-input.ts @@ -7,3 +7,28 @@ function textPrompt(text: string): PromptInput { export function textInput(text: string): PromptInput[] { return [textPrompt(text)]; } + +export function skillInput(name: string, rest = ""): PromptInput[] { + const command = `/${name}`; + return [ + { + type: "text", + text: `${command}${rest}`, + mentions: [ + { + start: 0, + end: command.length, + resource: { + kind: "command", + trigger: "/", + name, + source: "skill", + origin: "user", + label: name, + argumentHint: null, + }, + }, + ], + }, + ]; +} diff --git a/apps/server/test/threads/generated-thread-titles.test.ts b/apps/server/test/threads/generated-thread-titles.test.ts index 227cc23afb9..067103288ed 100644 --- a/apps/server/test/threads/generated-thread-titles.test.ts +++ b/apps/server/test/threads/generated-thread-titles.test.ts @@ -16,7 +16,7 @@ import { waitForQueuedCommandAfter, } from "../helpers/commands.js"; import { readJson } from "../helpers/json.js"; -import { textInput } from "../helpers/prompt-input.js"; +import { skillInput, textInput } from "../helpers/prompt-input.js"; import { seedEnvironment, seedHostSession, @@ -924,6 +924,45 @@ describe("generated thread titles", () => { }); }); + it("titles a skill invocation from the task, not the command token", async () => { + mockThreadMetadata({ title: "Drop stale release branches" }); + await withTestHarness(async (harness) => { + await expect( + generateThreadMetadataWithOutcome(harness.deps, { + input: skillInput("sync-repo", " and drop the stale release branches"), + threadId: "thr_skill_metadata", + }), + ).resolves.toMatchObject({ + metadata: { title: "Drop stale release branches" }, + }); + const prompt = piAiMocks.complete.mock.calls[0]?.[1].messages[0].content; + expect(prompt).toContain("and drop the stale release branches"); + expect(prompt).toContain( + "The prompt invokes these commands or skills: /sync-repo.", + ); + expect(prompt).not.toContain("/sync-repo and drop"); + }); + }); + + it("titles a bare skill invocation from what the skill does", async () => { + mockThreadMetadata({ title: "Generate the weekly report" }); + await withTestHarness(async (harness) => { + await expect( + generateThreadMetadataWithOutcome(harness.deps, { + input: skillInput("weekly-report"), + threadId: "thr_bare_skill_metadata", + }), + ).resolves.toMatchObject({ + metadata: { title: "Generate the weekly report" }, + }); + expect( + piAiMocks.complete.mock.calls[0]?.[1].messages[0].content, + ).toContain( + "The prompt invokes these commands or skills: /weekly-report.", + ); + }); + }); + it("does not retry non-transient metadata inference failures", async () => { piAiMocks.getModel.mockReturnValue({ provider: "test" }); piAiMocks.complete.mockRejectedValue(new Error("metadata failed")); @@ -940,6 +979,9 @@ describe("generated thread titles", () => { reason: "failed", }); expect(piAiMocks.complete).toHaveBeenCalledTimes(1); + expect( + piAiMocks.complete.mock.calls[0]?.[1].messages[0].content, + ).not.toContain("The prompt invokes these commands or skills"); }); }); }); diff --git a/apps/server/test/threads/title-generation.test.ts b/apps/server/test/threads/title-generation.test.ts index a2b4a5e2df3..3c2d80be5a8 100644 --- a/apps/server/test/threads/title-generation.test.ts +++ b/apps/server/test/threads/title-generation.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import type { PromptInput } from "@bb/domain"; import { + collectInvokedPromptCommands, deriveTitleFallback, sanitizeGeneratedTitle, shouldGenerateThreadTitle, @@ -14,6 +15,29 @@ function textInput(text: string): PromptInput { }; } +function skillInput(name: string, rest = ""): PromptInput { + const command = `/${name}`; + return { + type: "text", + text: `${command}${rest}`, + mentions: [ + { + start: 0, + end: command.length, + resource: { + kind: "command", + trigger: "/", + name, + source: "skill", + origin: "user", + label: name, + argumentHint: null, + }, + }, + ], + }; +} + describe("thread title generation", () => { it("does not generate titles for inputs shorter than five words", () => { expect(shouldGenerateThreadTitle([textInput("fix")])).toBe(false); @@ -55,6 +79,33 @@ describe("thread title generation", () => { expect(sanitizeGeneratedTitle(" ")).toBeNull(); }); + it("generates titles for invoked skills regardless of prompt length", () => { + expect(shouldGenerateThreadTitle([skillInput("sync-repo")])).toBe(true); + expect( + shouldGenerateThreadTitle([skillInput("sync-repo", " then deploy")]), + ).toBe(true); + }); + + it("keeps the raw command text as the fallback for invoked skills", () => { + expect(deriveTitleFallback([skillInput("sync-repo", " then deploy")])).toBe( + "/sync-repo then deploy", + ); + }); + + it("collects each invoked command once, in prompt order", () => { + expect( + collectInvokedPromptCommands([ + skillInput("sync-repo"), + textInput("then"), + skillInput("review-diff"), + skillInput("sync-repo"), + ]), + ).toEqual([ + { name: "sync-repo", trigger: "/" }, + { name: "review-diff", trigger: "/" }, + ]); + }); + it("keeps fallback derivation independent from title generation eligibility", () => { const input = [textInput("fix bug")]; diff --git a/packages/templates/src/templates/generate-thread-metadata.md b/packages/templates/src/templates/generate-thread-metadata.md index c3c4872bcca..4b3d1d15ae0 100644 --- a/packages/templates/src/templates/generate-thread-metadata.md +++ b/packages/templates/src/templates/generate-thread-metadata.md @@ -6,6 +6,7 @@ intent: Generate stable, operator-friendly metadata for threads without adding e editingNotes: Callers use tool-call structured output; the model calls a `result` tool with the schema. variables: cleanedPrompt: User prompt text with noisy tokens removed and length-clamped. + invokedCommands?: Comma-separated slash commands or skills the prompt invokes, when it invokes any. --- You create concise titles for coding tasks. Call the `result` tool with: @@ -13,5 +14,9 @@ Call the `result` tool with: Consider the user's intent when titling to make it useful. For instance, if they detail specific tools to use to solve a problem, it is the problem that should be the title, not the tools that should be used. +{{#if invokedCommands}} +The prompt invokes these commands or skills: {{invokedCommands}}. They name how the work is carried out, so title the work they are applied to. When the prompt names nothing else, title what the invoked command itself does. + +{{/if}} Task: {{cleanedPrompt}}