From 2988903b91c0e83a006afb649b4c03157b9ecf25 Mon Sep 17 00:00:00 2001 From: Robbie Court Date: Thu, 10 Sep 2026 15:15:43 +0000 Subject: [PATCH 1/3] Answer in the language the question was asked in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in the harness knew what language a question was written in. The synthesiser answered in English unless the message said otherwise, so at the NeuroFly table (10 September) a Persian question typed in Latin letters got an English clarifying question back, a Hungarian question was answered in Hungarian only because it ended "válaszolj magyarul", and "can you reply in persian?" was answered "yes, I can". The Hungarian answer is the reason for the design. Every check behind an answer -- the absence gate, the count repair, the grounding audit, the term and count linkers -- reads English. The Hungarian prose said 327 driver lines above a table that said 392, and carried no links, because nothing could read the sentence the number sat in. So the English pipeline is unchanged and translation is the LAST step: - The planner returns the language the message was written in and any language it asks for; lib/language.mjs turns that into one decision per turn. An explicit request pins the conversation; a typed turn is answered in its own language; a clicked chip inherits the conversation's. Term names in another language are written as their English VFB name by the planner, with a translation rung in the resolver as the backstop. - The English answer is synthesised silently and every gate runs on it. lib/translateAnswer.mjs then renders the finished markdown in the user's language, streamed, and a deterministic check requires every link target, every number (in any Unicode digit script) and every VFB id to survive. One retry with the misses named, then the English answer with a note. An unverified translation never ships. - A bare "reply in X" re-renders the previous answer, chips included, rather than answering "yes". Chip labels are localised; the query behind each stays English. Message bodies get dir="auto". English turns are byte-identical. The battery gains six tasks across Persian (Latin script), Hungarian, Japanese, Chinese, German and the language switch, with script, stopword, link and context checks; all pass against the live gateway. Unit suite 1,373/1,373. --- app/api/chat/route.js | 199 +++++++++++++- app/page.js | 6 + lib/battery/conversation.mjs | 69 +++++ lib/conversationContext.mjs | 15 +- lib/language.mjs | 147 ++++++++++ lib/liveHarness.mjs | 14 +- lib/orchestrator.mjs | 108 +++++++- lib/planner.mjs | 63 ++++- lib/translateAnswer.mjs | 274 +++++++++++++++++++ scripts/run-task-battery.mjs | 4 +- tests/task-battery/tasks.json | 141 ++++++++++ tests/unit/language.test.mjs | 493 ++++++++++++++++++++++++++++++++++ 12 files changed, 1503 insertions(+), 30 deletions(-) create mode 100644 lib/language.mjs create mode 100644 lib/translateAnswer.mjs create mode 100644 tests/unit/language.test.mjs diff --git a/app/api/chat/route.js b/app/api/chat/route.js index 17de8ba..402beb1 100644 --- a/app/api/chat/route.js +++ b/app/api/chat/route.js @@ -41,6 +41,9 @@ import { pickSeedIndividuals, parseSimilarityHits, groupSimilarByClass } from '. import { datasetAsked, groupHitsByDataset, bestHitInDataset } from '../../../lib/datasetAxis.mjs' import { parseMarkdownLinks } from '../../../lib/markdownLinks.mjs' import { minimizeHistory } from '../../../lib/conversationContext.mjs' +import { isEnglish, languageName } from '../../../lib/language.mjs' +import { translateMarkdown, translateChipLabels, translationFallbackNote, linkTargets } from '../../../lib/translateAnswer.mjs' +import { roleRequestOptions } from '../../../lib/roleProfiles.mjs' import { findLeakedIds, stripLeakedIds, collectGroundedIds, collectGroundedNumbers, findUngroundedNumbers, repairMistranscribedCounts } from '../../../lib/grounding.mjs' import { curatedCountsForRegion, curatedNoteForRegion, curatedAnswerRules, curatedArticle } from '../../../lib/curatedNeuronCounts.mjs' import { renderNeuronCountEstimate } from '../../../lib/neuronCount.mjs' @@ -11204,12 +11207,16 @@ async function requestNoToolFallbackResponse({ // one call that could not be configured at all. Left at the gateway default, a // reasoning model streams its whole chain of thought into a channel this reader // ignores — 34 to 73 seconds of empty pane before the first visible character. -async function streamSynthCompletion({ messages, model, apiBaseUrl, apiKey, sendEvent, onResponseId, sourceQuotes, sampling }) { +async function streamSynthCompletion({ messages, model, apiBaseUrl, apiKey, sendEvent, onResponseId, sourceQuotes, sampling, silent = false, signal = undefined }) { // Fenced code blocks are held until their closing fence and released closed — // the synthesiser re-indents a copied configuration often enough to lose its // last brace, and a streamed answer has no afterwards in which to fix that. // Inert outside a fence, and outside blocks that came from the documentation. const fences = createFenceRepairer(sourceQuotes) + // `silent`: accumulate the completion without showing it. The English draft + // of a non-English turn is written for the gates and linkers, not the reader; + // what the reader sees streams later, from the translation pass. + const emitDelta = silent ? () => {} : (text) => sendEvent('delta', { text }) let res try { res = await fetch(`${apiBaseUrl}${CHAT_COMPLETIONS_ENDPOINT}`, { @@ -11220,7 +11227,8 @@ async function streamSynthCompletion({ messages, model, apiBaseUrl, apiKey, send messages, stream: true, ...(sampling && typeof sampling === 'object' ? sampling : {}) - }) + }), + ...(signal ? { signal } : {}) }) } catch { // Network failure reaching ELM — degrade rather than crash the request. @@ -11232,7 +11240,7 @@ async function streamSynthCompletion({ messages, model, apiBaseUrl, apiKey, send let content = '' try { content = JSON.parse(text)?.choices?.[0]?.message?.content || '' } catch { content = '' } if (content) content = fences.push(content) + fences.flush() - if (content) sendEvent('delta', { text: content }) + if (content) emitDelta(content) return content } @@ -11259,15 +11267,120 @@ async function streamSynthCompletion({ messages, model, apiBaseUrl, apiKey, send // `full` accumulates what was EMITTED, not what arrived, so the answer // returned for linking and sanitising is the same text the reader saw. const out = delta ? fences.push(delta) : '' - if (out) { full += out; sendEvent('delta', { text: out }) } + if (out) { full += out; emitDelta(out) } } catch { /* keep-alive or partial chunk */ } } } const tail = fences.flush() - if (tail) { full += tail; sendEvent('delta', { text: tail }) } + if (tail) { full += tail; emitDelta(tail) } return full } +/** + * Render finished English markdown in the user's language, streaming the + * translation to the client, and verify it (lib/translateAnswer.mjs). On a + * verification failure the client is told to drop what it saw and the English + * text is returned with a note — never an unverified translation. + * + * Runs on the extract profile: transcription at temperature 0, thinking off. + * + * @returns {Promise<{ text: string, ok: boolean, attempts: number, reason: string }>} + */ +async function renderInLanguage({ text, language, kind = 'answer', sendEvent, apiBaseUrl, apiKey, apiModel, signal }) { + if (isEnglish(language) || !String(text || '').trim()) return { text, ok: true, attempts: 0, reason: '' } + const servedModels = await ensureServedModels({ baseUrl: apiBaseUrl, apiKey }) + const opts = roleRequestOptions('extract', { fallback: apiModel, available: servedModels }) + sendEvent('status', { message: `Translating into ${languageName(language)}`, phase: 'llm' }) + // The run deadline is a bound on the HARNESS: a question that spent the whole + // ten minutes on its queries has still earned its translation, so only a + // client that has gone stops this. The translation carries its own bound — + // the extract profile's per-attempt timeout — so it cannot run unbounded + // either. + const gone = () => signal?.aborted && signal.reason?.reason !== 'deadline' + const call = ({ messages }) => { + if (gone()) throwIfAborted(signal, 'translate') + return streamSynthCompletion({ + messages, model: opts.model, apiBaseUrl, apiKey, sendEvent, sourceQuotes: [], sampling: opts.sampling, + signal: AbortSignal.timeout(opts.timeoutMs || 120000) + }) + } + const startedAt = Date.now() + const r = await translateMarkdown({ + text, language, kind, call, + onDiscard: () => sendEvent('draft_discarded', { reason: 'translation-retry' }) + }) + // One line per translation, so the rate of verified translations, retries + // and fall-backs is readable from the container log — the same place the + // grounding audit reports. `links` is what the check had to preserve. + try { + console.error(`[VFBchat] TRANSLATION | language=${language} | kind=${kind} | ok=${r.ok} | attempts=${r.attempts} | links=${linkTargets(text).length} | chars=${String(text).length} | ms=${Date.now() - startedAt}${r.ok ? '' : ` | reason=${safeText(r.reason)}`}`) + } catch { /* logging best-effort */ } + if (!r.ok) { + sendEvent('draft_discarded', { reason: 'translation-unverified' }) + return { text: `${text}\n\n${translationFallbackNote(language)}`, ok: false, attempts: r.attempts, reason: r.reason } + } + // The translation carries the same links the English did — the check proved + // it — but it is model output, so it meets the same allow-list on its way out. + const { sanitizedText } = sanitizeAssistantOutput(r.text, getOutboundAllowList()) + return { text: sanitizedText || r.text, ok: true, attempts: r.attempts, reason: '' } +} + +/** + * Follow-on chip labels in the user's language. The query behind each chip + * stays English (it is what runs on a click); only the text changes. + */ +async function localiseFollowOns(followOns, language, { apiBaseUrl, apiKey, apiModel, signal }) { + const chips = Array.isArray(followOns) ? followOns : [] + if (isEnglish(language) || !chips.length) return chips + // Same rule as the translation: the run deadline does not cancel this, a + // departed client does. The call is bounded by the extract profile's budget. + if (signal?.aborted && signal.reason?.reason !== 'deadline') return chips + const servedModels = await ensureServedModels({ baseUrl: apiBaseUrl, apiKey }) + const opts = roleRequestOptions('extract', { fallback: apiModel, available: servedModels }) + const labels = await translateChipLabels({ + labels: chips.map(c => c.label || ''), + language, + callStructured: ({ messages, schema, schemaName }) => callStructured({ + baseUrl: apiBaseUrl, apiKey, model: opts.model, messages, schema, schemaName, + temperature: opts.temperature, timeoutMs: opts.timeoutMs, budgetMs: opts.budgetMs, extraBody: opts.extraBody + }) + }) + return chips.map((c, i) => (labels[i] && labels[i] !== c.label ? { ...c, label: labels[i] } : c)) +} + +/** + * The follow-on chips of the previous turn, as the client sent them back in + * its history, for the turn that re-renders that answer in another language. + * The client is not trusted with them: only the fields a chip is made of are + * taken, and only in the shapes the harness itself would have produced — + * a query type that reaches a URL, an id that is an id, a URL on a host the + * outbound gate already allows. + */ +function previousFollowOnsFrom(rawMessages) { + const last = [...(Array.isArray(rawMessages) ? rawMessages : [])].reverse() + .find(m => m && m.role === 'assistant' && Array.isArray(m.followOns) && m.followOns.length) + if (!last) return [] + const out = [] + for (const c of last.followOns.slice(0, 12)) { + if (!c || typeof c !== 'object') continue + const label = typeof c.label === 'string' ? c.label.trim().slice(0, 200) : '' + if (!label) continue + if (c.kind === 'vfb') { + const url = typeof c.url === 'string' ? c.url.trim() : '' + if (!/^https:\/\/(?:www\.)?virtualflybrain\.org\/reports\/[A-Za-z0-9_]+$/.test(url)) continue + out.push({ kind: 'vfb', label, url, title: typeof c.title === 'string' ? c.title.slice(0, 200) : undefined }) + continue + } + const query = typeof c.query === 'string' ? c.query.trim().slice(0, 300) : '' + const id = typeof c.id === 'string' ? c.id.trim() : '' + const queryType = typeof c.query_type === 'string' ? c.query_type.trim() : '' + if (!query) continue + const addressed = /^(?:FBbt|FBgn|FBal|FBti|FBtp|FBco|FBlc|FBrf|VFBexp|VFB)_[0-9a-zA-Z]+$/.test(id) && /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(queryType) + out.push({ kind: 'ask', label, query, ...(addressed ? { id, query_type: queryType } : {}), title: typeof c.title === 'string' ? c.title.slice(0, 200) : undefined }) + } + return out +} + // Attach harvested VFB thumbnails as image objects (fallback richness when the // synth model omitted them). Shape matches extractImagesFromResponseText. function mergeThumbnailImages(images = [], thumbnails = []) { @@ -11298,7 +11411,7 @@ function buildHarnessToolCatalogue() { return TOOL_DEFINITIONS.map(tool => ({ name: tool.name, purpose: tool.description || '', parameters: tool.parameters })) } -async function runRoleHarnessForRequest({ priorMessages, sendEvent, apiBaseUrl, apiKey, apiModel, userMessage, scene, context, focus, signal }) { +async function runRoleHarnessForRequest({ priorMessages, lastAssistantText = '', previousFollowOns = [], sendEvent, apiBaseUrl, apiKey, apiModel, userMessage, scene, context, focus, signal }) { const mcpClients = new Map() const dataResourceStore = createDataResourceStore() const toolState = { @@ -11372,8 +11485,8 @@ async function runRoleHarnessForRequest({ priorMessages, sendEvent, apiBaseUrl, if (obj && typeof obj === 'object' && obj.endpoint && Array.isArray(obj.top_partners)) return [] return buildConnectivityGraphs(obj).map(normalizeGraphSpec).filter(Boolean) }, - streamText: ({ messages, model, sourceQuotes, sampling }) => streamSynthCompletion({ - messages, model, apiBaseUrl, apiKey, sendEvent, sourceQuotes, sampling, + streamText: ({ messages, model, sourceQuotes, sampling, silent }) => streamSynthCompletion({ + messages, model, apiBaseUrl, apiKey, sendEvent, sourceQuotes, sampling, silent, onResponseId: (id) => { if (!responseId) responseId = id } }), onStatus: (status) => sendEvent('status', status), @@ -11392,6 +11505,42 @@ async function runRoleHarnessForRequest({ priorMessages, sendEvent, apiBaseUrl, console.log(`[VFBchat] PLAN | tier=${complexity.tier} agreement=${live.plannerAgreement ?? 'n/a'} votes=${live.plannerVotesUsed ?? 0} escalated=${Boolean(live.plannerEscalated)}`) } catch { /* logging best-effort */ } + // The language this turn is answered in (lib/language.mjs). English turns + // are untouched by everything below; a non-English turn had its synthesis + // buffered silently and is rendered — streamed — from here. + const language = live.language || 'en' + const languageDeps = { apiBaseUrl, apiKey, apiModel, signal } + + if (live.languageSwitch) { + // "Can you reply in Persian?" and nothing else. The context now carries + // the pinned language; the answer is the previous turn's answer in it — + // that is what was asked for, and "yes, I can" is not it. A conversation + // with no previous answer gets a short acknowledgement in the language + // instead, so the next question is asked knowing it will be understood. + const previous = String(lastAssistantText || '').trim() + const source = previous || `Yes — I will answer in ${languageName(language)} from now on. Ask me anything about Drosophila neuroanatomy, connectomes, gene expression or genetic tools in Virtual Fly Brain.` + const rendered = await renderInLanguage({ text: source, language, kind: 'answer', sendEvent, ...languageDeps }) + // The same answer keeps the same next steps: the previous turn's chips, + // labelled in the new language, so the conversation carries on from + // where it was rather than from a dead end. + const followOns = previous ? await localiseFollowOns(previousFollowOns, language, languageDeps) : [] + return { + ok: true, + responseText: rendered.text, + images: [], + graphs: [], + tables: [], + followOns, + sources: [], + terms: [], + context: live.context, + toolUsage: live.toolUsage, + toolRounds: live.toolRounds, + responseId, + blockedResponseDomains: [] + } + } + if (live.clarify) { // Through the same gates as every other exit. This branch used to return // `live.answer` raw and hard-code blockedResponseDomains: [] — so an @@ -11405,9 +11554,15 @@ async function runRoleHarnessForRequest({ priorMessages, sendEvent, apiBaseUrl, const { cleanedText } = stripLeakedToolCallJson(clarifyText) const { sanitizedText, blockedDomains } = sanitizeAssistantOutput(cleanedText, getOutboundAllowList()) const safeClarify = stripHarnessFraming(sanitizeInternalToolMentions(sanitizedText)) + const clarifyEnglish = stripLeakedIds(safeClarify, collectGroundedIds(userMessage, live.ledger)) + // A clarifying question is written by the planner in English (it is the + // one planner field that reaches the reader), so it is rendered like an + // answer. The Finglish question that got "Do you want to know how many + // split-GAL4 driver lines…?" back in English is the case. + const clarifyRendered = await renderInLanguage({ text: clarifyEnglish, language, kind: 'clarification', sendEvent, ...languageDeps }) return { ok: true, - responseText: stripLeakedIds(safeClarify, collectGroundedIds(userMessage, live.ledger)), + responseText: clarifyRendered.text, images: [], graphs: [], tables: [], @@ -11598,12 +11753,23 @@ async function runRoleHarnessForRequest({ priorMessages, sendEvent, apiBaseUrl, // looking at comes before the same entity aligned to a different template. const preferredTemplate = requestedTemplateFromScene(scene) const images = orderImagesByTemplate(rawImages, preferredTemplate).slice(0, 8) + // Last of all, and only on a non-English turn: the finished markdown — + // prose, notes, expression matrix, appendices, every link the linkers + // wrote — is rendered in the user's language and checked against itself + // (lib/translateAnswer.mjs). The chips' visible text is localised too; the + // query behind each chip stays English. Everything above this line is + // byte-identical to an English turn. + if (!isEnglish(language)) { + const rendered = await renderInLanguage({ text: built.responseText, language, kind: 'answer', sendEvent, ...languageDeps }) + built.responseText = rendered.text + } + const followOns = await localiseFollowOns(live.followOns || [], language, languageDeps) return { ...built, images, tables, responseId, // One list, whatever path the address arrived by, so the governance // counter and the debug payload do not have to know the difference. blockedResponseDomains: [...new Set([...(built.blockedResponseDomains || []), ...structuredBlocked])].sort(), - followOns: live.followOns || [], sources: live.sources || [], terms: live.terms || [], + followOns, sources: live.sources || [], terms: live.terms || [], // The ids this turn resolved and the catalogue queries it ran, as runnable // VFBquery Python. Carried on every turn so a client can offer it as a // button; the prose only carries it when the user asked for it. @@ -11799,10 +11965,15 @@ export async function POST(request) { // summarising prose that is already lean). The ids the stripped links used to // carry are not lost: they travel structurally in `context` now, which is // what makes cutting the prose safe. - const minimized = minimizeHistory( - messages.slice(0, -1).map(normalizeChatMessage).filter(Boolean) - ) + const normalizedPrior = messages.slice(0, -1).map(normalizeChatMessage).filter(Boolean) + const minimized = minimizeHistory(normalizedPrior) const rawPriorMessages = minimized.messages + // The previous answer AS SHOWN — links, tables and all — for the turn that + // asks for it again in another language. The minimised history below has + // had exactly that apparatus stripped, which is right for the planner and + // wrong for a re-rendering. + const lastAssistantText = [...normalizedPrior].reverse().find(m => m?.role === 'assistant' && typeof m.content === 'string')?.content || '' + const previousFollowOns = previousFollowOnsFrom(messages.slice(0, -1)) if (minimized.dropped) { try { console.log(`[VFBchat] HISTORY | kept=${rawPriorMessages.length} dropped=${minimized.dropped} chars=${minimized.chars}`) } catch { /* best-effort */ } } @@ -11824,6 +11995,8 @@ export async function POST(request) { try { const result = await runRoleHarnessForRequest({ priorMessages, + lastAssistantText, + previousFollowOns, signal, sendEvent, apiBaseUrl, diff --git a/app/page.js b/app/page.js index 11073c3..09d0d90 100644 --- a/app/page.js +++ b/app/page.js @@ -938,8 +938,13 @@ const ChatMessage = memo(function ChatMessage({ }}> {getDisplayName(msg.role)} + {/* dir="auto": a Persian, Arabic or Hebrew answer aligns to the right and + reads in its own direction; English keeps the default. Decided per + message from its first strong character, so a mixed conversation + renders each bubble the way its language reads. */}
{/* remark-gfm: without it react-markdown is CommonMark only, and the @@ -1952,6 +1957,7 @@ Feel free to ask about neural circuits, gene expression, connectome data, or any setInput(e.target.value)} onKeyDown={e => e.key === 'Enter' && handleSend()} diff --git a/lib/battery/conversation.mjs b/lib/battery/conversation.mjs index 59d5f78..ba33460 100644 --- a/lib/battery/conversation.mjs +++ b/lib/battery/conversation.mjs @@ -99,6 +99,43 @@ export function selectAskChip(followOns, index) { return chips[index] || null } +/** The prose of an answer: links, code and URLs out, so scripts and words are counted on what a reader reads. */ +function proseOf(answer = '') { + return String(answer) + .replace(/```[\s\S]*?```/g, ' ') + .replace(/\]\([^)]*\)/g, ']') + .replace(/https?:\/\/\S+/g, ' ') +} + +/** + * Share of the answer's letters that belong to a Unicode script ("Arabic", + * "Han", "Cyrillic", "Latin", …). Letters only — digits, punctuation and the + * English entity names inside link text all sit outside the count of what the + * translation was supposed to write, so the labels VFB keeps in English do not + * count against a Persian answer. + */ +export function scriptShare(answer = '', script = 'Latin') { + const prose = proseOf(answer).replace(/\[[^\]]*\]/g, ' ') + const letters = prose.match(/\p{L}/gu) || [] + if (!letters.length) return 0 + let re + try { re = new RegExp(`\\p{Script=${script}}`, 'u') } catch { return 0 } + const hits = letters.filter(ch => re.test(ch)).length + return hits / letters.length +} + +// Function words that are English and not also a word of some other Latin- +// alphabet language: "a", "an", "in", "is", "as", "to" are all words in +// Hungarian, German, Dutch, Spanish or Italian and are left out. +const ENGLISH_STOPWORDS = new Set(['the', 'of', 'and', 'are', 'with', 'for', 'that', 'this', 'by', 'from', 'holds', 'does', 'not', 'which', 'these', 'those', 'has', 'have', 'its', 'be', 'was', 'were', 'currently', 'records']) + +/** Share of the answer's words that are English function words — near zero in any other language. */ +export function englishStopwordRatio(answer = '') { + const all = proseOf(answer).replace(/\[[^\]]*\]/g, ' ').toLowerCase().match(/\p{L}+/gu) || [] + if (!all.length) return 0 + return all.filter(w => ENGLISH_STOPWORDS.has(w)).length / all.length +} + /** The address a clicked chip posts back, or null if it has none to post. */ export function chipFocus(chip) { if (!chip || !chip.id || !chip.query_type) return null @@ -199,6 +236,38 @@ export function checkTurn(expect, observed = {}) { } } + // The language checks. Deterministic by design: a script test where the + // language has its own script, an English-stopword ratio where it shares the + // Latin alphabet, and the links counted, because a translation that lost its + // links is the failure the whole translate-last design exists to prevent. + if (expect.answer_script) { + const share = scriptShare(answer, expect.answer_script) + if (share < 0.5) { + problems.push(`answer is not written in the ${expect.answer_script} script (${Math.round(share * 100)}% of its letters are)`) + } + } + if (expect.answer_not_english) { + const ratio = englishStopwordRatio(answer) + if (ratio > 0.06) { + problems.push(`answer reads as English (${Math.round(ratio * 100)}% of its words are English function words)`) + } + } + if (expect.context_lang) { + const want = typeof expect.context_lang === 'string' ? { code: expect.context_lang } : expect.context_lang + const got = observed.context?.lang || null + if (!got || got.code !== want.code) { + problems.push(`context language is ${got ? got.code : 'unset'}, expected ${want.code}`) + } else if (typeof want.pinned === 'boolean' && got.pinned !== want.pinned) { + problems.push(`context language ${got.code} is ${got.pinned ? '' : 'not '}pinned, expected ${want.pinned ? 'pinned' : 'not pinned'}`) + } + } + if (Number.isFinite(Number(expect.min_links))) { + const links = (answer.match(/\]\(https?:\/\//g) || []).length + if (links < Number(expect.min_links)) { + problems.push(`expected at least ${expect.min_links} link(s) in the answer, got ${links}`) + } + } + if (expect.python_block && !/```python\n[\s\S]*?```/.test(answer)) { problems.push('answer carries no python block for a turn that asked for the code') } diff --git a/lib/conversationContext.mjs b/lib/conversationContext.mjs index 5561595..c3ade9f 100644 --- a/lib/conversationContext.mjs +++ b/lib/conversationContext.mjs @@ -59,6 +59,7 @@ import { stripMarkdownLinks } from './markdownLinks.mjs' import { registryKey } from './ledger.mjs' import { ranQueries } from './reproduce.mjs' +import { sanitizeLang } from './language.mjs' /** Bump when the shape changes; an older/newer context is dropped, not guessed at. */ export const CONTEXT_VERSION = 1 @@ -192,7 +193,7 @@ export function buildTurnContext(ledger, { maxTerms = MAX_TERMS, maxQueries = MA if (registry.length >= MAX_REGISTRY) break } - return { v: CONTEXT_VERSION, terms, registry } + return { v: CONTEXT_VERSION, terms, registry, lang: sanitizeLang(ledger?.langContext) } } /** @@ -201,10 +202,14 @@ export function buildTurnContext(ledger, { maxTerms = MAX_TERMS, maxQueries = MA * possibly an empty one, so no caller needs a null check. */ export function sanitizeContext(raw) { - const empty = { v: CONTEXT_VERSION, terms: [], registry: [] } + const empty = { v: CONTEXT_VERSION, terms: [], registry: [], lang: null } if (!raw || typeof raw !== 'object') return empty // A context from a different version of this shape is not partially usable. if (Number(raw.v) !== CONTEXT_VERSION) return empty + // The conversation's language: what the last typed turn was written in, or + // what the user asked for ("reply in Persian" pins it). Validated like the + // rest — a code reaches a prompt, so it has to be a code. + const lang = sanitizeLang(raw.lang) const terms = [] const seenId = new Set() @@ -257,7 +262,7 @@ export function sanitizeContext(raw) { if (registry.length >= MAX_REGISTRY) break } - return { v: CONTEXT_VERSION, terms, registry } + return { v: CONTEXT_VERSION, terms, registry, lang } } /** @@ -319,7 +324,9 @@ export function mergeContext(prev, turn, { maxTerms = MAX_TERMS, maxRegistry = M if (registry.length >= maxRegistry) break } - return { v: CONTEXT_VERSION, terms: terms.slice(0, maxTerms), registry } + // This turn's language decision wins; a turn that made none (an older + // server, a result that never reached the harness) keeps the conversation's. + return { v: CONTEXT_VERSION, terms: terms.slice(0, maxTerms), registry, lang: b.lang || a.lang } } /** diff --git a/lib/language.mjs b/lib/language.mjs new file mode 100644 index 0000000..873fa30 --- /dev/null +++ b/lib/language.mjs @@ -0,0 +1,147 @@ +// Which language a turn is answered in, and where that decision comes from. +// +// Until September 2026 nothing in the harness knew what language the user wrote +// in. The synthesiser answered in English unless the question happened to say +// otherwise, so a Persian question written in Latin letters got an English +// clarification, a Hungarian question was answered in Hungarian only because it +// ended "válaszolj magyarul", and "can you reply in persian?" was treated as a +// fresh question ("yes, I can — how can I help?") rather than as an instruction +// about the answer it had just given. +// +// The language is decided HERE, once per turn, from three inputs, and everything +// downstream reads the decision rather than re-deriving it: +// +// 1. An explicit request ("reply in Persian", "válaszolj magyarul") PINS the +// conversation: every later turn is answered in that language until the +// user asks for another. That is the literal reading of the request, and it +// is what keeps a user who types English term names but reads Persian from +// being flipped back to English by their own vocabulary. +// 2. Otherwise a TYPED turn is answered in the language it was typed in, as +// the planner read it. The planner already runs a JSON call on every typed +// question that is not a template match, so this costs nothing. +// 3. A CLICKED follow-on chip carries an English query the user did not write, +// so it inherits the language of the conversation rather than of the chip. +// The deterministic typed paths (fast path, template, context chip) only +// match English syntax, so they are English turns. +// +// Codes are BCP-47 primary subtags (ISO 639-1 where one exists), lower case. The +// planner is asked for a code but may answer with a name; both are accepted. + +const NAME_TO_CODE = Object.freeze({ + english: 'en', persian: 'fa', farsi: 'fa', hungarian: 'hu', magyar: 'hu', german: 'de', deutsch: 'de', + french: 'fr', français: 'fr', francais: 'fr', spanish: 'es', español: 'es', espanol: 'es', castilian: 'es', + portuguese: 'pt', português: 'pt', italian: 'it', italiano: 'it', dutch: 'nl', nederlands: 'nl', + polish: 'pl', polski: 'pl', czech: 'cs', slovak: 'sk', romanian: 'ro', greek: 'el', turkish: 'tr', türkçe: 'tr', + russian: 'ru', ukrainian: 'uk', bulgarian: 'bg', serbian: 'sr', croatian: 'hr', slovenian: 'sl', + swedish: 'sv', norwegian: 'no', danish: 'da', finnish: 'fi', estonian: 'et', latvian: 'lv', lithuanian: 'lt', + hebrew: 'he', arabic: 'ar', urdu: 'ur', hindi: 'hi', bengali: 'bn', tamil: 'ta', telugu: 'te', marathi: 'mr', + gujarati: 'gu', punjabi: 'pa', chinese: 'zh', mandarin: 'zh', cantonese: 'zh', japanese: 'ja', korean: 'ko', + vietnamese: 'vi', thai: 'th', indonesian: 'id', malay: 'ms', filipino: 'tl', tagalog: 'tl', swahili: 'sw', + catalan: 'ca', basque: 'eu', galician: 'gl', welsh: 'cy', irish: 'ga', gaelic: 'gd', icelandic: 'is', + afrikaans: 'af', georgian: 'ka', armenian: 'hy', azerbaijani: 'az', kazakh: 'kk', uzbek: 'uz', mongolian: 'mn', + nepali: 'ne', sinhala: 'si', burmese: 'my', khmer: 'km', lao: 'lo', amharic: 'am', somali: 'so', hausa: 'ha', + yoruba: 'yo', igbo: 'ig', zulu: 'zu', xhosa: 'xh', albanian: 'sq', macedonian: 'mk', bosnian: 'bs', + belarusian: 'be', maltese: 'mt', luxembourgish: 'lb', esperanto: 'eo', latin: 'la', kurdish: 'ku', pashto: 'ps', + dari: 'fa', tajik: 'tg', malayalam: 'ml', kannada: 'kn', odia: 'or', assamese: 'as' +}) + +// English names for the prompt that asks for the translation. A code the table +// does not know is passed to the model as the code itself, which every model +// this runs on reads correctly ("answer in fa" is understood; it is just less +// natural than "answer in Persian"). +const CODE_TO_NAME = Object.freeze({ + en: 'English', fa: 'Persian', hu: 'Hungarian', de: 'German', fr: 'French', es: 'Spanish', pt: 'Portuguese', + it: 'Italian', nl: 'Dutch', pl: 'Polish', cs: 'Czech', sk: 'Slovak', ro: 'Romanian', el: 'Greek', tr: 'Turkish', + ru: 'Russian', uk: 'Ukrainian', bg: 'Bulgarian', sr: 'Serbian', hr: 'Croatian', sl: 'Slovenian', sv: 'Swedish', + no: 'Norwegian', da: 'Danish', fi: 'Finnish', et: 'Estonian', lv: 'Latvian', lt: 'Lithuanian', he: 'Hebrew', + ar: 'Arabic', ur: 'Urdu', hi: 'Hindi', bn: 'Bengali', ta: 'Tamil', te: 'Telugu', mr: 'Marathi', gu: 'Gujarati', + pa: 'Punjabi', zh: 'Chinese', ja: 'Japanese', ko: 'Korean', vi: 'Vietnamese', th: 'Thai', id: 'Indonesian', + ms: 'Malay', tl: 'Filipino', sw: 'Swahili', ca: 'Catalan', eu: 'Basque', gl: 'Galician', cy: 'Welsh', ga: 'Irish', + gd: 'Scottish Gaelic', is: 'Icelandic', af: 'Afrikaans', ka: 'Georgian', hy: 'Armenian', az: 'Azerbaijani', + kk: 'Kazakh', uz: 'Uzbek', mn: 'Mongolian', ne: 'Nepali', si: 'Sinhala', my: 'Burmese', km: 'Khmer', lo: 'Lao', + am: 'Amharic', so: 'Somali', ha: 'Hausa', yo: 'Yoruba', ig: 'Igbo', zu: 'Zulu', xh: 'Xhosa', sq: 'Albanian', + mk: 'Macedonian', bs: 'Bosnian', be: 'Belarusian', mt: 'Maltese', lb: 'Luxembourgish', eo: 'Esperanto', + la: 'Latin', ku: 'Kurdish', ps: 'Pashto', tg: 'Tajik', ml: 'Malayalam', kn: 'Kannada', or: 'Odia', as: 'Assamese' +}) + +// Scripts written right to left. Only used to choose the fallback note's +// direction hint; the client aligns every message with dir="auto" regardless. +const RTL = new Set(['fa', 'ar', 'he', 'ur', 'ps', 'ku', 'sd', 'yi', 'ug', 'dv']) + +/** + * Normalise whatever the planner (or a client) wrote for a language into a + * lower-case primary subtag, or '' when it is not one. + * + * 'fa' -> 'fa' 'fa-IR' -> 'fa' 'Persian' -> 'fa' 'Farsi (Latin)' -> 'fa' + * 'zh-Hant' -> 'zh' '' -> '' 'gibberish' -> '' + */ +export function normaliseLanguageCode(raw) { + if (typeof raw !== 'string') return '' + const s = raw.trim().toLowerCase() + if (!s) return '' + const tag = s.split(/[-_\s(]/)[0] + // Any ISO 639 alpha code, whether or not the name table knows it: the + // tables here are conveniences for prompts, not a list of supported + // languages. There is no such list — the model translates into whatever it + // was asked for and the check decides whether that shipped. + if (/^[a-z]{2,3}$/.test(tag)) return tag + const byName = NAME_TO_CODE[s] || NAME_TO_CODE[tag] + return byName || '' +} + +/** English display name for a code, for prompts ("translate into Persian"). */ +export function languageName(code) { + const c = normaliseLanguageCode(code) + return CODE_TO_NAME[c] || c || 'English' +} + +export function isEnglish(code) { + const c = normaliseLanguageCode(code) + return !c || c === 'en' +} + +export function isRightToLeft(code) { + return RTL.has(normaliseLanguageCode(code)) +} + +/** + * Validate a `lang` block from the conversation context. It arrives from the + * client, so it is checked, not trusted: `{ code, pinned }` or null. + */ +export function sanitizeLang(raw) { + if (!raw || typeof raw !== 'object') return null + const code = normaliseLanguageCode(raw.code) + if (!code) return null + return { code, pinned: raw.pinned === true } +} + +/** + * Decide the language for one turn. + * + * @param {object} o + * @param {string} [o.planLanguage] what the planner read the question as + * @param {string} [o.requestedLanguage] a language the message asked for, if any + * @param {{code:string,pinned:boolean}|null} [o.priorLang] the conversation's + * @param {'planner'|'focus'|'context-chip'|'template'|'fast-path'} [o.via='planner'] + * @returns {{ code: string, lang: {code:string,pinned:boolean}, pinnedByThisTurn: boolean }} + * `code` is what this turn is answered in; `lang` is what the conversation + * carries forward. + */ +export function decideTurnLanguage(o = {}) { + const requested = normaliseLanguageCode(o.requestedLanguage) + const prior = sanitizeLang(o.priorLang) + const via = o.via || 'planner' + if (requested) { + return { code: requested, lang: { code: requested, pinned: true }, pinnedByThisTurn: true } + } + if (prior && prior.pinned) { + return { code: prior.code, lang: prior, pinnedByThisTurn: false } + } + if (via === 'focus') { + const code = prior?.code || 'en' + return { code, lang: { code, pinned: false }, pinnedByThisTurn: false } + } + const typed = via === 'planner' ? normaliseLanguageCode(o.planLanguage) : 'en' + const code = typed || 'en' + return { code, lang: { code, pinned: false }, pinnedByThisTurn: false } +} diff --git a/lib/liveHarness.mjs b/lib/liveHarness.mjs index 18252d6..4467e90 100644 --- a/lib/liveHarness.mjs +++ b/lib/liveHarness.mjs @@ -28,7 +28,7 @@ export const MAX_TOOL_RESULT_CHARS = (() => { import { callStructured, callStructuredVoted, MIN_RETRY_MS } from './elmClient.mjs' import { resolveRoleModel, majorityVote } from './structuredOutput.mjs' import { roleForSchemaName, roleRequestOptions, PLANNER_ESCALATION } from './roleProfiles.mjs' -import { planVoteKey, votePlanWithEscalation } from './planner.mjs' +import { planVoteKey, votePlanWithEscalation, acceptBareLanguageRequest } from './planner.mjs' import { buildFollowOns, buildTermLinks, buildCountLinks } from './followOns.mjs' import { buildReproduction, withReproduction } from './reproduce.mjs' import { supersededCounts } from './countProvenance.mjs' @@ -262,6 +262,7 @@ export function buildLiveDeps(p) { policy: PLANNER_ESCALATION, voteKeyFn: planVoteKey, vote: majorityVote, + accept: acceptBareLanguageRequest, // `budgetMs` is what the phase has LEFT, not what a round is worth: the // escalation round must finish inside the phase budget, not merely start // inside it. Each of the k votes runs in parallel, so the round's wall @@ -319,13 +320,15 @@ export function buildLiveDeps(p) { // its sampling has to travel with the request. Without this the Qwen swap // ships the default (thinking ON) into the user-visible path: 34-73s of blank // pane before the first token, for prose that measured no better. - const callTextStream = ({ messages, model, sourceQuotes }) => { + const callTextStream = ({ messages, model, sourceQuotes, silent }) => { const opts = optionsFor('synth', model) return p.streamText({ messages, model: opts.model, sourceQuotes, - sampling: opts.sampling + sampling: opts.sampling, + // Accumulate without emitting: the English draft of a non-English turn. + silent: Boolean(silent) }) } @@ -382,6 +385,11 @@ export async function runLiveHarness(opts) { // snippet, ours replaces it rather than trailing after it. answer: withReproduction(r.answer || '', reproduction, opts.question || ''), clarify: Boolean(r.clarify), + // The language this turn is answered in (lib/language.mjs), and whether the + // turn was a bare "reply in X" — nothing looked up, the previous answer to + // be re-rendered by the caller, which holds the history. + language: r.ledger?.language || 'en', + languageSwitch: Boolean(r.languageSwitch), complete: Boolean(r.complete), ledger: r.ledger, trace: r.trace, diff --git a/lib/orchestrator.mjs b/lib/orchestrator.mjs index 9586ae9..bfc7865 100644 --- a/lib/orchestrator.mjs +++ b/lib/orchestrator.mjs @@ -23,6 +23,7 @@ import { PLAN_SCHEMA, buildPlannerMessages, normalizePlan, detectFastPath, detec import { serviceIdentityBlock } from './serviceIdentity.mjs' import { resolveQuestionToChip, resolveQuestionToTemplate, contextTermsForAnaphor } from './anaphora.mjs' import { sanitizeContext, seedLedgerFromContext, priorTermId, normName, nameKeys, contextTermsNamedIn } from './conversationContext.mjs' +import { decideTurnLanguage, isEnglish, languageName } from './language.mjs' import { wantsReproduction, ranQueries } from './reproduce.mjs' import { nextAction } from './controller.mjs' import { getMissingRequiredArgs, buildRepairMessages, mergeRepairedArgs, backfillIdArgs } from './toolRepair.mjs' @@ -193,13 +194,19 @@ export async function runHarness(question, deps) { let plan = detectFocusPlan(question, deps.focus) let contextChip = null let templateHit = null + // Which branch produced the plan. The language decision reads it: a clicked + // chip inherits the conversation's language, every other branch is either a + // typed question the planner read or an English-syntax match. + let planVia = 'planner' if (plan) { + planVia = 'focus' log({ step: 'plan', via: 'focus', id: plan.steps[0].args.id, query_type: plan.steps[0].args.query_type }) emit(deps, 'Running the query behind that suggestion', 'mcp') } else if ( (contextChip = resolveQuestionToChip(question, priorContext)) && (plan = detectFocusPlan(question, contextChip)) ) { + planVia = 'context-chip' // Built through detectFocusPlan on purpose: a typed follow-on and a clicked // one must be the SAME plan, or they would answer differently. log({ @@ -231,9 +238,11 @@ export async function runHarness(question, deps) { via_template: true }] } + planVia = 'template' log({ step: 'plan', via: 'template', term: templateHit.term, query_type: templateHit.query_type }) emit(deps, 'Planning (direct lookup)') } else if ((plan = detectFastPath(question))) { + planVia = 'fast-path' log({ step: 'plan', via: 'fast-path' }) emit(deps, 'Planning (direct lookup)') } else { @@ -330,6 +339,29 @@ export async function runHarness(question, deps) { priorContext ) + // The language this turn is answered in, decided once and read everywhere + // downstream (the resolver's translation rung, the silent synthesis stream, + // and the translation pass in route.js). See lib/language.mjs for the rules. + { + const decided = decideTurnLanguage({ + planLanguage: plan.language, + requestedLanguage: plan.requested_language, + priorLang: priorContext.lang, + via: planVia + }) + ledger.language = decided.code + ledger.langContext = decided.lang + log({ step: 'language', code: decided.code, via: planVia, pinned: decided.lang.pinned }) + // "Can you reply in Persian?" with nothing else asked. There is no lookup to + // run: the answer is the previous turn's answer, in the language asked for, + // and route.js has the previous turn. Returned like a clarification — a + // turn with no evidence — but flagged so it is rendered rather than asked. + if (plan.language_request_only) { + log({ step: 'language_request', code: decided.code }) + return { answer: '', ledger, trace, clarify: false, languageSwitch: true } + } + } + // Kick off the reviewed-docs site search NOW, concurrently with the VFB work, // so its result is ready to fold in by the time we synthesise. Skipped when the // plan is just going to ask a clarifying question (nothing to answer yet). @@ -740,6 +772,37 @@ export function intrinsicTermNames(question, names = []) { return out } +const ENGLISH_NAME_SCHEMA = { + type: 'object', + additionalProperties: false, + required: ['english_name'], + properties: { english_name: { type: 'string' } } +} + +/** + * The standard English name for an anatomical, cell-type or gene name written + * in another language — the wording VFB's index can answer to. Returns '' when + * the model cannot say, or when the call fails; the caller treats both as "no + * translation" and the name stays unmatched, exactly as before this rung. + */ +export async function englishNameFor(name, language, deps, models = {}) { + const messages = [ + { + role: 'system', + content: `You translate names of Drosophila anatomical structures, neuron types, cell types, genes and other biological entities from ${languageName(language)} into the standard English name used by FlyBase and Virtual Fly Brain (e.g. "ellipsoid body", "mushroom body", "Kenyon cell", "antennal lobe projection neuron"). Keep gene symbols, GAL4 and split-GAL4 line names, dataset names (FlyWire, hemibrain, MANC, BANC, FAFB) and any identifiers exactly as written. Drop grammatical endings and articles. If the name is already English, return it unchanged. If you cannot translate it, return an empty string. JSON only.` + }, + { role: 'user', content: JSON.stringify({ name: String(name), language: String(language) }) } + ] + try { + const r = await deps.callStructured({ messages, schema: ENGLISH_NAME_SCHEMA, schemaName: 'english_term_name', model: models.extract }) + const out = r?.ok ? String(r.value?.english_name || '').trim() : '' + // A "translation" that is a sentence is not a name. + return out.length > 0 && out.length <= 80 && !/[.!?]\s|\n/.test(out) ? out : '' + } catch { + return '' + } +} + async function resolveTerms(ledger, names, deps, models, log, speculative = new Set()) { // Fan out search_terms in parallel; pick the best id; fetch term info ONCE and // cache it. VFB-first: the term-info Description is primary evidence — it often @@ -952,6 +1015,40 @@ async function resolveTerms(ledger, names, deps, models, log, speculative = new } } } + // TRANSLATION. A name written in another language fails every rung above + // for a reason none of them can fix: VFB's labels and synonyms are English + // and the index does not translate, so "ellipszis test" returns nothing + // (or, worse, junk) however it is singularised or respelt. The planner is + // told to write English names on a non-English turn, which is what usually + // makes this rung unnecessary; it is the backstop for the name the planner + // copied as written. One structured call, one more search, and the id still + // has to earn its way through the ladder — the translation only supplies a + // wording. Symbols and line names are left as they are by the prompt. + if (!directId && !resolvedId && !isEnglish(ledger.language) && typeof deps.callStructured === 'function' && !budget.expired()) { + const english = await englishNameFor(name, ledger.language, deps, models) + if (english && norm(english) !== norm(name)) { + const trRaw = await raceDeadline(deps.runTool('vfb_search_terms', { query: english, rows: 30, minimize_results: false }), budget.remaining()) + if (trRaw === TIMED_OUT) { + ladderCutShort = true + log({ resolve_budget_spent: name, at: 'translation', ms: budget.spent(), skipped: english }) + emit(deps, 'Lookup is slow — answering from what has resolved so far', 'tool') + } else { + const translated = parseMaybe(trRaw) + if (!searchIsEmpty(translated)) { + const hit = exactTermMatchId(translated, english) || pickBestTermId(translated, english) + if (hit) { + search = translated + resolvedId = hit + log({ resolve_translation: name, as: english, id: hit, language: ledger.language }) + } else if (searchIsEmpty(search)) { + // Candidates to show, at least: the English wording found + // documents where the original found none. + search = translated + } + } + } + } + } // KIND GUARD. "Find neurons similar to DA1 using NBLAST" resolved DA1 to // FBbt_00007473, larval dorsal tracheal anastomosis 1 — an exact synonym // match, which the ladder rightly puts above everything else — and NBLAST @@ -2604,10 +2701,17 @@ NEVER OVERCLAIM — this is critical. VFB holds only PARTIAL data; what it has, { role: 'user', content: `${historyBlock}QUESTION:\n${ledger.question}\n\nRESOLVED ENTITIES (refer to these by their exact full names):\n${JSON.stringify(termNames)}\n\nEVIDENCE (JSON):\n${JSON.stringify(evidence)}${availableBlock}${unmatchedBlock}${docBlock}${docMissBlock}${guidanceBlock}${codeBlock}${closingRule}\n\nWrite the answer.` } ] // Stream tokens when the caller wired a streaming sink; otherwise one-shot. + // + // `silent`: the answer is written in English whatever the question's language + // — every gate and linker downstream reads English — and translated at the end + // (route.js). On a non-English turn the English draft must not be shown, so + // the sink is asked to accumulate without emitting; the translation is what + // streams to the reader. + const silent = !isEnglish(ledger.language) if (typeof deps.callTextStream === 'function') { - return await deps.callTextStream({ messages, model: models.synth, sourceQuotes }) + return await deps.callTextStream({ messages, model: models.synth, sourceQuotes, silent }) } - return await deps.callText({ messages, model: models.synth, sourceQuotes }) + return await deps.callText({ messages, model: models.synth, sourceQuotes, silent }) } /** Map a tool name to a short user-facing status line. */ diff --git a/lib/planner.mjs b/lib/planner.mjs index a1a046e..d28a1bd 100644 --- a/lib/planner.mjs +++ b/lib/planner.mjs @@ -55,12 +55,15 @@ export const INTENTS = [ export const PLAN_SCHEMA = { type: 'object', additionalProperties: false, - required: ['intent', 'underspecified', 'clarifying_question', 'terms_to_resolve', 'steps'], + required: ['intent', 'underspecified', 'clarifying_question', 'terms_to_resolve', 'steps', 'language', 'requested_language', 'language_request_only'], properties: { intent: { type: 'string', enum: INTENTS }, underspecified: { type: 'boolean' }, clarifying_question: { type: 'string' }, terms_to_resolve: { type: 'array', items: { type: 'string' } }, + language: { type: 'string' }, + requested_language: { type: 'string' }, + language_request_only: { type: 'boolean' }, steps: { type: 'array', items: { @@ -81,7 +84,10 @@ const PLANNER_SYSTEM = `You are the planner for a Virtual Fly Brain (VFB) assist Turn the user's question into a JSON plan. Do NOT answer the question. - intent: the single best category from the allowed list. - underspecified: true only if you genuinely cannot proceed without a clarifying detail; then put one short clarifying_question (else empty string). If the question uses a pronoun or back-reference ("it", "they", "them", "those", "these", "that one") that clearly points to an entity named in PRIOR CONVERSATION, resolve it from there — do NOT mark underspecified or ask which entity. A broad question about a named subject is NOT underspecified: "how do I use X", "what was in the latest release", "what is X for" are answerable as asked, and VFB documentation can be read for them. Never ask which "aspect", "part" or "details" of a subject the user wants — answer broadly instead. Ask only when you cannot tell WHICH ENTITY is meant. -- terms_to_resolve: anatomy / neuron / gene names mentioned that need resolving to VFB ids (use the user's natural-language names, not ids). When the current question refers back to an entity by pronoun, put that entity's full name here (taken from the prior conversation), not the pronoun. +- terms_to_resolve: anatomy / neuron / gene names mentioned that need resolving to VFB ids (use the user's natural-language names, not ids). When the current question refers back to an entity by pronoun, put that entity's full name here (taken from the prior conversation), not the pronoun. VFB's labels are English: when the question is written in another language, write each anatomical or cell-type name here as its standard ENGLISH name (Hungarian "ellipszis test" -> "ellipsoid body"; Persian "جسم قارچی" -> "mushroom body"; German "Kenyon-Zellen des Pilzkörpers" -> "Kenyon cell"), one entity per entry, and leave symbols, gene names, driver-line names and dataset names (GAL4, fru, FlyWire, hemibrain) exactly as written. Never put a whole phrase or a category word (driver lines, split-GAL4 lines, neurons, images) into an entry: "Welche split-GAL4-Linien markieren die Kenyon-Zellen" resolves "Kenyon cell", not "Kenyon-Zellen split-GAL4". +- language: the ISO 639-1 code of the language the user WROTE this message in (en, fa, hu, de, zh, …). Text in Latin letters can still be another language — Persian or Hindi typed in the Latin alphabet is fa or hi, not en. A message in English that merely contains foreign anatomical names is en. +- requested_language: the ISO 639-1 code of a language the message asks to be ANSWERED in ("reply in Persian", "válaszolj magyarul", "auf Deutsch bitte"), else "". This can differ from language ("can you reply in persian?" is written in en and requests fa). +- language_request_only: true only when the message asks for a language and asks nothing else — no VFB question, no entity, nothing to look up. Then also set steps to [] and terms_to_resolve to []. - steps: the minimal ordered tool calls needed. Each step has an id (s1, s2, …), one tool name from the catalogue, and "answers" — the specific sub-questions that step must satisfy. Prefer one macro tool over chaining primitives. Keep the plan as short as possible. VFB-FIRST: for "what is / function of / where is / what is known about X" questions, use vfb_get_term_info — its Description and Relationships (e.g. capable_of, is_part_of, synaptic regions) usually answer function/anatomy/containment directly. Use specialised tools (connectivity, neurotransmitter, taxonomy, genetic tools) only for their specific purpose. Do NOT plan a literature/PubMed step: papers are a last resort the controller adds only if VFB data and the available queries cannot answer. Output JSON only, matching the schema.` @@ -272,13 +278,29 @@ export function normalizePlan(raw = {}, question = '') { intent !== 'documentation' && !HOW_TO_QUESTION.test(String(question || '')) && !ASPECT_ONLY_CLARIFICATION.test(asked) + // The language fields are read by lib/language.mjs (decideTurnLanguage). They + // are normalised there, not here: this keeps the planner's raw reading so a + // wrong code and a name-instead-of-code are both recoverable. A bare language + // request is only honoured when the plan really is bare — a planner that + // flags it while also planning a lookup has misread the message, and the + // lookup is the safer half to keep. + const language = typeof raw.language === 'string' ? raw.language.trim() : '' + const requestedLanguage = typeof raw.requested_language === 'string' ? raw.requested_language.trim() : '' + const terms = (Array.isArray(raw.terms_to_resolve) ? raw.terms_to_resolve.map(String) : []) + .filter(Boolean).filter(n => !isServiceName(n) && !isLlmClientName(n)) + const languageRequestOnly = Boolean(raw.language_request_only) && Boolean(requestedLanguage) && + steps.length === 0 && terms.length === 0 return { intent, - underspecified, - clarifying_question: underspecified ? asked : '', - terms_to_resolve: (Array.isArray(raw.terms_to_resolve) ? raw.terms_to_resolve.map(String) : []) - .filter(Boolean).filter(n => !isServiceName(n) && !isLlmClientName(n)), - steps + // A bare language request is not underspecified: it is answered by + // re-rendering the previous turn, never by asking what the user meant. + underspecified: languageRequestOnly ? false : underspecified, + clarifying_question: (underspecified && !languageRequestOnly) ? asked : '', + terms_to_resolve: terms, + steps, + language, + requested_language: requestedLanguage, + language_request_only: languageRequestOnly } } @@ -392,6 +414,18 @@ export async function votePlanWithEscalation(o = {}) { let pool = (await o.sample(Math.max(1, o.votes || 1), remainingMs())) || [] if (!pool.length) return { ok: false, error: 'planner produced no valid plans', rounds: [] } + // A reading the caller will take from round one without a vote. "Can you + // reply in Persian?" is the case: the votes disagree about which tool to + // reach for on a message that names nothing to look up, and the escalation + // round buys three more opinions — five minutes of them, measured — about a + // question that has no plan to disagree over. See acceptBareLanguageRequest. + if (typeof o.accept === 'function') { + const accepted = o.accept(pool) + if (accepted) { + return { ok: true, value: accepted, agreement: 1, votes: pool.length, escalated: false, budgetExhausted: false, rounds: [{ agreement: 1, votes: pool.length, escalated: false, accepted: true }] } + } + } + let tally = o.vote(pool, voteKeyFn) const rounds = [{ agreement: tally.agreement ?? null, votes: pool.length, escalated: false }] let escalated = false @@ -428,6 +462,21 @@ export async function votePlanWithEscalation(o = {}) { } } +/** + * The one reading a planner round may settle without a vote: the message is a + * bare request for a language. Accepted when at least one vote read it that + * way AND no vote found anything to resolve — a vote that extracted an entity + * is evidence the message asked something, and then the votes are counted as + * usual. Returns the accepted raw plan, or null. + */ +export function acceptBareLanguageRequest(pool = []) { + const plans = (Array.isArray(pool) ? pool : []).map(raw => normalizePlan(raw, '')) + if (!plans.length) return null + if (plans.some(p => (p.terms_to_resolve || []).length > 0)) return null + const i = plans.findIndex(p => p.language_request_only) + return i === -1 ? null : pool[i] +} + const MULTI_STEP_CUE = /\b(?:connect(?:s|ed|ing|ions?|ivity)?|partners?|synap(?:se|ses|tic)|pre-?synaptic|post-?synaptic|afferents?|efferents?|innervat\w*|projects?\s+to|circuits?|upstream|downstream|between|compar(?:e|es|ed|ing|isons?)|pathways?|trac(?:e|es|ed|ing)|reciprocal|converg(?:e|es|ent|ence)|vs\.?|versus|and the|both)\b/i const SPECIFIC_ROLE_CUE = /\b(?:functions?|roles?|mechanisms?|evidence|(?:neuro)?transmitters?|express(?:es|ed|ing|ion)?|drivers?|gal4|splits?|stocks?|publications?|papers?|how many|counts?|inputs?|outputs?|similar(?:ity|ities)?|morpholog(?:y|ies|ical))\b/i diff --git a/lib/translateAnswer.mjs b/lib/translateAnswer.mjs new file mode 100644 index 0000000..18ab2ef --- /dev/null +++ b/lib/translateAnswer.mjs @@ -0,0 +1,274 @@ +// Translate a finished answer into the user's language — and prove nothing +// load-bearing was lost on the way. +// +// The answer is written in English whatever language the question came in. +// That is deliberate: every safety layer behind an answer — the absence gate, +// the count repair, the grounding audit, the term and count linkers — reads +// English, and a Hungarian sentence goes straight past all of them. The photo +// that settled the design (Cologne, 10 September 2026) had the prose saying +// 327 driver lines above a table that said 392, in Hungarian, with no links at +// all, because nothing could read the sentence the number sat in. +// +// So the English pipeline runs unchanged, and translation is the LAST step, on +// the final markdown, under rules a machine can check afterwards: +// +// - every markdown link target survives verbatim (the linkers' work); +// - every number survives (the grounding layer's work); +// - every VFB identifier survives; +// +// and a translation that fails the check is not shipped. One retry with the +// failures named, then the English answer goes out with a one-line note. The +// reader gets a verified answer in their language, or a verified answer in +// English — never an unverified one. +// +// Entity names stay English inside the translation, with the local name in +// brackets on first mention. VFB's labels are English, the links open English +// pages, and a reader who wants to find "ellipsoid body" in VFB needs the +// English string in front of them. + +import { languageName, isEnglish } from './language.mjs' + +const VFB_ID_RE = /\b(?:FBbt|FBgn|FBal|FBti|FBtp|FBco|FBlc|FBrf|VFBexp|VFB)_[0-9a-zA-Z]+\b/g + +// Every decimal-digit block Unicode defines (general category Nd), by the code +// point of its zero: each block runs 0-9 contiguously from there. The check +// normalises the OUTPUT through this before comparing, so a translation that +// wrote ۳۹۲ for 392, or ๓๙๒, or ৩৯২, still passes — the prompt asks for 0-9, +// but a reader of any of those is reading the same number. Generated from +// /\p{Nd}/u over the whole code space; regenerate if Unicode adds a script. +const DIGIT_ZEROS = [ + 0x30, 0x660, 0x6f0, 0x7c0, 0x966, 0x9e6, 0xa66, 0xae6, 0xb66, 0xbe6, 0xc66, 0xce6, 0xd66, 0xde6, 0xe50, 0xed0, + 0xf20, 0x1040, 0x1090, 0x17e0, 0x1810, 0x1946, 0x19d0, 0x1a80, 0x1a90, 0x1b50, 0x1bb0, 0x1c40, 0x1c50, 0xa620, + 0xa8d0, 0xa900, 0xa9d0, 0xa9f0, 0xaa50, 0xabf0, 0xff10, 0x104a0, 0x10d30, 0x10d40, 0x11066, 0x110f0, 0x11136, + 0x111d0, 0x112f0, 0x11450, 0x114d0, 0x11650, 0x116c0, 0x116d0, 0x116da, 0x11730, 0x118e0, 0x11950, 0x11bf0, + 0x11c50, 0x11d50, 0x11da0, 0x11de0, 0x11f50, 0x16130, 0x16a60, 0x16ac0, 0x16b50, 0x16d70, 0x1ccf0, 0x1d7ce, + 0x1d7d8, 0x1d7e2, 0x1d7ec, 0x1d7f6, 0x1e140, 0x1e2f0, 0x1e4f0, 0x1e5f1, 0x1e950, 0x1fbf0 +] +const DIGIT_VALUE = new Map() +for (const zero of DIGIT_ZEROS) for (let d = 0; d <= 9; d++) DIGIT_VALUE.set(zero + d, String(d)) + +export function asciiDigits(s = '') { + return String(s).replace(/\p{Nd}/gu, ch => DIGIT_VALUE.get(ch.codePointAt(0)) ?? ch) +} + +/** + * Every markdown link target and bare URL in the text, in order of appearance. + * Reads `](` … `)` by hand so a title in quotes and a `)` inside the title do + * not truncate the target. + */ +export function linkTargets(text = '') { + const out = [] + const s = String(text) + let i = 0 + while (i < s.length) { + const at = s.indexOf('](', i) + if (at === -1) break + let j = at + 2 + // Target runs to whitespace or the closing paren. + let url = '' + while (j < s.length && !/[\s)]/.test(s[j])) url += s[j++] + if (url) out.push(url) + i = j + } + for (const m of s.matchAll(/https?:\/\/[^\s)\]>"']+/g)) { + if (!out.includes(m[0])) out.push(m[0]) + } + return out +} + +/** Text with every markdown link target (and bare URL) blanked out. */ +function withoutUrls(text = '') { + let s = String(text) + for (const url of linkTargets(s)) s = s.split(url).join(' ') + return s +} + +/** + * The numbers the prose states, as canonical digit strings ("1,335" and + * "1 335" are both "1335"). Taken from the text with the URLs removed, so a + * query parameter's digits are not demanded of the prose. + */ +// Thousands and decimal separators a number may carry, in the scripts above: +// comma, full stop, narrow and ordinary no-break spaces, the Arabic thousands +// (U+066C) and decimal (U+066B) separators, and the apostrophe some locales use. +const SEPARATORS = ',.\\u202f\\u00a0 \\u066c\\u066b\\u2019\'' +const NUMBER_RE = new RegExp(`(? 200 && (ratio < 0.3 || ratio > 5)) { + return { ok: false, missing, reason: `length ratio ${ratio.toFixed(2)}` } + } + for (const url of linkTargets(src)) if (!out.includes(url)) missing.urls.push(url) + const outCanon = asciiDigits(withoutUrls(out)).replace(SEPARATOR_BEFORE_DIGIT_RE, '') + for (const n of proseNumbers(src)) if (!outCanon.includes(n)) missing.numbers.push(n) + // Ids the PROSE states must survive in the prose; an id that only lives in a + // link target is covered by the link check. + const outProse = withoutUrls(out) + for (const id of vfbIds(withoutUrls(src))) if (!outProse.includes(id)) missing.ids.push(id) + const ok = !missing.urls.length && !missing.numbers.length && !missing.ids.length + return { + ok, + missing, + reason: ok ? '' : [ + missing.urls.length && `${missing.urls.length} link(s)`, + missing.numbers.length && `${missing.numbers.length} number(s)`, + missing.ids.length && `${missing.ids.length} id(s)` + ].filter(Boolean).join(', ') + } +} + +const KEEP_RULES = `Keep EXACTLY as written, character for character: every markdown link — the text inside [ ] and everything inside the ( ) after it, including any quoted title; every number, written with the digits 0-9 (never localised digits, never spelled out); every identifier such as FBbt_00003682 or VFB_00101567; gene symbols, GAL4 and split-GAL4 line names, allele names, and dataset names (FlyWire, hemibrain, MANC, BANC, FAFB, FANC, neuprint, CATMAID). Keep the English names of anatomical structures, neuron types and cell types exactly as written wherever they appear — they are Virtual Fly Brain's labels and they are what the links open — and you may add the translated name in parentheses immediately after the FIRST mention of each. Keep the markdown structure unchanged: headings, list markers, tables (translate the header row only, never a cell value), emphasis, inline code and fenced blocks.` + +/** + * Messages for one translation call. + * + * @param {object} o + * @param {string} o.text finished English markdown + * @param {string} o.language target code + * @param {'answer'|'clarification'} [o.kind='answer'] + * @param {{urls:string[],numbers:string[],ids:string[]}} [o.missing] what a + * previous attempt dropped, named so the retry can fix it + */ +export function translationMessages({ text, language, kind = 'answer', missing = null }) { + const lang = languageName(language) + const what = kind === 'clarification' ? 'a short clarifying question' : 'an answer' + // "Whatever language it is in": the text is English on an ordinary turn, but + // a "reply in X" turn re-renders the previous answer, which may already be + // in a third language. + const system = `You are translating ${what} from a Virtual Fly Brain (VFB) chat assistant into ${lang}, from English or from whatever language it is currently written in. Translate the prose faithfully into natural, standard written ${lang}: do not add, drop, reorder or summarise information, do not answer the question yourself, and do not comment on the text. ${KEEP_RULES} Output only the translated markdown — no preamble, no notes, no code fence around it.` + const complaint = missing && (missing.urls.length || missing.numbers.length || missing.ids.length) + ? `\n\nYour previous translation dropped or altered: ${[ + ...missing.urls.map(u => `the link ${u}`), + ...missing.numbers.map(n => `the number ${n}`), + ...missing.ids.map(i => `the identifier ${i}`) + ].join('; ')}. Translate again keeping each of these exactly as in the English.` + : '' + return [ + { role: 'system', content: system }, + { role: 'user', content: `ENGLISH:\n${text}${complaint}\n\nTranslate into ${lang}.` } + ] +} + +/** + * Translate finished markdown, verify it, retry once, or give up honestly. + * + * @param {object} o + * @param {string} o.text + * @param {string} o.language + * @param {'answer'|'clarification'} [o.kind] + * @param {(o:{messages:object[], attempt:number}) => Promise} o.call + * runs one translation and returns the full text (it may stream as it goes) + * @param {(reason:string) => void} [o.onDiscard] called before a retry when the + * previous attempt was already shown — the client must drop it + * @param {number} [o.maxAttempts=2] + * @returns {Promise<{ ok: boolean, text: string, attempts: number, reason: string }>} + */ +export async function translateMarkdown({ text, language, kind = 'answer', call, onDiscard, maxAttempts = 2 }) { + if (isEnglish(language) || !String(text || '').trim()) { + return { ok: true, text: String(text || ''), attempts: 0, reason: 'not needed' } + } + let missing = null + let reason = '' + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + if (attempt > 1) { try { onDiscard?.(reason) } catch { /* best-effort */ } } + let out = '' + try { + out = await call({ messages: translationMessages({ text, language, kind, missing }), attempt }) + } catch (err) { + reason = `call failed: ${String(err?.message || err)}` + continue + } + const check = checkTranslation(text, out) + if (check.ok) return { ok: true, text: String(out).trim(), attempts: attempt, reason: '' } + missing = check.missing + reason = check.reason + } + return { ok: false, text: String(text || ''), attempts: maxAttempts, reason } +} + +/** + * The line that goes under an English answer when the translation could not be + * verified. In English on purpose: a translated note would itself be an + * unverified translation. + */ +export function translationFallbackNote(language) { + return `_This answer is shown in English: a ${languageName(language)} translation could not be verified against the data it cites._` +} + +export const CHIP_LABELS_SCHEMA = { + type: 'object', + additionalProperties: false, + required: ['labels'], + properties: { labels: { type: 'array', items: { type: 'string' } } } +} + +export function chipLabelMessages(labels, language) { + const lang = languageName(language) + return [ + { + role: 'system', + content: `Translate these short follow-up questions from a Virtual Fly Brain chat into ${lang}, one for one, same order, same count. Keep the English names of anatomical structures, neuron types and cell types exactly as written (they are VFB's labels), keep any number in parentheses at the end exactly as written, and keep gene symbols, line names and dataset names unchanged. Return JSON: {"labels": [...]}.` + }, + { role: 'user', content: JSON.stringify({ labels }) } + ] +} + +/** + * Translate follow-on chip labels. The QUERY behind each chip stays English — + * it is what runs when the chip is clicked, and the harness reads English — + * only the visible label changes. Returns the original labels on any failure + * or shape mismatch, so a chip can never lose its text. + * + * @param {object} o + * @param {string[]} o.labels + * @param {string} o.language + * @param {(o:{messages:object[], schema:object, schemaName:string}) => Promise<{ok:boolean, value?:any}>} o.callStructured + */ +export async function translateChipLabels({ labels, language, callStructured }) { + const src = (labels || []).map(l => String(l || '')) + if (isEnglish(language) || !src.length || typeof callStructured !== 'function') return src + try { + const r = await callStructured({ messages: chipLabelMessages(src, language), schema: CHIP_LABELS_SCHEMA, schemaName: 'chip_labels' }) + const out = r?.ok && Array.isArray(r.value?.labels) ? r.value.labels.map(l => String(l || '').trim()) : null + if (!out || out.length !== src.length || out.some(l => !l)) return src + // A label's count "(392)" is data; a translation that lost it is rejected + // for that label alone. + return out.map((l, i) => { + const n = src[i].match(/\((\d[\d,]*)\)\s*$/) + return n && !asciiDigits(l).includes(n[1]) ? src[i] : l + }) + } catch { + return src + } +} diff --git a/scripts/run-task-battery.mjs b/scripts/run-task-battery.mjs index 266e5b0..99a2caa 100644 --- a/scripts/run-task-battery.mjs +++ b/scripts/run-task-battery.mjs @@ -682,7 +682,9 @@ async function runTask(baseUrl, task, repetition, timeoutMs, runId) { problems: turnProblems }) - messages.push({ role: 'assistant', content: answer }) + // As the UI sends it: the assistant message carries its chips, which is + // how a "reply in " turn re-offers the previous turn's chips. + messages.push({ role: 'assistant', content: answer, followOns }) context = parsed.result?.context || context lastResult = parsed.result } diff --git a/tests/task-battery/tasks.json b/tests/task-battery/tasks.json index 23dfc2b..e78c4e1 100644 --- a/tests/task-battery/tasks.json +++ b/tests/task-battery/tasks.json @@ -823,5 +823,146 @@ } } ] + }, + { + "id": "L1", + "tier": 2, + "title": "Language \u2014 Persian typed in Latin letters is answered in Persian", + "category": "languages", + "turns": [ + { + "question": "chand ta driverline dar flywire data vojood dare?", + "expect": { + "answer_script": "Arabic", + "context_lang": { + "code": "fa", + "pinned": false + } + } + } + ] + }, + { + "id": "L2", + "tier": 2, + "title": "Language \u2014 Hungarian with an explicit request: links, count and chips survive, language pinned", + "category": "languages", + "turns": [ + { + "question": "sorold fel az ellipszis testet jelolo GAL4 torzseket, valaszolj magyarul", + "expect": { + "context_carries_id": "FBbt_00003678", + "no_unmatched_claim": true, + "answer_not_english": true, + "min_links": 1, + "min_followons": 2, + "context_lang": { + "code": "hu", + "pinned": true + } + } + } + ] + }, + { + "id": "L3", + "tier": 2, + "title": "Language \u2014 a Japanese question with a Japanese term name resolves and answers in Japanese", + "category": "languages", + "turns": [ + { + "question": "\u30ad\u30ce\u30b3\u4f53\u3068\u306f\u4f55\u3067\u3059\u304b\uff1f", + "expect": { + "context_carries_id": "FBbt_00005801", + "no_unmatched_claim": true, + "answer_not_english": true, + "min_links": 1, + "context_lang": { + "code": "ja", + "pinned": false + } + } + } + ] + }, + { + "id": "L4", + "tier": 2, + "title": "Language \u2014 a Chinese question is answered in Chinese with its links", + "category": "languages", + "turns": [ + { + "question": "\u8611\u83c7\u4f53\u6709\u54ea\u4e9b\u4e9a\u578b\uff1f", + "expect": { + "context_carries_id": "FBbt_00005801", + "no_unmatched_claim": true, + "answer_script": "Han", + "min_links": 1, + "context_lang": { + "code": "zh", + "pinned": false + } + } + } + ] + }, + { + "id": "L5", + "tier": 2, + "title": "Language \u2014 a German question with a symbol inside a translated name", + "category": "languages", + "turns": [ + { + "question": "Welche split-GAL4-Linien markieren die Kenyon-Zellen des Pilzk\u00f6rpers?", + "expect": { + "context_carries_id": "FBbt_00003686", + "no_unmatched_claim": true, + "answer_not_english": true, + "min_links": 1, + "context_lang": { + "code": "de", + "pinned": false + } + } + } + ] + }, + { + "id": "L6", + "tier": 7, + "title": "Language \u2014 \"can you reply in persian?\" re-renders the previous answer and pins the conversation, English follow-up included", + "category": "languages", + "turns": [ + { + "question": "What is the ellipsoid body?", + "expect": { + "context_carries_id": "FBbt_00003678", + "min_followons": 3 + } + }, + { + "question": "can you reply in persian?", + "expect": { + "answer_script": "Arabic", + "min_links": 1, + "context_lang": { + "code": "fa", + "pinned": true + } + } + }, + { + "question": "What is the fan-shaped body?", + "expect": { + "context_carries": "fan-shaped body", + "answer_script": "Arabic", + "min_links": 1, + "context_lang": { + "code": "fa", + "pinned": true + } + } + } + ] } ] diff --git a/tests/unit/language.test.mjs b/tests/unit/language.test.mjs new file mode 100644 index 0000000..b4e8251 --- /dev/null +++ b/tests/unit/language.test.mjs @@ -0,0 +1,493 @@ +// Answering in the user's language: the decision, the translation check, and +// the two places the harness reads the decision. +// +// The cases are the Cologne table, 10 September 2026 (feedback transcripts): +// - "chand ta driverline dar flywire data vojood dare?" — Persian in Latin +// letters — got an English clarifying question back. +// - "sorold fel az ellipszis testet jelolo GAL4 torzseket, valaszolj magyarul" +// was answered in Hungarian with no links, and said 327 where the table +// under it said 392. +// - "can you reply in persian?" was answered "yes, I can — how can I help?" +// instead of with the previous answer in Persian. +// +// Run: node --test tests/unit/language.test.mjs + +import { test } from 'node:test' +import assert from 'node:assert/strict' + +import { normaliseLanguageCode, languageName, isEnglish, isRightToLeft, sanitizeLang, decideTurnLanguage } from '../../lib/language.mjs' +import { normalizePlan, acceptBareLanguageRequest, votePlanWithEscalation } from '../../lib/planner.mjs' +import { sanitizeContext, mergeContext, buildTurnContext, CONTEXT_VERSION } from '../../lib/conversationContext.mjs' +import { + linkTargets, proseNumbers, vfbIds, asciiDigits, checkTranslation, translationMessages, + translateMarkdown, translateChipLabels, translationFallbackNote, chipLabelMessages +} from '../../lib/translateAnswer.mjs' +import { runHarness, englishNameFor } from '../../lib/orchestrator.mjs' +import { checkTurn, scriptShare, englishStopwordRatio } from '../../lib/battery/conversation.mjs' + +// --- codes ------------------------------------------------------------------- + +test('a code, a tag, or a name all normalise to the primary subtag', () => { + assert.equal(normaliseLanguageCode('fa'), 'fa') + assert.equal(normaliseLanguageCode('fa-IR'), 'fa') + assert.equal(normaliseLanguageCode(' Persian '), 'fa') + assert.equal(normaliseLanguageCode('Farsi (Latin script)'), 'fa') + assert.equal(normaliseLanguageCode('Hungarian'), 'hu') + assert.equal(normaliseLanguageCode('zh-Hant'), 'zh') + assert.equal(normaliseLanguageCode('EN'), 'en') + assert.equal(normaliseLanguageCode(''), '') + assert.equal(normaliseLanguageCode('unknown'), '') + assert.equal(normaliseLanguageCode(42), '') + assert.equal(normaliseLanguageCode('yue'), 'yue', 'any ISO 639 alpha code is a code, table or no table') + assert.equal(normaliseLanguageCode('haw-US'), 'haw') +}) + +test('names, direction, and what counts as English', () => { + assert.equal(languageName('fa'), 'Persian') + assert.equal(languageName('hu'), 'Hungarian') + assert.equal(languageName(''), 'English') + assert.equal(languageName('xx'), 'xx', 'an unknown code is passed through, never invented') + assert.equal(isEnglish(''), true) + assert.equal(isEnglish('en'), true) + assert.equal(isEnglish('English'), true) + assert.equal(isEnglish('fa'), false) + assert.equal(isRightToLeft('fa'), true) + assert.equal(isRightToLeft('hu'), false) +}) + +test('sanitizeLang accepts only a shape with a real code', () => { + assert.deepEqual(sanitizeLang({ code: 'fa', pinned: true }), { code: 'fa', pinned: true }) + assert.deepEqual(sanitizeLang({ code: 'Persian', pinned: 'yes' }), { code: 'fa', pinned: false }) + assert.equal(sanitizeLang({ code: 'gibberish' }), null) + assert.equal(sanitizeLang('fa'), null) + assert.equal(sanitizeLang(null), null) +}) + +// --- the decision ------------------------------------------------------------ + +test('a typed question is answered in the language the planner read', () => { + const d = decideTurnLanguage({ planLanguage: 'hu', priorLang: null, via: 'planner' }) + assert.equal(d.code, 'hu') + assert.deepEqual(d.lang, { code: 'hu', pinned: false }) +}) + +test('romanised Persian read as fa is answered in Persian', () => { + assert.equal(decideTurnLanguage({ planLanguage: 'fa' }).code, 'fa') +}) + +test('an explicit request pins the conversation, whatever the message was written in', () => { + const d = decideTurnLanguage({ planLanguage: 'en', requestedLanguage: 'Persian' }) + assert.equal(d.code, 'fa') + assert.deepEqual(d.lang, { code: 'fa', pinned: true }) + assert.equal(d.pinnedByThisTurn, true) +}) + +test('a pinned language survives an English typed turn and a clicked chip', () => { + const prior = { code: 'fa', pinned: true } + assert.equal(decideTurnLanguage({ planLanguage: 'en', priorLang: prior, via: 'planner' }).code, 'fa') + assert.equal(decideTurnLanguage({ priorLang: prior, via: 'focus' }).code, 'fa') + assert.equal(decideTurnLanguage({ priorLang: prior, via: 'fast-path' }).code, 'fa') +}) + +test('a new explicit request replaces a pin', () => { + const d = decideTurnLanguage({ requestedLanguage: 'de', priorLang: { code: 'fa', pinned: true } }) + assert.deepEqual(d.lang, { code: 'de', pinned: true }) +}) + +test('an unpinned conversation: chips inherit, typed English turns are English', () => { + const prior = { code: 'hu', pinned: false } + assert.equal(decideTurnLanguage({ priorLang: prior, via: 'focus' }).code, 'hu', 'a clicked chip carries an English query the user did not write') + assert.equal(decideTurnLanguage({ planLanguage: 'en', priorLang: prior, via: 'planner' }).code, 'en') + assert.equal(decideTurnLanguage({ priorLang: prior, via: 'template' }).code, 'en', 'a template match is English syntax') + assert.equal(decideTurnLanguage({ priorLang: prior, via: 'context-chip' }).code, 'en') +}) + +test('no information at all is English', () => { + assert.equal(decideTurnLanguage({}).code, 'en') + assert.equal(decideTurnLanguage({ planLanguage: 'gibberish' }).code, 'en') +}) + +// --- the planner's fields ---------------------------------------------------- + +test('normalizePlan carries the language fields through and tolerates their absence', () => { + const p = normalizePlan({ intent: 'other', steps: [], language: ' hu ', requested_language: '', language_request_only: false }) + assert.equal(p.language, 'hu') + assert.equal(p.requested_language, '') + assert.equal(p.language_request_only, false) + const old = normalizePlan({ intent: 'other', steps: [] }) + assert.equal(old.language, '') + assert.equal(old.requested_language, '') + assert.equal(old.language_request_only, false) +}) + +test('a bare language request is only honoured when the plan really is bare', () => { + const bare = normalizePlan({ intent: 'other', steps: [], terms_to_resolve: [], language: 'en', requested_language: 'fa', language_request_only: true, underspecified: true, clarifying_question: 'Which language?' }) + assert.equal(bare.language_request_only, true) + assert.equal(bare.underspecified, false, 'a language request is never a clarification') + assert.equal(bare.clarifying_question, '') + + const withLookup = normalizePlan({ intent: 'term_info', steps: [{ id: 's1', tool: 'vfb_get_term_info', answers: ['x'] }], terms_to_resolve: ['ellipsoid body'], language: 'hu', requested_language: 'hu', language_request_only: true }) + assert.equal(withLookup.language_request_only, false, 'a planner that also planned a lookup misread the message; the lookup wins') + assert.equal(withLookup.requested_language, 'hu', 'the request still pins the language') + + const noCode = normalizePlan({ intent: 'other', steps: [], language: 'en', requested_language: '', language_request_only: true }) + assert.equal(noCode.language_request_only, false, 'a request with no language named is not a request') +}) + +test('a bare language request is accepted from round one, unless any vote found something to resolve', async () => { + const bare = { intent: 'other', underspecified: false, clarifying_question: '', terms_to_resolve: [], steps: [], language: 'en', requested_language: 'fa', language_request_only: true } + const docs = { intent: 'documentation', underspecified: false, clarifying_question: '', terms_to_resolve: [], steps: [{ id: 's1', tool: 'search_reviewed_docs', answers: ['x'] }], language: 'en', requested_language: 'fa', language_request_only: false } + const entity = { ...docs, terms_to_resolve: ['mushroom body'] } + assert.equal(acceptBareLanguageRequest([docs, bare, docs]), bare) + assert.equal(acceptBareLanguageRequest([docs, bare, entity]), null, 'a vote that extracted an entity means the message asked something') + assert.equal(acceptBareLanguageRequest([docs, docs]), null) + assert.equal(acceptBareLanguageRequest([]), null) + + // Through the vote: no escalation round is bought for it. + let sampled = 0 + const r = await votePlanWithEscalation({ + votes: 3, + policy: { minAgreement: 0.67, extraVotes: 3, maxRounds: 1 }, + sample: async () => { sampled++; return [docs, bare, { ...docs, intent: 'other' }] }, + vote: (pool) => ({ value: pool[0], agreement: 1 / 3 }), + accept: acceptBareLanguageRequest + }) + assert.equal(r.ok, true) + assert.equal(r.value, bare) + assert.equal(r.escalated, false) + assert.equal(sampled, 1, 'one round only') + assert.equal(r.rounds[0].accepted, true) +}) + +// --- the context ------------------------------------------------------------- + +test('the context carries a validated lang block, and this turn wins over the last', () => { + const clean = sanitizeContext({ v: CONTEXT_VERSION, terms: [], registry: [], lang: { code: 'Persian', pinned: true } }) + assert.deepEqual(clean.lang, { code: 'fa', pinned: true }) + assert.equal(sanitizeContext({ v: CONTEXT_VERSION, terms: [], registry: [], lang: { code: 'nope' } }).lang, null) + assert.equal(sanitizeContext(null).lang, null) + + const prev = { v: CONTEXT_VERSION, terms: [], registry: [], lang: { code: 'hu', pinned: false } } + const turn = { v: CONTEXT_VERSION, terms: [], registry: [], lang: { code: 'fa', pinned: true } } + assert.deepEqual(mergeContext(prev, turn).lang, { code: 'fa', pinned: true }) + assert.deepEqual(mergeContext(prev, { v: CONTEXT_VERSION, terms: [], registry: [] }).lang, { code: 'hu', pinned: false }, 'a turn that decided nothing keeps the conversation language') +}) + +test('buildTurnContext writes the ledger decision into the context', () => { + assert.deepEqual(buildTurnContext({ terms: {}, registry: {}, langContext: { code: 'fa', pinned: true } }).lang, { code: 'fa', pinned: true }) + assert.equal(buildTurnContext({ terms: {}, registry: {} }).lang, null) +}) + +// --- the translation check --------------------------------------------------- + +const EN = [ + 'The [ellipsoid body](https://www.virtualflybrain.org/reports/FBbt_00003678 "Open ellipsoid body in Virtual Fly Brain") is a neuropil.', + 'VFB holds [392](https://v2.virtualflybrain.org/org.geppetto.frontend/geppetto?q=FBbt_00003678,TransgeneExpressionHere "Run in VFB") driver lines, 1,335 images and 3 subclasses (FBbt_00003678).' +].join('\n\n') + +test('link targets are read to the closing paren, titles and all', () => { + assert.deepEqual(linkTargets(EN), [ + 'https://www.virtualflybrain.org/reports/FBbt_00003678', + 'https://v2.virtualflybrain.org/org.geppetto.frontend/geppetto?q=FBbt_00003678,TransgeneExpressionHere' + ]) + assert.deepEqual(linkTargets('see https://flybase.org/reports/FBgn0000014 now'), ['https://flybase.org/reports/FBgn0000014']) +}) + +test('prose numbers are canonical digit strings, and never come from a URL', () => { + assert.deepEqual(proseNumbers(EN).sort(), ['1335', '3', '392'].sort()) + assert.deepEqual(proseNumbers('1 335 cells and 2.5 mm'), ['1335', '25']) + assert.deepEqual(vfbIds(EN), ['FBbt_00003678']) + assert.equal(asciiDigits('۳۹۲ and ٣'), '392 and 3') + // Every script's digits, not a hand-picked few: Thai, Bengali, Devanagari, + // Myanmar, fullwidth, and a digit run inside a symbol left alone. + assert.equal(asciiDigits('๓๙๒ ৩৯২ ३९२ ၃၉၂ 392'), '392 392 392 392 392') + assert.deepEqual(proseNumbers('R66A08 and GAL4 drive 12 cells'), ['12']) +}) + +test('a faithful translation passes, whichever digits it used', () => { + const hu = EN + .replace('is a neuropil.', '(ellipszis test) egy neuropil.') + .replace('driver lines, 1,335 images and 3 subclasses', 'driver vonalat, 1 335 képet és 3 alosztályt tart nyilván') + assert.equal(checkTranslation(EN, hu).ok, true) + const fa = EN.replace('392', '۳۹۲').replace('1,335', '۱٬۳۳۵') + const check = checkTranslation(EN, fa) + assert.equal(check.ok, true, check.reason) +}) + +test('a translation that dropped a link, a number or an id fails, and says which', () => { + const noLink = EN.replace('[392](https://v2.virtualflybrain.org/org.geppetto.frontend/geppetto?q=FBbt_00003678,TransgeneExpressionHere "Run in VFB")', '392') + const c1 = checkTranslation(EN, noLink) + assert.equal(c1.ok, false) + assert.equal(c1.missing.urls.length, 1) + assert.match(c1.reason, /1 link/) + + const wrongNumber = EN.replace('1,335', '1,325') + const c2 = checkTranslation(EN, wrongNumber) + assert.equal(c2.ok, false) + assert.deepEqual(c2.missing.numbers, ['1335']) + + const noId = EN.replace(' (FBbt_00003678)', '') + const c3 = checkTranslation(EN, noId) + assert.equal(c3.ok, false) + assert.deepEqual(c3.missing.ids, ['FBbt_00003678']) +}) + +test('an empty or wildly resized output fails', () => { + assert.equal(checkTranslation(EN, '').ok, false) + const long = 'x'.repeat(300) + assert.equal(checkTranslation(long, 'short').reason.startsWith('length ratio'), true) +}) + +test('the prompt names the language, the invariants, and what a retry must fix', () => { + const m = translationMessages({ text: EN, language: 'hu' }) + assert.equal(m[0].role, 'system') + assert.match(m[0].content, /into Hungarian/) + assert.match(m[0].content, /character for character/) + assert.match(m[1].content, /Translate into Hungarian\.$/) + assert.match(translationMessages({ text: EN, language: 'yue' })[0].content, /into yue,/, 'a code the name table does not know is still asked for') + const retry = translationMessages({ text: EN, language: 'hu', missing: { urls: ['https://x'], numbers: ['392'], ids: [] } }) + assert.match(retry[1].content, /dropped or altered: the link https:\/\/x; the number 392/) + const clar = translationMessages({ text: 'Which one?', language: 'fa', kind: 'clarification' }) + assert.match(clar[0].content, /clarifying question/) +}) + +test('translateMarkdown: English and empty text are returned untouched, with no call', async () => { + let calls = 0 + const call = async () => { calls++; return 'x' } + assert.deepEqual(await translateMarkdown({ text: EN, language: 'en', call }), { ok: true, text: EN, attempts: 0, reason: 'not needed' }) + assert.equal((await translateMarkdown({ text: ' ', language: 'fa', call })).attempts, 0) + assert.equal(calls, 0) +}) + +test('translateMarkdown: a verified first attempt ships', async () => { + const hu = EN.replace('is a neuropil', 'egy neuropil') + const r = await translateMarkdown({ text: EN, language: 'hu', call: async () => hu }) + assert.equal(r.ok, true) + assert.equal(r.text, hu) + assert.equal(r.attempts, 1) +}) + +test('translateMarkdown: a dropped link is retried with the complaint, and the client drops the draft', async () => { + const bad = EN.replace('[392](https://v2.virtualflybrain.org/org.geppetto.frontend/geppetto?q=FBbt_00003678,TransgeneExpressionHere "Run in VFB")', '392') + const good = EN.replace('is a neuropil', 'egy neuropil') + const seen = [] + let discarded = 0 + const r = await translateMarkdown({ + text: EN, language: 'hu', + call: async ({ messages, attempt }) => { seen.push(messages[1].content); return attempt === 1 ? bad : good }, + onDiscard: () => { discarded++ } + }) + assert.equal(r.ok, true) + assert.equal(r.attempts, 2) + assert.equal(discarded, 1) + assert.match(seen[1], /dropped or altered: the link/) +}) + +test('translateMarkdown: two failures give up honestly with the English', async () => { + const r = await translateMarkdown({ text: EN, language: 'hu', call: async () => 'rövid' }) + assert.equal(r.ok, false) + assert.equal(r.text, EN) + assert.equal(r.attempts, 2) + assert.ok(r.reason) + assert.match(translationFallbackNote('hu'), /shown in English/) + assert.match(translationFallbackNote('hu'), /Hungarian/) +}) + +test('translateMarkdown: a throwing call counts as a failed attempt', async () => { + let n = 0 + const r = await translateMarkdown({ text: EN, language: 'hu', call: async () => { n++; throw new Error('gateway 502') } }) + assert.equal(r.ok, false) + assert.equal(n, 2) + assert.match(r.reason, /gateway 502/) +}) + +// --- chips ------------------------------------------------------------------- + +const CHIPS = ['Which driver lines label the ellipsoid body? (392)', 'Which neurons have synaptic terminals in the ellipsoid body? (102)'] + +test('chip labels are translated one for one; the query is never touched', async () => { + const labels = await translateChipLabels({ + labels: CHIPS, language: 'hu', + callStructured: async ({ messages, schemaName }) => { + assert.equal(schemaName, 'chip_labels') + assert.match(messages[0].content, /into Hungarian/) + return { ok: true, value: { labels: ['Mely driver vonalak jelölik az ellipsoid body-t? (392)', 'Mely neuronoknak vannak szinaptikus végződései az ellipsoid body-ban? (102)'] } } + } + }) + assert.equal(labels.length, 2) + assert.match(labels[0], /^Mely/) + assert.match(chipLabelMessages(CHIPS, 'fa')[1].content, /392/) +}) + +test('chip labels fall back to English on any shape mismatch, and per label when a count is lost', async () => { + assert.deepEqual(await translateChipLabels({ labels: CHIPS, language: 'en', callStructured: async () => { throw new Error('never') } }), CHIPS) + assert.deepEqual(await translateChipLabels({ labels: CHIPS, language: 'hu', callStructured: async () => ({ ok: true, value: { labels: ['only one'] } }) }), CHIPS) + assert.deepEqual(await translateChipLabels({ labels: CHIPS, language: 'hu', callStructured: async () => ({ ok: false }) }), CHIPS) + assert.deepEqual(await translateChipLabels({ labels: CHIPS, language: 'hu', callStructured: async () => { throw new Error('502') } }), CHIPS) + const partial = await translateChipLabels({ labels: CHIPS, language: 'hu', callStructured: async () => ({ ok: true, value: { labels: ['Mely driver vonalak? (392)', 'Mely neuronok?'] } }) }) + assert.equal(partial[0], 'Mely driver vonalak? (392)') + assert.equal(partial[1], CHIPS[1], 'the label that lost its count keeps its English') +}) + +// --- the battery's language checks ------------------------------------------ + +test('scriptShare and englishStopwordRatio read the prose, not the links', () => { + const fa = 'VFB تعداد [70](https://v2.virtualflybrain.org/x "Run in VFB") نوع نورون را برای [ellipsoid body](https://www.virtualflybrain.org/reports/FBbt_00003678 "Open") ثبت کرده است.' + assert.ok(scriptShare(fa, 'Arabic') > 0.7, 'the English label inside the link text does not count against the Persian') + assert.ok(scriptShare(fa, 'Latin') < 0.3) + assert.equal(scriptShare('', 'Arabic'), 0) + assert.equal(scriptShare('abc', 'NotAScript'), 0) + const hu = 'A VFB jelenleg [392](https://v2.virtualflybrain.org/x "Run in VFB") transzgén expressziós jelentést tart nyilván az ellipszis testre (ellipsoid body) vonatkozóan.' + assert.ok(englishStopwordRatio(hu) < 0.06, `hungarian ratio ${englishStopwordRatio(hu)}`) + const en = 'VFB holds 392 transgene expression reports for the ellipsoid body, and these are the ones with GAL4 drivers.' + assert.ok(englishStopwordRatio(en) > 0.2, `english ratio ${englishStopwordRatio(en)}`) +}) + +test('checkTurn: the language expectations name what went wrong', () => { + const en = 'VFB holds [392](https://v2.virtualflybrain.org/x "Run in VFB") reports for the ellipsoid body.' + const problems = checkTurn( + { answer_script: 'Arabic', answer_not_english: true, context_lang: { code: 'fa', pinned: true }, min_links: 2 }, + { answer: en, followOns: [], context: { lang: { code: 'fa', pinned: false } } } + ) + assert.equal(problems.length, 4, problems.join('\n')) + assert.match(problems[0], /not written in the Arabic script/) + assert.match(problems[1], /reads as English/) + assert.match(problems[2], /not pinned, expected pinned/) + assert.match(problems[3], /at least 2 link/) + assert.deepEqual(checkTurn({ context_lang: 'hu' }, { answer: 'x', followOns: [], context: {} }), ['context language is unset, expected hu']) + const fa = 'VFB تعداد [70](https://v2.virtualflybrain.org/x "Run in VFB") نوع نورون را ثبت کرده است.' + assert.deepEqual(checkTurn({ answer_script: 'Arabic', answer_not_english: true, context_lang: 'fa', min_links: 1 }, { answer: fa, followOns: [], context: { lang: { code: 'fa', pinned: true } } }), []) +}) + +// --- the harness ------------------------------------------------------------- + +const TOOL_DEFS = [ + { name: 'vfb_search_terms', purpose: 'search terms', parameters: { type: 'object', required: ['query'], properties: { query: { type: 'string' }, rows: { type: 'number' }, minimize_results: { type: 'boolean' } } } }, + { name: 'vfb_get_term_info', purpose: 'term info', parameters: { type: 'object', required: ['id'], properties: { id: { type: 'string' } } } } +] +const EB = { short_form: 'FBbt_00003678', label: 'EB (ellipsoid body)', original_label: 'ellipsoid body', facets_annotation: ['Anatomy', 'Class'] } + +function makeDeps({ plan, hits = {}, englishName = '', context = null, focus = null }) { + const calls = { searches: [], structured: [], synth: [] } + return { + calls, + toolDefs: TOOL_DEFS, + models: { planner: 'm', extract: 'm', synth: 'm' }, + maxToolRounds: 4, + context, + focus, + async callStructured({ schemaName }) { + calls.structured.push(schemaName) + if (schemaName === 'plan') return { ok: true, value: plan } + if (schemaName === 'english_term_name') return englishName ? { ok: true, value: { english_name: englishName } } : { ok: false } + if (schemaName === 'extract') return { ok: true, value: { relevant: true, answered: true, claim: 'c', verbatim: 'v' } } + return { ok: false } + }, + async callText(o) { calls.synth.push(o); return 'FINAL ANSWER' }, + async runTool(name, args) { + if (name === 'vfb_search_terms') { + calls.searches.push(args.query) + const docs = hits[args.query] + return docs ? { response: { docs } } : { response: { docs: [] } } + } + if (name === 'vfb_get_term_info') return { Id: args.id, Name: 'ellipsoid body', Publications: [] } + return { ok: true } + } + } +} + +test('a Hungarian question is answered in Hungarian: the ledger says so and the English draft is silent', async () => { + const deps = makeDeps({ + plan: { intent: 'term_info', underspecified: false, clarifying_question: '', terms_to_resolve: ['ellipsoid body'], steps: [], language: 'hu', requested_language: '', language_request_only: false }, + hits: { 'ellipsoid body': [EB] } + }) + const r = await runHarness('sorold fel az ellipszis testet jelolo GAL4 torzseket', deps) + assert.equal(r.ledger.language, 'hu') + assert.deepEqual(r.ledger.langContext, { code: 'hu', pinned: false }) + assert.equal(r.ledger.terms['ellipsoid body'].id, 'FBbt_00003678') + assert.ok(deps.calls.synth.length >= 1) + assert.equal(deps.calls.synth[0].silent, true, 'the English draft is accumulated, not shown') + assert.ok(r.trace.some(e => e.step === 'language' && e.code === 'hu')) +}) + +test('an English question streams as before', async () => { + const deps = makeDeps({ + plan: { intent: 'term_info', underspecified: false, clarifying_question: '', terms_to_resolve: ['ellipsoid body'], steps: [], language: 'en', requested_language: '', language_request_only: false }, + hits: { 'ellipsoid body': [EB] } + }) + const r = await runHarness('Tell me about the ellipsoid body and its driver lines', deps) + assert.equal(r.ledger.language, 'en') + assert.equal(deps.calls.synth[0].silent, false) + assert.ok(!deps.calls.structured.includes('english_term_name'), 'no translation rung on an English turn') +}) + +test('the translation rung: a name the planner copied as written is resolved through its English name', async () => { + const deps = makeDeps({ + plan: { intent: 'term_info', underspecified: false, clarifying_question: '', terms_to_resolve: ['ellipszis test'], steps: [], language: 'hu', requested_language: '', language_request_only: false }, + hits: { 'ellipsoid body': [EB] }, + englishName: 'ellipsoid body' + }) + const r = await runHarness('mi az ellipszis test?', deps) + assert.equal(r.ledger.terms['ellipszis test'].id, 'FBbt_00003678') + assert.ok(deps.calls.structured.includes('english_term_name')) + assert.ok(deps.calls.searches.includes('ellipsoid body')) + assert.ok(r.trace.some(e => e.resolve_translation === 'ellipszis test' && e.as === 'ellipsoid body' && e.id === 'FBbt_00003678')) +}) + +test('the translation rung: no English name, no change — the term abstains as before', async () => { + const deps = makeDeps({ + plan: { intent: 'term_info', underspecified: false, clarifying_question: '', terms_to_resolve: ['valami'], steps: [], language: 'hu', requested_language: '', language_request_only: false }, + hits: {}, englishName: '' + }) + const r = await runHarness('mi az a valami?', deps) + assert.equal(r.ledger.terms['valami'].id, null) +}) + +test('englishNameFor refuses a sentence and survives a failed call', async () => { + const deps = { async callStructured() { return { ok: true, value: { english_name: 'The ellipsoid body is a neuropil. It lies centrally.' } } } } + assert.equal(await englishNameFor('x', 'hu', deps), '') + assert.equal(await englishNameFor('x', 'hu', { async callStructured() { throw new Error('boom') } }), '') + assert.equal(await englishNameFor('x', 'hu', { async callStructured() { return { ok: true, value: { english_name: ' ellipsoid body ' } } } }), 'ellipsoid body') +}) + +test('"can you reply in persian?" is a language switch: nothing looked up, the language pinned', async () => { + const deps = makeDeps({ + plan: { intent: 'other', underspecified: false, clarifying_question: '', terms_to_resolve: [], steps: [], language: 'en', requested_language: 'fa', language_request_only: true } + }) + const r = await runHarness('can you reply in persian?', deps) + assert.equal(r.languageSwitch, true) + assert.equal(r.answer, '') + assert.equal(r.ledger.language, 'fa') + assert.deepEqual(r.ledger.langContext, { code: 'fa', pinned: true }) + assert.equal(deps.calls.searches.length, 0) + assert.equal(deps.calls.synth.length, 0) +}) + +test('a pinned conversation answers a typed English question in the pinned language', async () => { + const deps = makeDeps({ + plan: { intent: 'term_info', underspecified: false, clarifying_question: '', terms_to_resolve: ['ellipsoid body'], steps: [], language: 'en', requested_language: '', language_request_only: false }, + hits: { 'ellipsoid body': [EB] }, + context: { v: CONTEXT_VERSION, terms: [], registry: [], lang: { code: 'fa', pinned: true } } + }) + const r = await runHarness('Tell me about the ellipsoid body and its driver lines', deps) + assert.equal(r.ledger.language, 'fa') + assert.deepEqual(r.ledger.langContext, { code: 'fa', pinned: true }) +}) + +test('a clicked chip inherits the conversation language', async () => { + const deps = makeDeps({ + plan: null, + hits: {}, + context: { v: CONTEXT_VERSION, terms: [], registry: [], lang: { code: 'hu', pinned: false } }, + focus: { id: 'FBbt_00003678', query_type: 'TransgeneExpressionHere' } + }) + deps.runTool = async (name, args) => { + if (name === 'vfb_get_term_info') return { Id: 'FBbt_00003678', Name: 'ellipsoid body', Publications: [] } + if (name === 'vfb_run_query') return { rows: [], count: 0 } + return { response: { docs: [] } } + } + deps.toolDefs = [...TOOL_DEFS, { name: 'vfb_run_query', purpose: 'run', parameters: { type: 'object', required: ['id', 'query_type'], properties: { id: { type: 'string' }, query_type: { type: 'string' } } } }] + const r = await runHarness('Which driver lines label the ellipsoid body?', deps) + assert.equal(r.ledger.language, 'hu') + assert.ok(r.trace.some(e => e.step === 'language' && e.via === 'focus')) +}) From b0fc2de4ef91ec1f1605f2eae5aee946c724e2b4 Mon Sep 17 00:00:00 2001 From: Robbie Court Date: Thu, 10 Sep 2026 16:23:38 +0000 Subject: [PATCH 2/3] Resolve a bare class symbol to its class, and retry a stale class-connectivity zero MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "what cell types are downstream of KCg?" (#66) answered about one hemibrain cell, KCg-d_R (FlyEM-HB:1003837184). VFB's index ranks hundreds of KCg-* individuals above the class whose symbol is "KCg", so the class never reached the thirty rows the resolver saw and the token-superset guess took the first individual carrying the token. The #59 class lift only fired for names shaped like " neurons"; it now also fires for a name that matched no document exactly and has an exact match among classes — exact beats guess, across the search the ranking hid. A name that IS an individual's label or synonym is left alone as before. Once resolved to the class, the answer was still "VFB does not currently hold data on downstream connectivity": the v3-cached edge is serving the count-0 results VFBquery emitted for failed class-connectivity aggregations before v1.22.51, under the MCP's own URL shapes (offset=0&limit=25 and limit=2500), while the plain URL returns 3,886 rows. A class-connectivity count 0 is now retried once with force_refresh on the same per-request budget as a -1, and a second zero is believed. Logged as CLASS CONNECTIVITY ZERO RETRY with the count after and the recompute time. Also register the roll-up classes and the self-row from a class-partner summary, not only the ranked partners, so the names the claim itself uses ("adult neuron", "adult CNS neuron", "adult interneuron") can be linked in the answer. Unit tests for all three; suite 1,378/1,380 (two skips). Verified live: "KCg" now lifts VFB_jrchjw0f -> FBbt_00049828 and answers with 3,370 downstream classes ranked by synapses per connected pair, in 97 s. --- app/api/chat/route.js | 34 ++++++++++++++++++-- lib/classPartners.mjs | 9 +++++- lib/orchestrator.mjs | 19 ++++++++++- lib/runQueryRetry.mjs | 38 ++++++++++++++++++++++ tests/unit/classLift.test.mjs | 53 +++++++++++++++++++++++++++++++ tests/unit/classPartners.test.mjs | 13 ++++++++ tests/unit/runQueryRetry.test.mjs | 22 +++++++++++++ 7 files changed, 183 insertions(+), 5 deletions(-) diff --git a/app/api/chat/route.js b/app/api/chat/route.js index 402beb1..c5c683b 100644 --- a/app/api/chat/route.js +++ b/app/api/chat/route.js @@ -52,7 +52,8 @@ import { annotateFailedRunQuery, createForceRefreshBudget, forceRefreshKey, - isFailedRunQueryPayload + isFailedRunQueryPayload, + isSuspiciousZeroRunQuery } from '../../../lib/runQueryRetry.mjs' import { parseThumbnailEntity, @@ -1261,7 +1262,14 @@ function getForceRefreshBudget(context) { async function callMcpToolTextWithForceRefresh(client, name, args, { budget } = {}) { const text = mcpResultToText(await callMcpToolWithRetry(client, name, args)) - if (!FORCE_REFRESH_RETRY_TOOLS.has(name) || !isFailedRunQueryPayload(text)) return text + if (!FORCE_REFRESH_RETRY_TOOLS.has(name)) return text + const failed = isFailedRunQueryPayload(text) + // A class-connectivity count 0 is retried like a -1 (#66): the edge cache + // kept the zeros VFBquery used to emit for a failed aggregation, and one is + // indistinguishable from an empty set until it has been recomputed once. + // See SUSPICIOUS_ZERO_QUERY_TYPES for the scope; a second zero is believed. + const suspiciousZero = !failed && isSuspiciousZeroRunQuery(name, args, text) + if (!failed && !suspiciousZero) return text // `budget` was passed by exactly one of the 27 call sites — the generic MCP // routing path. Every macro tool left it empty and fell back to a fresh @@ -1272,13 +1280,24 @@ async function callMcpToolTextWithForceRefresh(client, name, args, { budget } = // that is seven X-Force-Refresh recomputes against the shared upstream instead // of two. The request's allowance now rides on the client. const allowance = budget || client?.[FORCE_REFRESH_BUDGET] || createForceRefreshBudget(1) + // annotateFailedRunQuery leaves a count-0 payload untouched, so the + // suspicious-zero path returns the original result when the budget is spent. if (!allowance.tryConsume(forceRefreshKey(name, args))) return annotateFailedRunQuery(text) - console.error(`[VFBchat] ${name} returned count -1 — retrying once with force_refresh | args=${safeToolArgs(args)}`) + const why = failed ? 'count -1' : 'count 0 for class connectivity' + console.error(`[VFBchat] ${name} returned ${why} — retrying once with force_refresh | args=${safeToolArgs(args)}`) + const startedAt = Date.now() try { const retryText = mcpResultToText( await callMcpToolWithRetry(client, name, { ...args, force_refresh: true }) ) + if (suspiciousZero) { + // Recorded either way: a zero that became a result is a cache the edge + // is still serving, and a zero that stayed zero is a class that really + // has no connectome-annotated instances. Both are worth counting. + const after = Number(parseMaybeJsonCount(retryText)) + console.error(`[VFBchat] CLASS CONNECTIVITY ZERO RETRY | query_type=${args?.query_type} | id=${safeToolArgs({ id: args?.id })} | count_after=${Number.isFinite(after) ? after : 'n/a'} | ms=${Date.now() - startedAt}`) + } if (!isFailedRunQueryPayload(retryText)) return retryText return annotateFailedRunQuery(retryText) } catch (error) { @@ -1287,6 +1306,15 @@ async function callMcpToolTextWithForceRefresh(client, name, args, { budget } = } } +function parseMaybeJsonCount(text) { + try { + const parsed = typeof text === 'string' ? JSON.parse(text) : text + return parsed && typeof parsed === 'object' ? parsed.count : undefined + } catch { + return undefined + } +} + // The version sent here is what the MCP server records against every call this // client makes, so it comes from the one source in lib/appVersion.mjs. A // hard-coded copy goes stale at the first release nobody remembers to edit it diff --git a/lib/classPartners.mjs b/lib/classPartners.mjs index 5260be4..9c8ff2a 100644 --- a/lib/classPartners.mjs +++ b/lib/classPartners.mjs @@ -449,7 +449,14 @@ export function summariseClassPartners(parsed, { label = '', partnerFilter = '', partners: ranked.partners, aggregates: ranked.aggregates, self: ranked.self, - rows: ranked.partners.map(r => ({ name: r.label, id: r.id })) + // Every named class, not only the ranked partners: the roll-up classes and + // the self-row are named in the claim and in the table too, and `rows` is + // what the caller registers for linking. Registering partners alone left + // "adult neuron, adult CNS neuron, adult interneuron" as the only names in + // a DNp32 answer without a link (Korean turn, 10 Sep 2026). + rows: [...ranked.partners, ...ranked.aggregates, ...ranked.self] + .filter(r => r?.id) + .map(r => ({ name: r.label, id: r.id })) } } diff --git a/lib/orchestrator.mjs b/lib/orchestrator.mjs index bfc7865..2c89cae 100644 --- a/lib/orchestrator.mjs +++ b/lib/orchestrator.mjs @@ -1091,8 +1091,25 @@ async function resolveTerms(ledger, names, deps, models, log, speculative = new // knows about EPG neurons (#59). A plural or generic name that landed // on an INDIVIDUAL without matching its label exactly is a name for a // class: search the symbol alone among classes, and take an exact match. + // + // The same lift applies to a BARE symbol that landed on an individual by + // GUESS (#66). "what cell types are downstream of KCg?" resolved "KCg" + // to KCg-d_R (FlyEM-HB:1003837184), one hemibrain cell, and the answer + // listed that cell's 221 partners as the cell types downstream of KCg. + // VFB's index ranks the five hundred KCg-* individuals above the class + // whose symbol IS "KCg" (gamma Kenyon cell, FBbt_00100247), so the class + // never reached the thirty rows the ladder saw, and the token-superset + // rule took the first individual carrying the token. Nothing there was + // an exact match — the individual's label is "KCg-d_R (…)", not "KCg". + // A name that matched NO document exactly, and whose exact match among + // classes exists, names that class: exact beats guess, which is the + // ladder's own first rule, applied across the search the ranking hid. + // A name that IS an individual's label or synonym ("EPG neuron" in #59 + // matched a synonym exactly) is only lifted when its shape says type. const chosenNow = validSearchDocs(search).find(d => sfOf(d) === resolvedId) - if (chosenNow && docIsIndividual(chosenNow) && namesAType(name) && norm(docLabel(chosenNow)) !== norm(name)) { + const liftable = chosenNow && docIsIndividual(chosenNow) && norm(docLabel(chosenNow)) !== norm(name) && + (namesAType(name) || !exactTermMatchId(search, name)) + if (liftable) { const symbol = stripEntityNoun(name) if (symbol && !budget.expired()) { const classRaw = await raceDeadline( diff --git a/lib/runQueryRetry.mjs b/lib/runQueryRetry.mjs index c576752..ffa4f85 100644 --- a/lib/runQueryRetry.mjs +++ b/lib/runQueryRetry.mjs @@ -57,6 +57,44 @@ export function isFailedRunQueryPayload(text) { return Number.isFinite(count) && count < 0 } +/** + * Query types whose empty result is worth one forced recompute (#66). + * + * Class connectivity is an aggregate over every instance of the class, built + * live by VFBquery. Before v1.22.51 (4 Sep 2026) any backend hiccup during + * that aggregation collapsed to an empty list, which was reported as count 0 + * "exact" and stored by every cache in front of it — the Solr result cache + * and the nginx edge, under EVERY URL variant that had been asked. VFBquery no + * longer produces those zeros, but the edge still serves the ones it kept: + * "what cell types are downstream of KCg?" got count 0 for gamma Kenyon cell + * (FBbt_00100247) on the MCP's `offset=0&limit=…` slots while the plain URL + * returned 3,886 rows, and the chat answered "VFB does not currently hold + * data on downstream connectivity" about the most-studied cell type in the + * fly brain. + * + * So for these two query types alone, count 0 is treated the way count -1 is: + * retried once past the cache, on the same per-request budget. A genuine zero + * — a class with no connectome-annotated instances — recomputes quickly, + * because the aggregate is over nothing, and comes back 0 again; that second + * zero is believed. Everything else keeps the rule above: 0 is 0. + */ +export const SUSPICIOUS_ZERO_QUERY_TYPES = new Set(['DownstreamClassConnectivity', 'UpstreamClassConnectivity']) + +/** + * An empty class-connectivity result that may be a stale cached failure rather + * than an empty set. Only for run_query, only for the query types above, and + * only when the payload is a bona fide "count 0, no rows" — a payload that + * carries rows, an error, or a negative count is somebody else's case. + */ +export function isSuspiciousZeroRunQuery(toolName, args = {}, text) { + if (toolName !== 'run_query') return false + if (!SUSPICIOUS_ZERO_QUERY_TYPES.has(String(args?.query_type || ''))) return false + const parsed = parsePayload(text) + if (!parsed || parsed.error) return false + if (Number(parsed.count) !== 0) return false + return !Array.isArray(parsed.rows) || parsed.rows.length === 0 +} + /** * Add the failure explanation to a payload, leaving any existing note in place. * Returns the text unchanged if it is not a recognisable failed payload, so this diff --git a/tests/unit/classLift.test.mjs b/tests/unit/classLift.test.mjs index fa2e240..89e5db1 100644 --- a/tests/unit/classLift.test.mjs +++ b/tests/unit/classLift.test.mjs @@ -76,6 +76,59 @@ test('"EPG neurons" is lifted from the instance the singular found to the class assert.deepEqual(deps.calls[2].filter, ['class']) }) +// Issue #66: "what cell types are downstream of KCg?" resolved the bare symbol +// to KCg-d_R (FlyEM-HB:1003837184). VFB's index ranks hundreds of KCg-* +// individuals above the class whose symbol is "KCg", so the class never +// reached the rows the ladder saw and the token-superset guess took the first +// individual. No document matched "KCg" exactly; the class does, so it wins. +const KCG_INSTANCES = ['1003837184', '1004514584', '1004514714'].map((acc, i) => ({ + short_form: `VFB_jrchjw0${i}`, + label: `KCg-d_R (KCg-d_R (FlyEM-HB:${acc}))`, + original_label: `KCg-d_R (FlyEM-HB:${acc})`, + facets_annotation: ['Entity', 'Individual', 'Neuron', 'Adult', 'has_neuron_connectivity'] +})) +const KCG_CLASS = { short_form: 'FBbt_00100247', label: 'KCg (gamma Kenyon cell)', original_label: 'gamma Kenyon cell', synonym: ['KCg', 'gamma KC'], facets_annotation: ['Entity', 'Class', 'Neuron'] } + +test('a bare symbol that landed on an instance by guess is lifted to the class the symbol names exactly (#66)', async () => { + const deps = makeDeps('what cell types are downstream of KCg?', 'KCg') + deps.runTool = (orig => async (name, args) => { + if (name === 'vfb_search_terms') { + deps.calls.push({ q: args.query, filter: args.filter_types || null }) + if ((args.filter_types || []).includes('class')) return { results: args.query === 'KCg' ? [KCG_CLASS] : [] } + return { results: KCG_INSTANCES } + } + if (name === 'vfb_get_term_info') { + return args.id === 'FBbt_00100247' + ? { Id: args.id, Name: 'gamma Kenyon cell', IsClass: true, SuperTypes: ['Class', 'Neuron'], Publications: [], Queries: [] } + : { Id: args.id, Name: 'KCg-d_R (FlyEM-HB:1003837184)', IsIndividual: true, SuperTypes: ['Individual', 'Neuron'], Publications: [], Queries: [] } + } + return orig(name, args) + })(deps.runTool) + const r = await runHarness('what cell types are downstream of KCg?', deps) + assert.equal(r.ledger.terms.KCg.id, 'FBbt_00100247') + assert.ok(r.trace.some(e => e.resolve_lift_to_class === 'KCg' && e.from === 'VFB_jrchjw00' && e.to === 'FBbt_00100247'), 'lift logged') + const classCall = deps.calls.find(c => c.filter && c.filter.includes('class')) + assert.ok(classCall && classCall.q === 'KCg', 'the symbol was searched among classes') +}) + +test('a bare name that matches an instance exactly (label sans accession) is not lifted', async () => { + const deps = makeDeps('what is downstream of KCg-d_R?', 'KCg-d_R') + deps.runTool = (orig => async (name, args) => { + if (name === 'vfb_search_terms') { + deps.calls.push({ q: args.query, filter: args.filter_types || null }) + // A class search would find a sibling symbol; it must never be made. + if ((args.filter_types || []).includes('class')) return { results: [{ short_form: 'FBbt_00110932', label: 'KCg-d (gamma dorsal Kenyon cell)', original_label: 'gamma dorsal Kenyon cell', synonym: ['KCg-d'], facets_annotation: ['Entity', 'Class', 'Neuron'] }] } + return { results: KCG_INSTANCES } + } + if (name === 'vfb_get_term_info') return { Id: args.id, Name: 'KCg-d_R (FlyEM-HB:1003837184)', IsIndividual: true, SuperTypes: ['Individual', 'Neuron'], Publications: [], Queries: [] } + return orig(name, args) + })(deps.runTool) + const r = await runHarness('what is downstream of KCg-d_R?', deps) + assert.equal(r.ledger.terms['KCg-d_R'].id, 'VFB_jrchjw00') + assert.ok(!r.trace.some(e => e.resolve_lift_to_class), 'no lift') + assert.ok(!deps.calls.some(c => c.filter && c.filter.includes('class')), 'no class search') +}) + test('a name that IS the instance label is left on the instance', async () => { const deps = makeDeps('what is EPG-5L#3 (FAFB:4087066) connected to?', 'EPG-5L#3 (FAFB:4087066)') deps.runTool = (orig => async (name, args) => { diff --git a/tests/unit/classPartners.test.mjs b/tests/unit/classPartners.test.mjs index 4c76d73..0956b8d 100644 --- a/tests/unit/classPartners.test.mjs +++ b/tests/unit/classPartners.test.mjs @@ -333,6 +333,19 @@ test('the claim answers the question that was asked', () => { assert.equal(s.rows[0].id, 'FBbt_90000029') }) +test('every class the claim names is registrable: roll-ups and the self-row too', () => { + // The caller registers `rows` for linking. The roll-up classes are named in + // the claim and shown in the table, so an answer that mentions them must be + // able to link them — a DNp32 answer left "adult neuron, adult CNS neuron, + // adult interneuron" as its only unlinked names. + const s = summariseClassPartners(downstreamPayload(), { label: 'Kenyon cell' }) + const names = s.rows.map(r => r.name) + for (const r of s.aggregates) assert.ok(names.includes(r.label), `roll-up ${r.label} is in rows`) + for (const r of s.self) assert.ok(names.includes(r.label), `self ${r.label} is in rows`) + assert.ok(s.rows.every(r => r.id), 'every row carries an id') + assert.equal(s.rows.length, s.partners.length + s.aggregates.length + s.self.length) +}) + test('the collapsed names are surfaced in the claim, not swallowed', () => { const c = summariseClassPartners(downstreamPayload(), { label: 'Kenyon cell' }).claim assert.ok(/VFB lists the same connections under/.test(c), c) diff --git a/tests/unit/runQueryRetry.test.mjs b/tests/unit/runQueryRetry.test.mjs index abd2ea6..413d388 100644 --- a/tests/unit/runQueryRetry.test.mjs +++ b/tests/unit/runQueryRetry.test.mjs @@ -110,3 +110,25 @@ test('the default allowance is small and positive', () => { assert.ok(DEFAULT_FORCE_REFRESH_BUDGET > 0 && DEFAULT_FORCE_REFRESH_BUDGET <= 3) assert.equal(createForceRefreshBudget().remaining, DEFAULT_FORCE_REFRESH_BUDGET) }) + +// ------------------------------------------- suspicious class-connectivity zero + +import { isSuspiciousZeroRunQuery, SUSPICIOUS_ZERO_QUERY_TYPES } from '../../lib/runQueryRetry.mjs' + +test('an empty class-connectivity result is retried once, like a -1 (#66)', () => { + const zero = '{"count":0,"count_status":"exact","rows":[]}' + assert.equal(isSuspiciousZeroRunQuery('run_query', { id: 'FBbt_00100247', query_type: 'DownstreamClassConnectivity', limit: 2500 }, zero), true) + assert.equal(isSuspiciousZeroRunQuery('run_query', { id: 'FBbt_00100247', query_type: 'UpstreamClassConnectivity' }, { count: 0 }), true) + assert.deepEqual([...SUSPICIOUS_ZERO_QUERY_TYPES].sort(), ['DownstreamClassConnectivity', 'UpstreamClassConnectivity']) +}) + +test('every other zero is still a zero', () => { + const zero = '{"count":0,"count_status":"exact","rows":[]}' + assert.equal(isSuspiciousZeroRunQuery('run_query', { id: 'FBbt_00100247', query_type: 'NeuronsPartHere' }, zero), false) + assert.equal(isSuspiciousZeroRunQuery('get_term_info', { id: 'FBbt_00100247' }, zero), false) + assert.equal(isSuspiciousZeroRunQuery('run_query', { id: 'FBbt_00100247', query_type: 'DownstreamClassConnectivity' }, '{"count":3886,"rows":[{}]}'), false) + assert.equal(isSuspiciousZeroRunQuery('run_query', { id: 'FBbt_00100247', query_type: 'DownstreamClassConnectivity' }, '{"count":0,"rows":[{"id":"x"}]}'), false) + assert.equal(isSuspiciousZeroRunQuery('run_query', { id: 'FBbt_00100247', query_type: 'DownstreamClassConnectivity' }, '{"count":-1,"rows":[]}'), false) + assert.equal(isSuspiciousZeroRunQuery('run_query', { id: 'FBbt_00100247', query_type: 'DownstreamClassConnectivity' }, '{"error":"boom","count":0}'), false) + assert.equal(isSuspiciousZeroRunQuery('run_query', { id: 'FBbt_00100247', query_type: 'DownstreamClassConnectivity' }, 'not json'), false) +}) From 261840e7505a2ec5f9db693e6ea5e4f223836ea1 Mon Sep 17 00:00:00 2001 From: Robbie Court Date: Fri, 11 Sep 2026 01:54:56 +0000 Subject: [PATCH 3/3] Ask the user which reading was meant when a name matches several classes exactly "KCg" is the symbol of both "adult gamma Kenyon cell" (FBbt_00049828) and "gamma Kenyon cell" (FBbt_00100247), and exactTermMatchId handed back whichever VFB ranked first. When several class documents match the wording exactly on the same rung, the question decides if it names a stage; failing that the extract model is asked whether the question settles it (a dataset, a qualifier, an earlier mention) and told to answer -1 otherwise; failing that the turn becomes a clarifying question naming the readings, with one chip per reading that re-asks the question about that term's own label. Only classes tie: same-named individuals (five hemibrain cells labelled KCg-d_R) keep the answer-one-and-disclose-the-rest behaviour. Names the harness wrote itself (speculative) are never asked about. A label match still outranks synonym matches, so most names are not ties at all. Live: "what cell types are downstream of KCg?" now asks, with two chips; clicking "adult gamma Kenyon cell" answers with 3,370 downstream classes. "EPG neurons" still resolves without a question. Suite 1,387/1,389. --- app/api/chat/route.js | 33 +++++- lib/orchestrator.mjs | 141 ++++++++++++++++++++++++ tests/unit/clarifyReadingChips.test.mjs | 36 ++++++ tests/unit/classLift.test.mjs | 88 +++++++++++++++ 4 files changed, 297 insertions(+), 1 deletion(-) create mode 100644 tests/unit/clarifyReadingChips.test.mjs diff --git a/app/api/chat/route.js b/app/api/chat/route.js index c5c683b..9ddb897 100644 --- a/app/api/chat/route.js +++ b/app/api/chat/route.js @@ -11384,6 +11384,31 @@ async function localiseFollowOns(followOns, language, { apiBaseUrl, apiKey, apiM * a query type that reaches a URL, an id that is an id, a URL on a host the * outbound gate already allows. */ +/** + * One chip per reading the resolver could not choose between (#66 follow-up): + * the user's own question with the ambiguous name swapped for the reading's + * VFB label, so clicking it re-asks the question about exactly that term. When + * the name is not literally in the question (the planner paraphrased it) the + * label is prefixed instead, which the next turn's planner reads as the term. + * Exported for the unit test; no model, no network. + */ +export function clarifyReadingChips(question = '', options = []) { + const out = [] + const seen = new Set() + for (const o of Array.isArray(options) ? options : []) { + const label = String(o?.label || '').trim() + const name = String(o?.name || '').trim() + if (!label || seen.has(label.toLowerCase())) continue + seen.add(label.toLowerCase()) + const q = String(question || '').trim() + const re = name ? new RegExp(`(?= 6) break + } + return out +} + function previousFollowOnsFrom(rawMessages) { const last = [...(Array.isArray(rawMessages) ? rawMessages : [])].reverse() .find(m => m && m.role === 'assistant' && Array.isArray(m.followOns) && m.followOns.length) @@ -11588,13 +11613,19 @@ async function runRoleHarnessForRequest({ priorMessages, lastAssistantText = '', // answer. The Finglish question that got "Do you want to know how many // split-GAL4 driver lines…?" back in English is the case. const clarifyRendered = await renderInLanguage({ text: clarifyEnglish, language, kind: 'clarification', sendEvent, ...languageDeps }) + // When the resolver asked which of several exact readings was meant, the + // readings are offered as chips: each re-asks THIS question with the + // ambiguous name replaced by the term's own VFB label, which the next + // turn resolves exactly. Labels are localised like any other chip. + const readingChips = clarifyReadingChips(userMessage, live.ledger?.clarifyOptions) + const clarifyFollowOns = readingChips.length ? await localiseFollowOns(readingChips, language, languageDeps) : [] return { ok: true, responseText: clarifyRendered.text, images: [], graphs: [], tables: [], - followOns: [], + followOns: clarifyFollowOns, sources: [], // Even a clarifying turn carries the context forward. A question the // harness could not answer without more information is exactly the diff --git a/lib/orchestrator.mjs b/lib/orchestrator.mjs index 2c89cae..11aa3d9 100644 --- a/lib/orchestrator.mjs +++ b/lib/orchestrator.mjs @@ -403,6 +403,10 @@ export async function runHarness(question, deps) { const terms = speculative.length ? [...action.terms, ...speculative] : action.terms emit(deps, `Resolving ${terms.length} term${terms.length === 1 ? '' : 's'} in VFB`, 'mcp') await resolveTerms(ledger, terms, deps, models, log, new Set(speculative)) + // A name the resolver could not read one way (askWhichReading) turns + // the turn into a clarifying question: nothing below should plan, inject + // or run a step against a term the user has not yet chosen. + if (ledger.underspecified && ledger.clarifyingQuestion) continue displaceBackReferences(ledger, question, questionSymbols, log) // Deterministic graph routing: a connectivity/graph question about a single // resolved NEURON TYPE always runs the connectivity tool, so a graph appears @@ -939,6 +943,12 @@ async function resolveTerms(ledger, names, deps, models, log, speculative = new // nameVariants) and the loop stops at the first that is accepted, so the // usual single-plural case still costs exactly one extra search. let resolvedId = directId || pickBestTermId(search, name) + // The wording the chosen document was matched AGAINST — the name, or the + // variant, translation or symbol that replaced it below. The tie check at + // the end of the ladder asks which other documents match this wording + // exactly, and asking that of the original name after a variant matched + // would find nothing. + let matchedAs = name // Why the retry happened, in the vocabulary the trace already uses: the // three states are distinguishable and a resolution that came from a variant // is only explainable afterwards if the trace says which state it was in. @@ -977,6 +987,7 @@ async function resolveTerms(ledger, names, deps, models, log, speculative = new if (accepted) { search = retry resolvedId = accepted + matchedAs = exact ? variant : name log({ resolve_retry: name, as: variant, reason: retryReason }) break } @@ -1010,6 +1021,7 @@ async function resolveTerms(ledger, names, deps, models, log, speculative = new if (hit) { search = probed resolvedId = hit.id + matchedAs = hit.as || name log({ resolve_spelling: name, as: hit.as, id: hit.id, edits: hit.dist, probe }) break } @@ -1039,6 +1051,7 @@ async function resolveTerms(ledger, names, deps, models, log, speculative = new if (hit) { search = translated resolvedId = hit + matchedAs = english log({ resolve_translation: name, as: english, id: hit, language: ledger.language }) } else if (searchIsEmpty(search)) { // Candidates to show, at least: the English wording found @@ -1122,9 +1135,44 @@ async function resolveTerms(ledger, names, deps, models, log, speculative = new log({ resolve_lift_to_class: name, from: resolvedId, to: classId, as: symbol }) resolvedId = classId search = classSearch + matchedAs = symbol } } } + // TIED EXACT MATCHES. "KCg" is the symbol of BOTH "adult gamma Kenyon + // cell" (FBbt_00049828) and "gamma Kenyon cell" (FBbt_00100247), and + // exactTermMatchId hands back whichever VFB ranked first — a coin the + // user never saw tossed. When several documents match the wording + // exactly on the same rung, the question decides if it can (a stage + // word: "adult", "larval"); failing that the model is asked whether the + // question itself settles it; failing that the USER is asked, with the + // readings as chips, rather than answered about a term they may not + // have meant. A single exact match, or a clear winner between rungs + // (a label beats a synonym), is not a tie and none of this runs. + // A speculative name is one WE wrote, so the user cannot be asked which + // of its readings they meant; VFB's first stays. And only CLASSES tie: + // five hemibrain cells all labelled "KCg-d_R" are not readings a user + // can choose between by name — that case is answered about one and the + // others are disclosed (sameKindAlternatives, below). + const tied = speculative.has(name) ? [] : exactTermMatchDocs(search, matchedAs).filter(d => !docIsIndividual(d)) + if (tied.length > 1 && tied.some(d => sfOf(d) === resolvedId)) { + let pick = decideTiedReadingFromQuestion(ledger.question, tied) + let via = pick ? 'question' : '' + if (!pick && typeof deps.callStructured === 'function' && !budget.expired()) { + pick = await decideTiedReadingWithModel(ledger.question, name, tied, deps, models) + via = pick ? 'model' : '' + } + if (pick) { + if (sfOf(pick) !== resolvedId) resolvedId = sfOf(pick) + log({ resolve_disambiguated: name, as: matchedAs, to: resolvedId, via, among: tied.map(sfOf) }) + } else { + const options = tied.map(d => ({ id: sfOf(d), label: docLabel(d) })) + askWhichReading(ledger, name, options) + addTerm(ledger, name, { id: null, attempted: true, candidates: options.map(o => o.label), ambiguous: true, speculative: speculative.has(name) }) + log({ resolve_ambiguous: name, as: matchedAs, candidates: options.map(o => o.id) }) + return + } + } // A bare symbol that several same-kind terms carry ("DA1": DA1 PN, DA1 // lPN, DA1 vPN, MN-DA1) has been READ, not matched. The reading is used — // an answer beats a clarifying question here — but the answer must say @@ -4877,6 +4925,99 @@ export function exactTermMatchId(search, queryName = '') { return singExact ? sfOf(singExact) : null } +/** + * EVERY document that matches `queryName` exactly on the strongest rung any + * document reaches — the same three rungs as exactTermMatchId, and the same + * precedence, so the first of these is always what exactTermMatchId returns. + * Two documents here are a tie the ladder cannot break on its own: "KCg" is + * the symbol of both "adult gamma Kenyon cell" and "gamma Kenyon cell", and + * nothing in the wording prefers one. One document, or none, is not a tie. + * De-duplicated by id: VFB returns a row per matching synonym when asked to. + */ +export function exactTermMatchDocs(search, queryName = '') { + const q = norm(queryName) + if (!q) return [] + const valid = validSearchDocs(search) + if (!valid.length) return [] + const uniq = (docs) => { + const seen = new Set() + return docs.filter(d => { const id = sfOf(d); if (!id || seen.has(id)) return false; seen.add(id); return true }) + } + const byLabel = uniq(valid.filter(d => norm(docLabel(d)) === q)) + if (byLabel.length) return byLabel + const bySyn = uniq(valid.filter(d => docSynonyms(d).includes(q))) + if (bySyn.length) return bySyn + const qTokFull = toks(q) + if (!qTokFull.length) return [] + return uniq(valid.filter(d => sameTokenSet(toks(docLabel(d)), qTokFull))) +} + +const STAGE_WORDS = ['adult', 'larval', 'larva', 'embryonic', 'embryo', 'pupal', 'pupa'] +const stageIn = (text) => STAGE_WORDS.filter(w => new RegExp(`\\b${w}\\b`, 'i').test(String(text || ''))) +const sameStage = (a, b) => a.replace(/^(larva|embryo|pupa)$/, m => ({ larva: 'larval', embryo: 'embryonic', pupa: 'pupal' })[m]) === + b.replace(/^(larva|embryo|pupa)$/, m => ({ larva: 'larval', embryo: 'embryonic', pupa: 'pupal' })[m]) + +/** + * The one tied reading the QUESTION picks out, or null. Deterministic and + * narrow: a stage word in the question ("adult", "larval") selects the one + * candidate whose label carries that stage. A question that names no stage + * does not prefer the stage-agnostic class — that is exactly the choice the + * user is asked about. + */ +export function decideTiedReadingFromQuestion(question = '', docs = []) { + const asked = stageIn(question) + if (!asked.length) return null + const hits = docs.filter(d => stageIn(docLabel(d)).some(s => asked.some(a => sameStage(a.toLowerCase(), s.toLowerCase())))) + return hits.length === 1 ? hits[0] : null +} + +const TIED_READING_SCHEMA = { + type: 'object', + additionalProperties: false, + required: ['choice'], + properties: { choice: { type: 'integer' } } +} + +/** + * Ask the model whether the question itself settles which tied reading was + * meant. It may only choose on evidence in the question — a dataset, a stage, + * a qualifier, an earlier mention — and returns null (ask the user) otherwise. + * One structured call on the extract profile; any failure is "unsure". + */ +export async function decideTiedReadingWithModel(question, name, docs, deps, models = {}) { + const candidates = docs.map((d, i) => ({ index: i, label: docLabel(d) })) + const messages = [ + { + role: 'system', + content: 'A name in a user\'s question about Drosophila neuroanatomy matches more than one Virtual Fly Brain term exactly. Decide which term the user meant ONLY if the question itself decides it — a life stage, a dataset, a qualifier, or wording that fits exactly one candidate. If the question does not decide, answer -1 so the user can be asked. Never guess from which candidate seems more common. Reply with JSON {"choice": }.' + }, + { role: 'user', content: JSON.stringify({ question: String(question || ''), name: String(name), candidates }) } + ] + try { + const r = await deps.callStructured({ messages, schema: TIED_READING_SCHEMA, schemaName: 'tied_term_reading', model: models.extract }) + const i = r?.ok ? Number(r.value?.choice) : -1 + return Number.isInteger(i) && i >= 0 && i < docs.length ? docs[i] : null + } catch { + return null + } +} + +/** + * Turn an unbroken tie into the turn's clarifying question, with the readings + * carried as options for the follow-on chips. Written in English like the + * planner's own clarifying question, so it is rendered in the user's language + * by the same path. Labels only — an id in the text would be stripped as a + * leak, and is not what a reader chooses by. + */ +export function askWhichReading(ledger, name, options = []) { + const labels = options.map(o => o.label).filter(Boolean) + if (labels.length < 2) return + const list = labels.length === 2 ? `${labels[0]} or ${labels[1]}` : `${labels.slice(0, -1).join(', ')} or ${labels[labels.length - 1]}` + ledger.underspecified = true + ledger.clarifyingQuestion = `"${name}" matches more than one VFB term: ${list}. Which did you mean?` + ledger.clarifyOptions = [...(ledger.clarifyOptions || []), ...options.map(o => ({ name, id: o.id, label: o.label }))] +} + export function pickBestTermId(search, queryName = '') { const valid = validSearchDocs(search) if (!valid.length) return null diff --git a/tests/unit/clarifyReadingChips.test.mjs b/tests/unit/clarifyReadingChips.test.mjs new file mode 100644 index 0000000..4913105 --- /dev/null +++ b/tests/unit/clarifyReadingChips.test.mjs @@ -0,0 +1,36 @@ +// When the resolver asks which of several exact readings a name meant, the +// readings are offered as chips that re-ask the question about exactly that +// term. No model, no network. +// +// Run: node --test tests/unit/clarifyReadingChips.test.mjs + +import { test } from 'node:test' +import assert from 'node:assert/strict' + +import { clarifyReadingChips } from '../../app/api/chat/route.js' + +const OPTIONS = [ + { name: 'KCg', id: 'FBbt_00049828', label: 'adult gamma Kenyon cell' }, + { name: 'KCg', id: 'FBbt_00100247', label: 'gamma Kenyon cell' } +] + +test('each reading becomes a chip that re-asks the question with the label in place of the name', () => { + const chips = clarifyReadingChips('what cell types are downstream of KCg?', OPTIONS) + assert.deepEqual(chips.map(c => c.kind), ['ask', 'ask']) + assert.deepEqual(chips.map(c => c.label), ['adult gamma Kenyon cell', 'gamma Kenyon cell']) + assert.deepEqual(chips.map(c => c.query), [ + 'what cell types are downstream of adult gamma Kenyon cell?', + 'what cell types are downstream of gamma Kenyon cell?' + ]) +}) + +test('the name is replaced as a whole word only; a paraphrased question gets the label appended', () => { + const chips = clarifyReadingChips('what is downstream of KCg-d?', OPTIONS) + assert.equal(chips[0].query, 'what is downstream of KCg-d? (I mean adult gamma Kenyon cell)') + assert.equal(clarifyReadingChips('', OPTIONS)[1].query, 'gamma Kenyon cell') +}) + +test('duplicates and empties are dropped; nothing without options', () => { + assert.equal(clarifyReadingChips('q', [...OPTIONS, OPTIONS[0], { label: '' }]).length, 2) + assert.deepEqual(clarifyReadingChips('q', undefined), []) +}) diff --git a/tests/unit/classLift.test.mjs b/tests/unit/classLift.test.mjs index 89e5db1..cbaedbd 100644 --- a/tests/unit/classLift.test.mjs +++ b/tests/unit/classLift.test.mjs @@ -139,3 +139,91 @@ test('a name that IS the instance label is left on the instance', async () => { assert.equal(r.ledger.terms['EPG-5L#3 (FAFB:4087066)'].id, 'VFB_001012bq') assert.ok(!r.trace.some(e => e.resolve_lift_to_class), 'no lift') }) + +// --- tied exact readings ------------------------------------------------------ +// +// "KCg" is the symbol of BOTH "adult gamma Kenyon cell" and "gamma Kenyon cell". +// The question decides when it names a stage; the model is asked whether the +// question decides otherwise; and when neither can, the USER is asked, with the +// readings as options, rather than answered about VFB's first-ranked one. + +import { exactTermMatchDocs, decideTiedReadingFromQuestion, askWhichReading } from '../../lib/orchestrator.mjs' + +const KCG_ADULT = { short_form: 'FBbt_00049828', label: 'KCg (adult gamma Kenyon cell)', original_label: 'adult gamma Kenyon cell', synonym: ['KCg'], facets_annotation: ['Entity', 'Class', 'Neuron', 'Adult'] } +const KCG_ANY = { short_form: 'FBbt_00100247', label: 'KCg (gamma Kenyon cell)', original_label: 'gamma Kenyon cell', synonym: ['KCg', 'gamma KC'], facets_annotation: ['Entity', 'Class', 'Neuron'] } +const KCG_D = { short_form: 'FBbt_00110932', label: 'KCg-d (gamma dorsal Kenyon cell)', original_label: 'gamma dorsal Kenyon cell', synonym: ['KCg-d'], facets_annotation: ['Entity', 'Class', 'Neuron'] } + +test('exactTermMatchDocs returns every document on the strongest rung, and nothing weaker', () => { + const search = { results: [KCG_ADULT, KCG_ANY, KCG_D] } + assert.deepEqual(exactTermMatchDocs(search, 'KCg').map(d => d.short_form), ['FBbt_00049828', 'FBbt_00100247']) + // A label match outranks the synonym matches: no tie. + assert.deepEqual(exactTermMatchDocs({ results: [KCG_ANY, { ...KCG_D, original_label: 'KCg', label: 'KCg (FBbt_x)', short_form: 'FBbt_x' }] }, 'KCg').map(d => d.short_form), ['FBbt_x']) + assert.deepEqual(exactTermMatchDocs(search, 'KCg-d').map(d => d.short_form), ['FBbt_00110932']) + assert.deepEqual(exactTermMatchDocs(search, 'nothing'), []) +}) + +test('a stage word in the question decides a tie; no stage word decides nothing', () => { + assert.equal(decideTiedReadingFromQuestion('what is downstream of adult KCg?', [KCG_ADULT, KCG_ANY])?.short_form, 'FBbt_00049828') + assert.equal(decideTiedReadingFromQuestion('larval KCg outputs', [KCG_ADULT, KCG_ANY]), null) + assert.equal(decideTiedReadingFromQuestion('what cell types are downstream of KCg?', [KCG_ADULT, KCG_ANY]), null) +}) + +test('askWhichReading writes the clarifying question and the options', () => { + const ledger = {} + askWhichReading(ledger, 'KCg', [{ id: 'FBbt_00049828', label: 'adult gamma Kenyon cell' }, { id: 'FBbt_00100247', label: 'gamma Kenyon cell' }]) + assert.equal(ledger.underspecified, true) + assert.equal(ledger.clarifyingQuestion, '"KCg" matches more than one VFB term: adult gamma Kenyon cell or gamma Kenyon cell. Which did you mean?') + assert.deepEqual(ledger.clarifyOptions.map(o => o.id), ['FBbt_00049828', 'FBbt_00100247']) + assert.ok(!/FBbt_/.test(ledger.clarifyingQuestion), 'no ids in the question') +}) + +function tiedDeps(question, { decide = -1 } = {}) { + const deps = makeDeps(question, 'KCg') + const orig = deps.callStructured + deps.callStructured = async (req) => { + if (req.schemaName === 'tied_term_reading') { deps.calls.push({ tie: JSON.parse(req.messages[1].content) }); return { ok: true, value: { choice: decide } } } + return orig(req) + } + deps.runTool = async (name, args) => { + if (name === 'vfb_search_terms') { + deps.calls.push({ q: args.query, filter: args.filter_types || null }) + if ((args.filter_types || []).includes('class')) return { results: [KCG_ADULT, KCG_ANY, KCG_D] } + return { results: KCG_INSTANCES } + } + if (name === 'vfb_get_term_info') return { Id: args.id, Name: args.id === 'FBbt_00049828' ? 'adult gamma Kenyon cell' : 'gamma Kenyon cell', IsClass: true, SuperTypes: ['Class', 'Neuron'], Publications: [], Queries: [] } + return { ok: true } + } + return deps +} + +test('a tie the question and the model cannot break asks the user, with the readings as options', async () => { + const deps = tiedDeps('what cell types are downstream of KCg?') + const r = await runHarness('what cell types are downstream of KCg?', deps) + assert.equal(r.clarify, true) + assert.equal(r.answer, '"KCg" matches more than one VFB term: adult gamma Kenyon cell or gamma Kenyon cell. Which did you mean?') + assert.deepEqual(r.ledger.clarifyOptions.map(o => o.id), ['FBbt_00049828', 'FBbt_00100247']) + assert.equal(r.ledger.terms.KCg.id, null) + assert.equal(r.ledger.terms.KCg.ambiguous, true) + const tie = deps.calls.find(c => c.tie) + assert.ok(tie, 'the model was consulted') + assert.deepEqual(tie.tie.candidates.map(c => c.label), ['adult gamma Kenyon cell', 'gamma Kenyon cell']) + assert.ok(r.trace.some(e => e.resolve_ambiguous === 'KCg'), 'ambiguity logged') + assert.ok(!r.trace.some(e => e.run_step), 'no step ran against an unchosen term') +}) + +test('a tie the model can break from the question is not asked about', async () => { + const deps = tiedDeps('what cell types are downstream of KCg in the hemibrain?', { decide: 0 }) + const r = await runHarness('what cell types are downstream of KCg in the hemibrain?', deps) + assert.ok(!r.clarify) + assert.equal(r.ledger.terms.KCg.id, 'FBbt_00049828') + assert.ok(r.trace.some(e => e.resolve_disambiguated === 'KCg' && e.via === 'model'), 'model decision logged') +}) + +test('a stage word settles the tie before the model is consulted', async () => { + const deps = tiedDeps('what cell types are downstream of adult KCg?') + const r = await runHarness('what cell types are downstream of adult KCg?', deps) + assert.ok(!r.clarify) + assert.equal(r.ledger.terms.KCg.id, 'FBbt_00049828') + assert.ok(r.trace.some(e => e.resolve_disambiguated === 'KCg' && e.via === 'question')) + assert.ok(!deps.calls.some(c => c.tie), 'model not consulted') +})