diff --git a/backend/app/store.py b/backend/app/store.py index 13bf18a..b05ee91 100644 --- a/backend/app/store.py +++ b/backend/app/store.py @@ -79,11 +79,11 @@ def _application_data_dir() -> Path: Your final response must be exactly one JSON object: { - \"evaluation\": {\"score\":\"number from 0 to 10\",\"approval\":\"approved|rejected|pending\",\"summary\":\"string\",\"behavior_trace\": {\"persona_goal\":\"string\",\"expectation\":\"string\",\"interpretation\":\"string\",\"evidence\":\"string\",\"impact\":\"string\",\"next_step\":\"string\"}}, + \"evaluation\": {\"score\":\"number from 0 to 10\",\"approval\":\"approved|rejected|pending\",\"summary\":\"string\",\"behavior_trace\": {\"persona_goal\":\"string\",\"current_action\":\"string\",\"decision\":\"string\",\"next_action\":\"string\",\"evidence\":\"string\"}}, \"improvements\": [{\"title\":\"string\",\"status\":\"proposed|adopted|rejected\",\"rationale\":\"string\",\"acceptanceEvidence\":\"string\"}], \"reported_issues\": [{\"title\":\"string\",\"severity\":\"low|medium|high|critical\",\"evidence\":\"string\",\"reproduction\":\"string\",\"status\":\"open|acknowledged|resolved\"}] } -For an evaluated AI, include behavior_trace and fill every field. This is an evidence-backed persona journey, not the evaluator's procedure and not hidden reasoning: persona_goal is the persona's stated goal in this session; expectation is the information or reassurance the persona needs before safely proceeding; interpretation is the persona's concise, first-person reading of the rendered experience; evidence states only the observable source-backed facts that support that reading; impact states how the experience affects the persona's confidence or ability to continue; next_step is the specific safe next check or journey step. Use the persona's wording where useful, but do not invent motives, feelings, beliefs, or facts beyond the declared persona and observed evidence. Compare with the immediately previous iteration when that evidence is supplied. Do not narrate repeated mechanics such as navigation, waits, screenshots, or generic control inspection. Do not reveal hidden reasoning or evaluator chain-of-thought. Do not include behavior_trace for non-AI targets. behavior_summary is deprecated and should be omitted. Always include both array keys, using empty arrays when there are no items.""" +For an evaluated AI, include behavior_trace and fill every field. This is an evidence-backed persona journey, not the evaluator's procedure and not hidden reasoning. Keep the visible journey concise: persona_goal is the persona's stable, first-person wish in one sentence; current_action is a first-person sentence describing the one meaningful action the persona took in this iteration; decision is the persona's first-person judgment or resulting choice from that action; next_action is the one specific, safe next action in first person. Do not describe navigation, waits, screenshots, generic control inspection, or other repeated mechanics in any visible journey field. evidence is a concise source-backed factual record for the evidence drawer, not a visible journey item. Use the persona's wording where useful, but do not invent motives, feelings, beliefs, or facts beyond the declared persona and observed evidence. Do not reveal hidden reasoning or evaluator chain-of-thought. Do not include behavior_trace for non-AI targets. behavior_summary is deprecated and should be omitted. Always include both array keys, using empty arrays when there are no items.""" LEGACY_OPERATIONAL_MANAGER_PROMPT = """You are an approval-first operations manager for recurring AI evaluations. Preserve the task safety boundary, collect observable evidence, and never claim success without stated acceptance evidence. Escalate required approvals @@ -1192,9 +1192,12 @@ def display_fields(item: Any, fields: tuple[str, ...]) -> dict[str, str]: behavior_trace, ( "persona_goal", + "current_action", + "decision", + "next_action", + "evidence", "expectation", "interpretation", - "evidence", "impact", "next_step", "purpose", @@ -4568,7 +4571,20 @@ def _validated_supervisor_result(text: str) -> dict[str, Any]: raise ValueError("supervisor evaluation behavior_summary is invalid") if "behavior_trace" in evaluation: trace = evaluation["behavior_trace"] - persona_trace_fields = { + persona_journey_trace_fields = { + "persona_goal", + "current_action", + "decision", + "next_action", + "evidence", + } + compact_persona_trace_fields = { + "persona_goal", + "current_action", + "next_action", + "evidence", + } + expanded_persona_trace_fields = { "persona_goal", "expectation", "interpretation", @@ -4580,7 +4596,12 @@ def _validated_supervisor_result(text: str) -> dict[str, Any]: if ( not isinstance(trace, dict) or frozenset(trace) - not in {frozenset(persona_trace_fields), frozenset(legacy_trace_fields)} + not in { + frozenset(persona_journey_trace_fields), + frozenset(compact_persona_trace_fields), + frozenset(expanded_persona_trace_fields), + frozenset(legacy_trace_fields), + } or not all(isinstance(trace[field], str) and trace[field].strip() for field in trace) ): raise ValueError("supervisor evaluation behavior_trace is invalid") diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index a4b9721..3e3c5a3 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -880,6 +880,34 @@ def test_supervisor_result_normalizes_a_numeric_string_score(): def test_supervisor_result_accepts_a_persona_journey_trace(): + result = store_module.ConsoleStore._validated_supervisor_result( + '{"evaluation":{"score":8,"approval":"pending","summary":"ok","behavior_trace":' + '{"persona_goal":"I want to confirm my account is usable",' + '"current_action":"I checked whether my balance and holdings agree",' + '"decision":"I decided not to make a change while they disagree",' + '"next_action":"I will wait for the balance, then check my holdings",' + '"evidence":"The visible balance is still loading"}},' + '"improvements":[],"reported_issues":[]}' + ) + assert ( + result["evaluation"]["behavior_trace"]["current_action"] + == "I checked whether my balance and holdings agree" + ) + + +def test_supervisor_result_accepts_a_compact_persona_trace_for_existing_runs(): + result = store_module.ConsoleStore._validated_supervisor_result( + '{"evaluation":{"score":8,"approval":"pending","summary":"ok","behavior_trace":' + '{"persona_goal":"I want to confirm my account is usable",' + '"current_action":"I decided to wait for the balance",' + '"next_action":"I will check my holdings",' + '"evidence":"The visible balance is still loading"}},' + '"improvements":[],"reported_issues":[]}' + ) + assert result["evaluation"]["behavior_trace"]["current_action"] == "I decided to wait for the balance" + + +def test_supervisor_result_accepts_an_expanded_persona_trace_for_existing_runs(): result = store_module.ConsoleStore._validated_supervisor_result( '{"evaluation":{"score":8,"approval":"pending","summary":"ok","behavior_trace":' '{"persona_goal":"Confirm the account is usable","expectation":"A visible balance",' diff --git a/frontend/src/domain/models.ts b/frontend/src/domain/models.ts index bb7faa0..5e44fef 100644 --- a/frontend/src/domain/models.ts +++ b/frontend/src/domain/models.ts @@ -11,7 +11,7 @@ export type RunStepResult = { step_id:string; phase?:'before_all'|'before_each'| export type WorkflowGraphNode = { id:string; title:string; phase?:string|null; inputs?:string[]; outputs?:string[]; description?:string|null; status?:'idle'|'running'|'succeeded'|'failed'|'skipped' } export type WorkflowGraphEdge = { source:string; target:string; kind?:'execution'|'data'|'condition'|'loop'|'error'; label?:string|null; source_port?:string|null; target_port?:string|null } export type WorkflowGraphDefinition = { nodes:WorkflowGraphNode[]; edges:WorkflowGraphEdge[] } -export type BehaviorTrace = { persona_goal?:string; expectation?:string; interpretation?:string; evidence?:string; impact?:string; next_step?:string; purpose?:string; rationale?:string; observation?:string; decision?:string; next_action?:string } +export type BehaviorTrace = { persona_goal?:string; current_action?:string; next_action?:string; evidence?:string; expectation?:string; interpretation?:string; impact?:string; next_step?:string; purpose?:string; rationale?:string; observation?:string; decision?:string } export type SupervisorEvaluation = { score:number; approval:'approved'|'rejected'|'pending'; behavior_summary?:string; behavior_trace?:BehaviorTrace; summary:string } export type SupervisorResult = { evaluation?:SupervisorEvaluation; improvements:Record[]; reported_issues:Record[] } export type SupervisorRecord = { iteration:number; candidate_id?:string|null; status:'pending'|'completed'|'not_configured'|'invalid_response'|'failed'; prompt?:string; response?:SupervisorResult; error?:string; recorded_at?:string } diff --git a/frontend/src/features/evaluations/page.tsx b/frontend/src/features/evaluations/page.tsx index 927d372..2108606 100644 --- a/frontend/src/features/evaluations/page.tsx +++ b/frontend/src/features/evaluations/page.tsx @@ -85,7 +85,7 @@ const copy = localeMessageMap>("evaluations"); type SupervisorResultTranslation = { prompt?: string; response: { - evaluation: { behavior_summary?: string; behavior_trace?: { persona_goal?: string; expectation?: string; interpretation?: string; evidence?: string; impact?: string; next_step?: string; purpose?: string; rationale?: string; observation?: string; decision?: string; next_action?: string }; summary?: string }; + evaluation: { behavior_summary?: string; behavior_trace?: { persona_goal?: string; current_action?: string; next_action?: string; evidence?: string; expectation?: string; interpretation?: string; impact?: string; next_step?: string; purpose?: string; rationale?: string; observation?: string; decision?: string }; summary?: string }; improvements: Record[]; reported_issues: Record[]; }; diff --git a/frontend/src/features/evaluations/run-detail-result-panel.tsx b/frontend/src/features/evaluations/run-detail-result-panel.tsx index c2012f5..41596c2 100644 --- a/frontend/src/features/evaluations/run-detail-result-panel.tsx +++ b/frontend/src/features/evaluations/run-detail-result-panel.tsx @@ -1,4 +1,5 @@ -import type { ReactNode } from "react"; +import { useState, type ReactNode } from "react"; +import { FileSearch } from "lucide-react"; import { intlLocales, type Locale } from "../../locales"; type Messages = Record; @@ -7,7 +8,7 @@ type BehaviorTrace = { iteration: number; recordedAt?: string; summary?: string; - trace?: { persona_goal?: string; expectation?: string; interpretation?: string; evidence?: string; impact?: string; next_step?: string; purpose?: string; rationale?: string; observation?: string; decision?: string; next_action?: string }; + trace?: { persona_goal?: string; current_action?: string; decision?: string; next_action?: string; evidence?: string; expectation?: string; interpretation?: string; impact?: string; next_step?: string; purpose?: string; rationale?: string; observation?: string }; }; const time = (locale: Locale, value?: string) => value ? new Intl.DateTimeFormat(intlLocales[locale], { dateStyle: "medium", timeStyle: "medium" }).format(new Date(value)) : "—"; @@ -19,10 +20,20 @@ function ResultList({ items, kind, locale, empty }: { items: RecordItem[]; kind: } export function EvaluationResultPanel({ error, records, summaries, improvements, issues, l, locale }: { error?: ReactNode; records: unknown[]; summaries: BehaviorTrace[]; improvements: RecordItem[]; issues: RecordItem[]; l: Messages; locale: Locale }) { + const [evidenceIteration, setEvidenceIteration] = useState(null); const traceFields = (item: BehaviorTrace) => { if (!item.trace) return []; const trace = item.trace; - const fields = trace.persona_goal ? [ + const fields = trace.current_action && trace.decision ? [ + [l.tracePersonaGoal, trace.persona_goal], + [l.traceSessionAction, trace.current_action], + [l.traceCurrentDecision, trace.decision], + [l.traceNextAction, trace.next_action], + ] : trace.current_action ? [ + [l.tracePersonaGoal, trace.persona_goal], + [l.traceCurrentAction, trace.current_action], + [l.traceNextAction, trace.next_action], + ] : trace.persona_goal ? [ [l.tracePersonaGoal, trace.persona_goal], [l.traceExpectation, trace.expectation], [l.traceInterpretation, trace.interpretation], @@ -38,5 +49,5 @@ export function EvaluationResultPanel({ error, records, summaries, improvements, ]; return fields.filter((field): field is [string | undefined, string] => Boolean(field[1])); }; - return <>{error}{records.length ? <>{summaries.length > 0 &&

{l.observedBehavior}

{summaries.map((item) =>
{traceFields(item).length ?
{traceFields(item).map(([label, value]) =>
{label}
{value}
)}
:

{item.summary}

}
Iteration#{item.iteration}
)}
}

{l.proposals}

{l.issues}

:

{l.noMatchingResults}

}; + return <>{error}{records.length ? <>{summaries.length > 0 &&

{l.observedBehavior}

{summaries.map((item) =>
{traceFields(item).length ? <>
{traceFields(item).map(([label, value]) =>
{label}
{value}
)}
{evidenceIteration === item.iteration && item.trace?.evidence &&
{l.traceEvidence}

{item.trace.evidence}

} :

{item.summary}

}
Iteration#{item.iteration}{item.trace?.evidence && }
)}
}

{l.proposals}

{l.issues}

:

{l.noMatchingResults}

}; } diff --git a/frontend/src/locales/languages/en.json b/frontend/src/locales/languages/en.json index 8a57b1e..b817ab0 100644 --- a/frontend/src/locales/languages/en.json +++ b/frontend/src/locales/languages/en.json @@ -662,16 +662,21 @@ "proposals": "Proposed improvements", "observedBehavior": "Persona journey", "tracePersonaGoal": "Persona goal", + "traceCurrentAction": "Current action", + "traceSessionAction": "This session's action", + "traceCurrentDecision": "This session's decision", + "traceNextAction": "Next action", "traceExpectation": "Expectation", "traceInterpretation": "Persona interpretation", "traceEvidence": "Observed experience", "traceImpact": "User impact", "traceNextStep": "Next journey step", + "viewEvidence": "View evidence", + "hideEvidence": "Hide evidence", "tracePurpose": "Purpose", "traceRationale": "Rationale", "traceObservation": "Observation", "traceDecision": "Decision", - "traceNextAction": "Next action", "issues": "Reported issues", "status": "Run status", "phase": "Current phase", diff --git a/frontend/src/locales/languages/ja.json b/frontend/src/locales/languages/ja.json index a8ffadb..8fc7f1a 100644 --- a/frontend/src/locales/languages/ja.json +++ b/frontend/src/locales/languages/ja.json @@ -661,16 +661,21 @@ "proposals": "改善提案", "observedBehavior": "ペルソナジャーニー", "tracePersonaGoal": "ペルソナの目的", + "traceCurrentAction": "今回の行動", + "traceSessionAction": "今回したこと", + "traceCurrentDecision": "今回の判断", + "traceNextAction": "次の行動", "traceExpectation": "期待", "traceInterpretation": "ペルソナの解釈", "traceEvidence": "確認された体験", "traceImpact": "ユーザーへの影響", "traceNextStep": "次のジャーニー", + "viewEvidence": "証拠を見る", + "hideEvidence": "証拠を隠す", "tracePurpose": "目的", "traceRationale": "根拠", "traceObservation": "観測", "traceDecision": "判断", - "traceNextAction": "次の確認", "issues": "報告された問題", "status": "実行状態", "phase": "現在のフェーズ", diff --git a/frontend/src/locales/languages/ko.json b/frontend/src/locales/languages/ko.json index dd06b6a..c10b73c 100644 --- a/frontend/src/locales/languages/ko.json +++ b/frontend/src/locales/languages/ko.json @@ -661,16 +661,21 @@ "proposals": "개선 제안", "observedBehavior": "페르소나 여정", "tracePersonaGoal": "페르소나 목표", + "traceCurrentAction": "이번 행동", + "traceSessionAction": "이번에 한 일", + "traceCurrentDecision": "이번 판단", + "traceNextAction": "다음 행동", "traceExpectation": "기대", "traceInterpretation": "페르소나의 해석", "traceEvidence": "확인된 경험", "traceImpact": "사용자 영향", "traceNextStep": "다음 여정", + "viewEvidence": "증거 보기", + "hideEvidence": "증거 숨기기", "tracePurpose": "목적", "traceRationale": "근거", "traceObservation": "관찰", "traceDecision": "결정", - "traceNextAction": "다음 확인", "issues": "보고된 문제", "status": "실행 상태", "phase": "현재 단계",