diff --git a/ai/src/ai_server/chain/feedback_generation_chain.py b/ai/src/ai_server/chain/feedback_generation_chain.py index 4b4cf82c..147444ff 100644 --- a/ai/src/ai_server/chain/feedback_generation_chain.py +++ b/ai/src/ai_server/chain/feedback_generation_chain.py @@ -1,17 +1,23 @@ from __future__ import annotations +import asyncio +from dataclasses import dataclass from typing import Protocol +import structlog from langchain_core.output_parsers import PydanticOutputParser from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import Runnable from pydantic import BaseModel, Field from ai_server.chain.prompts.feedback_generation import HUMAN_PROMPT, SYSTEM_PROMPT +from ai_server.chain.prompts import feedback_panel from ai_server.config.settings import Settings from ai_server.core.client import CoreClient from ai_server.observability.llm_logging_callback import CoreAiLogCallback +log = structlog.get_logger(__name__) + class FeedbackResult(BaseModel): overall_score: float | None = Field(None, description="0~100") @@ -105,3 +111,221 @@ def build_feedback_generation_chain( callbacks=callbacks, ) return prompt | llm | parser + + +# ── 멀티 면접관 패널 ────────────────────────────────────────────────────────── +# 단일 평가자 대신 직군·논리·커뮤니케이션 평가위원이 각자 한 축을 채점(병렬) → +# 가중평균으로 종합. A=평가만 / B=직군별(+단일직군도 다관점) / C=가중평균 / D=프롬프트 멀티콜. + + +class EvaluatorResult(BaseModel): + score: float | None = Field(None, description="0~100, 산정 불가 시 null") + strength: str | None = None + weakness: str | None = None + keywords: list[str] = Field(default_factory=list) + + +@dataclass(frozen=True) +class _EvaluatorSpec: + key: str # 'technical' | 'logic' | 'communication' + label: str # 요약 표기용 ('기술'/'인성'/'논리'/'전달') + persona: str + dimension_name: str + dimension_guide: str + + +def _domain_spec(job_category: str, mode: str) -> _EvaluatorSpec: + # PERSONALITY 모드는 기술 평가자를 인성·협업 평가자로 교체(사용자 결정). + if (mode or "").upper() == "PERSONALITY": + return _EvaluatorSpec( + key="technical", + label="인성", + persona="인성·협업 중심 면접관", + dimension_name="인성·협업 역량", + dimension_guide=( + "- 협업/갈등 해결, 성장 경험, 태도, 자기주도성을 봅니다. " + "기술 정확도는 평가하지 않습니다." + ), + ) + return _EvaluatorSpec( + key="technical", + label="기술", + persona=f"{job_category} 직군 시니어 기술 면접관", + dimension_name="기술 정확도·깊이", + dimension_guide=( + "- 기술 정확성, 깊이, trade-off, 근거를 봅니다. 질문의 '기대 신호'를 " + "답변이 얼마나 짚었는지를 핵심 근거로 삼습니다." + ), + ) + + +_LOGIC_SPEC = _EvaluatorSpec( + key="logic", + label="논리", + persona="논리·문제해결 평가위원", + dimension_name="논리·인과관계 명확성", + dimension_guide=( + "- 주장→근거→결론의 인과, trade-off 설명의 일관성, 문제 구조화를 봅니다." + ), +) + +_COMM_SPEC = _EvaluatorSpec( + key="communication", + label="전달", + persona="커뮤니케이션·전달력 평가위원", + dimension_name="명료성·구조화·전달력", + dimension_guide=( + "- 답변의 구조(STAR 등)·간결성·명료성을 보고, 음성 분석(WPM·무음·간투어)이 " + "있으면 전달력 판단에 적극 활용합니다." + ), +) + + +def build_panel_evaluator_chain( + settings: Settings, core_client: CoreClient | None = None +) -> Runnable: + """패널 평가위원 1명용 체인. persona/dimension 을 invoke 변수로 받아 N회 재사용.""" + from langchain_openai import ChatOpenAI + + parser = PydanticOutputParser(pydantic_object=EvaluatorResult) + prompt = ChatPromptTemplate.from_messages( + [ + ("system", feedback_panel.SYSTEM_PROMPT), + ("human", feedback_panel.HUMAN_PROMPT), + ] + ).partial(format_instructions=parser.get_format_instructions()) + + callbacks = [] + if core_client is not None: + callbacks.append( + CoreAiLogCallback( + core_client=core_client, + request_type="generate.feedback.panel", + default_model=settings.llm_pro_model, + ) + ) + + llm = ChatOpenAI( + model=settings.llm_pro_model, + temperature=settings.llm_pro_temperature, + api_key=settings.llm_api_key or None, + base_url=settings.llm_base_url, + callbacks=callbacks, + ) + return prompt | llm | parser + + +def _weighted_overall(pairs: list[tuple[float | None, float]]) -> float | None: + """(score, weight) 중 score 가 있는 것만 가중평균. 전부 None 이면 None.""" + present = [(s, w) for s, w in pairs if s is not None and w > 0] + if not present: + return None + total_w = sum(w for _, w in present) + return round(sum(s * w for s, w in present) / total_w) + + +def _merge_notes(items: list[tuple[str, str | None]]) -> str | None: + parts = [f"[{label}] {note.strip()}" for label, note in items if note and note.strip()] + return " ".join(parts) if parts else None + + +def _dedup_keywords(keywords: list[str], cap: int = 8) -> list[str]: + seen: set[str] = set() + out: list[str] = [] + for kw in keywords: + k = (kw or "").strip() + if k and k not in seen: + seen.add(k) + out.append(k) + if len(out) >= cap: + break + return out + + +class PanelFeedbackGenerator: + """직군·논리·커뮤니케이션 평가위원을 병렬 호출 → 가중평균 종합. FeedbackGenerator 호환.""" + + def __init__(self, chain: Runnable, *, weights: tuple[float, float, float] = (0.5, 0.25, 0.25)) -> None: + self._chain = chain + self._w_tech, self._w_logic, self._w_comm = weights + + async def generate( + self, + *, + job_category: str, + mode: str, + total_question_count: int | None, + end_reason: str | None, + transcript: str, + rag_context: str, + voice_analysis_summary: str = "", + score_basis: str = "(없음)", + ) -> FeedbackResult: + specs = [_domain_spec(job_category, mode), _LOGIC_SPEC, _COMM_SPEC] + shared = { + "job_category": job_category, + "mode": mode, + "total_question_count": total_question_count or 0, + "end_reason": end_reason or "USER_REQUEST", + "transcript": transcript, + "score_basis": score_basis or "(없음)", + "rag_context": rag_context or "(none)", + "voice_analysis_summary": voice_analysis_summary + or "No voice analysis summary was provided.", + } + raw = await asyncio.gather( + *( + self._chain.ainvoke( + { + **shared, + "persona": s.persona, + "dimension_name": s.dimension_name, + "dimension_guide": s.dimension_guide, + } + ) + for s in specs + ), + return_exceptions=True, + ) + + results: dict[str, EvaluatorResult] = {} + for spec, r in zip(specs, raw): + if isinstance(r, EvaluatorResult): + results[spec.key] = r + else: + log.warning( + "feedback.panel.evaluator_failed", + evaluator=spec.key, + error=str(r), + ) + results[spec.key] = EvaluatorResult() + + tech = results["technical"] + logic = results["logic"] + comm = results["communication"] + domain_label = specs[0].label + + overall = _weighted_overall( + [ + (tech.score, self._w_tech), + (logic.score, self._w_logic), + (comm.score, self._w_comm), + ] + ) + strengths = _merge_notes( + [(domain_label, tech.strength), ("논리", logic.strength), ("전달", comm.strength)] + ) + weaknesses = _merge_notes( + [(domain_label, tech.weakness), ("논리", logic.weakness), ("전달", comm.weakness)] + ) + keywords = _dedup_keywords(tech.keywords + logic.keywords + comm.keywords) + + return FeedbackResult( + overall_score=overall, + technical_accuracy=tech.score, + logic_score=logic.score, + communication_score=comm.score, + strengths_summary=strengths, + weaknesses_summary=weaknesses, + improvement_keywords=keywords, + ) diff --git a/ai/src/ai_server/chain/prompts/feedback_panel.py b/ai/src/ai_server/chain/prompts/feedback_panel.py new file mode 100644 index 00000000..4929ae3c --- /dev/null +++ b/ai/src/ai_server/chain/prompts/feedback_panel.py @@ -0,0 +1,28 @@ +# 멀티 면접관 패널 — 단일 평가위원 프롬프트. +# persona/평가축(dimension)을 변수로 주입해 같은 체인을 N회(직군·논리·커뮤) 호출한다. + +SYSTEM_PROMPT = ( + "당신은 IT 직군 면접 평가 패널의 한 평가위원입니다.\n" + "역할: {persona}\n" + "당신은 **오직 '{dimension_name}' 한 축만** 평가합니다. 다른 축은 평가하지 마세요.\n" + "{dimension_guide}\n" + "- 점수는 0~100 정수, 산정 불가(짧거나 빈 답변 등) 시 null.\n" + "- 점수 앵커: 90~100 정확·구체적이며 근거·trade-off까지 깊이 있음 / " + "70~89 대체로 정확하나 일부 깊이·근거 부족 / 50~69 방향은 맞으나 추상적 / " + "30~49 부분적으로만 타당하고 핵심 누락 多 / 0~29 부정확하거나 거의 무응답.\n" + "- '점수 기준값(score_basis)'에 해당 축 기준값이 있으면 그 값에서 ±15점 이내로 산정한다.\n" + "- 점수를 매기기 전에 강점/약점 근거를 먼저 정리한 뒤 산정한다(즉흥 점수 금지).\n" + "- strength/weakness 는 각각 한 줄(한국어, 구체적으로). keywords 는 이 축에서 보완할 " + "개선 키워드 0~4개(짧은 명사구).\n" + "- 응답은 반드시 지정된 JSON 스키마를 따른다." +) + +HUMAN_PROMPT = ( + "직군: {job_category} / 면접 모드: {mode} / 질문 수: {total_question_count} / " + "종료 사유: {end_reason}\n\n" + "=== 면접 전사 ===\n{transcript}\n\n" + "=== 점수 기준값 (per-answer 평가 집계) ===\n{score_basis}\n\n" + "=== 참고 문서 컨텍스트(RAG) ===\n{rag_context}\n\n" + "=== 음성 분석 ===\n{voice_analysis_summary}\n\n" + "{format_instructions}" +) diff --git a/ai/src/ai_server/messaging/runner.py b/ai/src/ai_server/messaging/runner.py index 372a54bd..19c15818 100644 --- a/ai/src/ai_server/messaging/runner.py +++ b/ai/src/ai_server/messaging/runner.py @@ -20,8 +20,8 @@ build_streaming_followup_generator, ) from ai_server.chain.feedback_generation_chain import ( - LlmFeedbackGenerator, - build_feedback_generation_chain, + PanelFeedbackGenerator, + build_panel_evaluator_chain, ) from ai_server.chain.question_generation_chain import ( LlmQuestionGenerator, @@ -210,9 +210,9 @@ def __init__(self, settings: Settings) -> None: rag_timeout_sec=settings.followup_rag_timeout_sec, ) - # 종합 피드백 생성 (US-24) - feedback_generator = LlmFeedbackGenerator( - build_feedback_generation_chain(settings, core_client=core_client) + # 종합 피드백 생성 (US-24) — 멀티 면접관 패널(직군·논리·커뮤 평가위원 병렬 → 가중평균) + feedback_generator = PanelFeedbackGenerator( + build_panel_evaluator_chain(settings, core_client=core_client) ) self._feedback_consumer = FeedbackConsumer( generator=feedback_generator, diff --git a/ai/tests/test_feedback_panel.py b/ai/tests/test_feedback_panel.py new file mode 100644 index 00000000..2737b7bc --- /dev/null +++ b/ai/tests/test_feedback_panel.py @@ -0,0 +1,96 @@ +import pytest + +from ai_server.chain.feedback_generation_chain import ( + EvaluatorResult, + PanelFeedbackGenerator, +) + +# 평가축(dimension_name) 으로 라우팅하는 가짜 체인. +TECH = "기술 정확도·깊이" +PERSONALITY = "인성·협업 역량" +LOGIC = "논리·인과관계 명확성" +COMM = "명료성·구조화·전달력" + + +class _FakeChain: + def __init__(self, by_dim: dict[str, EvaluatorResult]): + self._by_dim = by_dim + self.calls: list[str] = [] + + async def ainvoke(self, variables): + dim = variables["dimension_name"] + self.calls.append(dim) + return self._by_dim[dim] + + +async def _run(by_dim, **kw): + gen = PanelFeedbackGenerator(_FakeChain(by_dim)) + return await gen.generate( + job_category=kw.get("job_category", "BACKEND"), + mode=kw.get("mode", "TECHNICAL"), + total_question_count=5, + end_reason="MAX_QUESTIONS_REACHED", + transcript="t", + rag_context="(none)", + voice_analysis_summary="", + score_basis="(없음)", + ) + + +@pytest.mark.asyncio +async def test_weighted_overall_and_dimension_mapping(): + r = await _run( + { + TECH: EvaluatorResult(score=80, strength="설계 깊이", keywords=["JPA"]), + LOGIC: EvaluatorResult(score=60, strength="인과 명확", keywords=["trade-off"]), + COMM: EvaluatorResult(score=40, strength="간결", keywords=["STAR"]), + } + ) + assert r.technical_accuracy == 80 + assert r.logic_score == 60 + assert r.communication_score == 40 + # 0.5*80 + 0.25*60 + 0.25*40 = 65 + assert r.overall_score == 65 + assert "[기술]" in r.strengths_summary and "[논리]" in r.strengths_summary + assert set(r.improvement_keywords) == {"JPA", "trade-off", "STAR"} + + +@pytest.mark.asyncio +async def test_overall_reweights_when_a_dimension_is_null(): + r = await _run( + { + TECH: EvaluatorResult(score=80), + LOGIC: EvaluatorResult(score=None), + COMM: EvaluatorResult(score=40), + } + ) + # logic None → (80*0.5 + 40*0.25) / 0.75 = 66.67 → 67 + assert r.logic_score is None + assert r.overall_score == 67 + + +@pytest.mark.asyncio +async def test_personality_mode_swaps_domain_to_behavioral(): + r = await _run( + { + PERSONALITY: EvaluatorResult(score=70, strength="협업 태도 우수"), + LOGIC: EvaluatorResult(score=50), + COMM: EvaluatorResult(score=60), + }, + mode="PERSONALITY", + ) + # 기술 평가자 자리가 인성·협업 평가자로 교체됨 → technical_accuracy 슬롯에 인성 점수 + assert r.technical_accuracy == 70 + assert "[인성]" in r.strengths_summary + + +@pytest.mark.asyncio +async def test_keyword_dedup(): + r = await _run( + { + TECH: EvaluatorResult(score=70, keywords=["동시성", "트랜잭션"]), + LOGIC: EvaluatorResult(score=70, keywords=["트랜잭션"]), + COMM: EvaluatorResult(score=70, keywords=["두괄식"]), + } + ) + assert r.improvement_keywords == ["동시성", "트랜잭션", "두괄식"]